sbmt-outbox 5.0.0

data/app/models/sbmt/outbox/base_item_config.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class BaseItemConfig
+      DEFAULT_BUCKET_SIZE = 16
+      DEFAULT_PARTITION_STRATEGY = :number
+
+      def initialize(box_name)
+        self.box_name = box_name
+
+        validate!
+      end
+
+      def owner
+        return @owner if defined? @owner
+
+        @owner = options[:owner].presence || Outbox.yaml_config[:owner].presence
+      end
+
+      def bucket_size
+        @bucket_size ||= (options[:bucket_size] || Outbox.yaml_config.fetch(:bucket_size, DEFAULT_BUCKET_SIZE)).to_i
+      end
+
+      def partition_size
+        @partition_size ||= (options[:partition_size] || 1).to_i
+      end
+
+      def retention
+        @retention ||= ActiveSupport::Duration.parse(options[:retention] || "P1W")
+      end
+
+      def max_retries
+        @max_retries ||= (options[:max_retries] || 0).to_i
+      end
+
+      def minimal_retry_interval
+        @minimal_retry_interval ||= (options[:minimal_retry_interval] || 10).to_i
+      end
+
+      def maximal_retry_interval
+        @maximal_retry_interval ||= (options[:maximal_retry_interval] || 600).to_i
+      end
+
+      def multiplier_retry_interval
+        @multiplier_retry_interval ||= (options[:multiplier_retry_interval] || 2).to_i
+      end
+
+      def retry_strategies
+        @retry_strategies ||= Array.wrap(options[:retry_strategies]).map do |str_name|
+          "Sbmt::Outbox::RetryStrategies::#{str_name.camelize}".constantize
+        end
+      end
+
+      def partition_strategy
+        return @partition_strategy if defined?(@partition_strategy)
+
+        str_name = options.fetch(:partition_strategy, DEFAULT_PARTITION_STRATEGY)
+        @partition_strategy = "Sbmt::Outbox::PartitionStrategies::#{str_name.camelize}Partitioning".constantize
+      end
+
+      def transports
+        return @transports if defined?(@transports)
+
+        values = options.fetch(:transports, [])
+
+        if values.is_a?(Hash)
+          values = values.each_with_object([]) do |(key, params), memo|
+            memo << params.merge!(class: key)
+          end
+        end
+
+        @transports = values.each_with_object({}) do |params, memo|
+          params = params.symbolize_keys
+          event_name = params.delete(:event_name) || :_all_
+          memo[event_name] ||= []
+          namespace = params.delete(:class)&.camelize
+          raise ArgumentError, "Transport name cannot be blank" if namespace.blank?
+
+          factory = "#{namespace}::OutboxTransportFactory".safe_constantize
+          memo[event_name] << if factory
+            factory.build(**params.symbolize_keys)
+          else
+            namespace.constantize.new(**params.symbolize_keys)
+          end
+        end
+      end
+
+      private
+
+      attr_accessor :box_name
+
+      def options
+        @options ||= lookup_config || {}
+      end
+
+      def lookup_config
+        raise NotImplementedError
+      end
+
+      def validate!
+        raise ConfigError, "Bucket size should be greater or equal to partition size" if partition_size > bucket_size
+      end
+    end
+  end
+end

data/app/models/sbmt/outbox/inbox_item.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class InboxItem < BaseItem
+      self.abstract_class = true
+
+      class << self
+        alias_method :inbox_name, :box_name
+
+        def box_type
+          :inbox
+        end
+
+        def lookup_config
+          Sbmt::Outbox::InboxItemConfig
+        end
+      end
+
+      delegate :inbox_name, :config, to: "self.class"
+
+      private
+
+      def default_options
+        {}
+      end
+
+      def default_log_details
+        {
+          uuid: uuid,
+          status: status,
+          created_at: created_at.to_datetime.rfc3339(6),
+          errors_count: errors_count
+        }
+      end
+    end
+  end
+end

data/app/models/sbmt/outbox/inbox_item_config.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class InboxItemConfig < BaseItemConfig
+      private
+
+      def lookup_config
+        Outbox.yaml_config.dig(:inbox_items, box_name)
+      end
+    end
+  end
+end

data/app/models/sbmt/outbox/outbox_item.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class OutboxItem < BaseItem
+      self.abstract_class = true
+
+      IDEMPOTENCY_HEADER_NAME = "Idempotency-Key"
+      SEQUENCE_HEADER_NAME = "Sequence-ID"
+      EVENT_TIME_HEADER_NAME = "Created-At"
+      OUTBOX_HEADER_NAME = "Outbox-Name"
+      DISPATCH_TIME_HEADER_NAME = "Dispatched-At"
+
+      class << self
+        alias_method :outbox_name, :box_name
+
+        def box_type
+          :outbox
+        end
+
+        def lookup_config
+          Sbmt::Outbox::OutboxItemConfig
+        end
+      end
+
+      delegate :outbox_name, :config, to: "self.class"
+
+      private
+
+      def default_options
+        {
+          headers: {
+            OUTBOX_HEADER_NAME => outbox_name,
+            IDEMPOTENCY_HEADER_NAME => uuid,
+            SEQUENCE_HEADER_NAME => id.to_s,
+            EVENT_TIME_HEADER_NAME => created_at&.to_datetime&.rfc3339(6),
+            DISPATCH_TIME_HEADER_NAME => Time.current.to_datetime.rfc3339(6)
+          }
+        }
+      end
+
+      def default_log_details
+        {
+          uuid: uuid,
+          status: status,
+          created_at: created_at.to_datetime.rfc3339(6),
+          errors_count: errors_count
+        }
+      end
+    end
+  end
+end

data/app/models/sbmt/outbox/outbox_item_config.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class OutboxItemConfig < BaseItemConfig
+      private
+
+      def lookup_config
+        Outbox.yaml_config.dig(:outbox_items, box_name) || Outbox.yaml_config.dig(:items, box_name)
+      end
+    end
+  end
+end

data/config/initializers/schked.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+begin
+  require "schked"
+
+  Schked.config.paths << Sbmt::Outbox::Engine.root.join("config", "schedule.rb")
+rescue LoadError
+  # optional dependency
+end

data/config/initializers/yabeda.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+Yabeda.configure do
+  # error_counter retry_counter sent_counter fetch_error_counter discarded_counter
+  group :outbox do
+    counter :created_counter,
+      tags: %i[type name partition owner],
+      comment: "The total number of created messages"
+
+    counter :sent_counter,
+      tags: %i[type name partition owner],
+      comment: "The total number of processed messages"
+
+    counter :error_counter,
+      tags: %i[type name partition owner],
+      comment: "Errors (excepting retries) that occurred while processing messages"
+
+    counter :retry_counter,
+      tags: %i[type name partition owner],
+      comment: "Retries that occurred while processing messages"
+
+    counter :discarded_counter,
+      tags: %i[type name partition owner],
+      comment: "The total number of discarded messages"
+
+    counter :fetch_error_counter,
+      tags: %i[type name partition owner],
+      comment: "Errors that occurred while fetching messages"
+
+    gauge :last_stored_event_id,
+      tags: %i[type name partition owner],
+      comment: "The ID of the last stored event"
+
+    gauge :last_sent_event_id,
+      tags: %i[type name partition owner],
+      comment: "The ID of the last sent event. " \
+                "If the message order is not preserved, the value may be inaccurate"
+
+    histogram :process_latency,
+      tags: %i[type name partition owner],
+      unit: :seconds,
+      buckets: [1, 2.5, 5, 10, 15, 30, 45, 60, 90, 120, 180, 240, 300, 600, 1200].freeze,
+      comment: "A histogram outbox process latency"
+  end
+
+  group :box_worker do
+    counter :job_counter,
+      tags: %i[type name partition worker_number state],
+      comment: "The total number of processed jobs"
+
+    counter :job_timeout_counter,
+      tags: %i[type name partition_key worker_number],
+      comment: "Requeue of a job that occurred while processing the batch"
+
+    counter :job_items_counter,
+      tags: %i[type name partition worker_number],
+      comment: "The total number of processed items in jobs"
+
+    histogram :job_execution_runtime,
+      comment: "A histogram of the job execution time",
+      unit: :seconds,
+      tags: %i[type name partition worker_number],
+      buckets: [0.5, 1, 2.5, 5, 10, 15, 30, 45, 60, 90, 120, 180, 240, 300, 600]
+
+    histogram :item_execution_runtime,
+      comment: "A histogram of the item execution time",
+      unit: :seconds,
+      tags: %i[type name partition worker_number],
+      buckets: [0.5, 1, 2.5, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 240, 300]
+  end
+end

data/config/schedule.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+every "10m", as: "Sbmt::Outbox::DeleteStaleOutboxItemsJob", overlap: false, timeout: "60s" do
+  Sbmt::Outbox::DeleteStaleOutboxItemsJob.enqueue
+end
+
+every "10m", as: "Sbmt::Outbox::DeleteStaleInboxItemsJob", overlap: false, timeout: "60s" do
+  Sbmt::Outbox::DeleteStaleInboxItemsJob.enqueue
+end

data/exe/outbox

@@ -0,0 +1,16 @@
+#!/usr/local/bin/ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "sbmt/outbox/cli"
+
+# rubocop:disable Lint/RescueException
+begin
+  Sbmt::Outbox::CLI.start(ARGV)
+rescue Exception => e
+  warn "Outbox exited with error"
+  warn(e.message) if e.respond_to?(:message)
+  warn(e.backtrace.join("\n")) if e.respond_to?(:backtrace) && e.backtrace.respond_to?(:join)
+  exit 1
+end
+# rubocop:enable Lint/RescueException

data/lib/generators/helpers/config.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Outbox
+  module Generators
+    module Helpers
+      module Config
+        CONFIG_PATH = "config/outbox.yml"
+
+        private
+
+        def config_exists?
+          File.exist?(CONFIG_PATH)
+        end
+
+        def check_config!
+          return if config_exists?
+
+          if yes?("Seems like `config/outbox.yml` doesn't exist. Would you like to generate it?")
+            generate "outbox:install"
+          else
+            raise Rails::Generators::Error, "Something went wrong: `config/outbox.yml` is missing. " \
+                                        "Please generate one by running `bin/rails g outbox:install` " \
+                                        "or add it manually."
+          end
+        end
+
+        def add_item_to_config(config_block_name, item_template_data)
+          template_data_with_parent = <<~RUBY
+            #{config_block_name}:
+            #{optimize_indentation(item_template_data, 2)}
+          RUBY
+
+          data, after = if File.binread(CONFIG_PATH).match?(/^\s*#{config_block_name}:/)
+            # if config already contains non-empty/non-commented-out inbox_items/outbox_items block
+            [optimize_indentation(item_template_data, 4), /^\s*#{config_block_name}:\s*\n/]
+          else
+            # there is no config for our items
+            # so we just set it up initially
+            [optimize_indentation(template_data_with_parent, 2), /^default:.+?\n/]
+          end
+          inject_into_file CONFIG_PATH, data, after: after
+        end
+      end
+    end
+  end
+end

data/lib/generators/helpers/initializer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Outbox
+  module Generators
+    module Helpers
+      module Initializer
+        OUTBOX_INITIALIZER_PATH = "config/initializers/outbox.rb"
+
+        private
+
+        def add_item_to_initializer(attr_name)
+          template_data_with_push = <<~RUBY
+            "#{namespaced_item_class_name}",
+          RUBY
+
+          template_data_with_append = <<~RUBY
+            #{attr_name} << "#{namespaced_item_class_name}"
+          RUBY
+
+          initial_template_data = <<~RUBY
+            #{attr_name}.push(
+              "#{namespaced_item_class_name}"
+            )
+
+          RUBY
+
+          content = File.binread(OUTBOX_INITIALIZER_PATH)
+          data, after = if content.match?(/^\s*#{attr_name}\.push/)
+            [optimize_indentation(template_data_with_push, 4), /^\s*#{attr_name}\.push\(\n/]
+          elsif content.match?(/^\s*#{attr_name}\s+<</)
+            [optimize_indentation(template_data_with_append, 2), /^\s*#{attr_name}\s+<< ".+?\n/]
+          else
+            # there is no config for items, so set it up initially
+            [optimize_indentation(initial_template_data, 2), /^Rails.application.config.outbox.tap do.+?\n/]
+          end
+          inject_into_file OUTBOX_INITIALIZER_PATH, data, after: after
+        end
+      end
+    end
+  end
+end

data/lib/generators/helpers/items.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Outbox
+  module Generators
+    module Helpers
+      module Items
+        def namespaced_item_class_name
+          file_path.camelize
+        end
+
+        def item_path
+          file_path
+        end
+      end
+    end
+  end
+end

data/lib/generators/helpers/migration.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Outbox
+  module Generators
+    module Helpers
+      module Migration
+        private
+
+        def create_migration_file(migration_class_name, migration_table_name)
+          return false if find_existing_migration(migration_class_name.tableize)
+
+          result = generate "rails:migration", migration_class_name, "--no-timestamps"
+          return unless result
+
+          migration_filepath = find_existing_migration(migration_class_name.tableize)
+          return unless migration_filepath
+
+          patch_migration_with_template_data(migration_filepath, migration_table_name)
+        end
+
+        def find_existing_migration(name)
+          base_path = "db/migrate"
+          found_files = Dir.glob("*_#{name}.rb", base: base_path)
+          return if found_files.size != 1
+
+          "#{base_path}/#{found_files[0]}"
+        end
+
+        def patch_migration_with_template_data(migration_filepath, table_name)
+          data_to_replace = /^\s*create_table :#{table_name}.+?end\n/m
+
+          template_data = <<~RUBY
+            create_table :#{table_name} do |t|
+              t.uuid :uuid, null: false
+              t.string :event_key, null: false
+              t.integer :bucket, null: false
+              t.integer :status, null: false, default: 0
+              t.jsonb :options
+              t.binary :payload, null: false # when using mysql the column type should be mediumblob
+              t.integer :errors_count, null: false, default: 0
+              t.text :error_log
+              t.timestamp :processed_at
+              t.timestamps null: false
+            end
+
+            add_index :#{table_name}, :uuid, unique: true
+            add_index :#{table_name}, [:status, :bucket]
+            add_index :#{table_name}, :event_key
+            add_index :#{table_name}, :created_at
+          RUBY
+
+          gsub_file(migration_filepath, data_to_replace, optimize_indentation(template_data, 4))
+        end
+
+        def create_inbox_model_file(path)
+          template "inbox_item.rb", File.join("app/models", "#{path}.rb")
+        end
+
+        def create_outbox_model_file(path)
+          template "outbox_item.rb", File.join("app/models", "#{path}.rb")
+        end
+
+        def migration_class_name
+          "Create" + namespaced_item_class_name.gsub("::", "").pluralize
+        end
+
+        def migration_table_name
+          namespaced_item_class_name.tableize.tr("/", "_")
+        end
+      end
+    end
+  end
+end

data/lib/generators/helpers/paas.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Outbox
+  module Generators
+    module Helpers
+      module Paas
+        APP_MANIFEST_PATH = "configs/app.toml"
+
+        private
+
+        def paas_app?
+          File.exist?(APP_MANIFEST_PATH)
+        end
+      end
+    end
+  end
+end

data/lib/generators/helpers/values.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Outbox
+  module Generators
+    module Helpers
+      module Values
+        VALUES_PATH = "configs/values.yaml"
+
+        private
+
+        def add_item_to_values(deployment_name, item_path)
+          template_data = <<~RUBY
+            #{deployment_name}:
+              replicas:
+                _default: 1
+                prod: 2
+              command:
+                - /bin/sh
+                - -c
+                - exec bundle exec outbox start --box #{item_path} --concurrency 4
+              readinessProbe:
+                httpGet:
+                  path: /readiness/outbox
+                  port: SET-UP-YOUR-HEALTHCHECK-PORT-HERE
+              livenessProbe:
+                httpGet:
+                  path: /liveness/outbox
+                  port: SET-UP-YOUR-HEALTHCHECK-PORT-HERE
+              resources:
+                prod:
+                  requests:
+                    cpu: "500m"
+                    memory: "512Mi"
+                  limits:
+                    cpu: "1"
+                    memory: "1Gi"
+
+          RUBY
+
+          inject_into_file VALUES_PATH, optimize_indentation(template_data, 2), after: /^deployments:\s*\n/
+        end
+
+        def dasherize_item(item_path)
+          item_path.tr("/", "-").dasherize
+        end
+      end
+    end
+  end
+end

data/lib/generators/helpers.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "helpers/config"
+require_relative "helpers/initializer"
+require_relative "helpers/items"
+require_relative "helpers/migration"
+require_relative "helpers/paas"
+require_relative "helpers/values"

data/lib/generators/outbox/install/USAGE

@@ -0,0 +1,10 @@
+Description:
+    Generates initial environment for Outbox
+
+Example:
+    bin/rails generate install
+
+    This will create:
+      Outboxfile
+      config/initializers/outbox.rb
+      config/outbox.yml

data/lib/generators/outbox/install/install_generator.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "generators/outbox"
+
+module Outbox
+  module Generators
+    class InstallGenerator < Base
+      source_root File.expand_path("templates", __dir__)
+
+      class_option :skip_outboxfile, type: :boolean, default: false, desc: "Skip creating Outboxfile"
+      class_option :skip_initializer, type: :boolean, default: false, desc: "Skip creating config/initializers/outbox.rb"
+      class_option :skip_config, type: :boolean, default: false, desc: "Skip creating config/outbox.yml"
+
+      def create_outboxfile
+        return if options[:skip_outboxfile]
+
+        copy_file "Outboxfile", "Outboxfile"
+      end
+
+      def create_initializer
+        return if options[:skip_initializer]
+
+        copy_file "outbox.rb", OUTBOX_INITIALIZER_PATH
+      end
+
+      def create_config
+        return if options[:skip_config]
+
+        copy_file "outbox.yml", CONFIG_PATH
+      end
+    end
+  end
+end

data/lib/generators/outbox/install/templates/Outboxfile

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "config/environment"

data/lib/generators/outbox/install/templates/outbox.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+Rails.application.config.outbox.tap do |config|
+  # setup custom ErrorTracker
+  # config.error_tracker = "ErrorTracker"
+
+  # customize redis
+  # config.redis = {url: ENV.fetch("REDIS_URL", "redis://127.0.0.1:6379")}
+
+  # setup custom batch process middlewares
+  # config.batch_process_middlewares << "MyBatchProcessMiddleware"
+
+  # setup custom item process middlewares
+  # config.item_process_middlewares << "MyItemProcessMiddleware"
+
+  # config.process_items.tap do |x|
+  #   # maximum processing time of the batch, after which the batch will be considered hung and processing will be aborted
+  #   x[:general_timeout] = 180
+  #   # maximum patch processing time, after which the processing of the patch will be aborted in the current thread,
+  #   # and the next thread that picks up the batch will start processing from the same place
+  #   x[:cutoff_timeout] = 60
+  #   # batch size
+  #   x[:batch_size] = 200
+  # end
+
+  # config.worker.tap do |worker|
+  #   # number of batches that one thread will process per rate interval
+  #   worker[:rate_limit] = 10
+  #   # rate interval in seconds
+  #   worker[:rate_interval] = 60
+  # end
+end