sbmt-outbox 5.0.0

data/lib/generators/outbox/install/templates/outbox.yml

@@ -0,0 +1,51 @@
+default: &default
+  owner: owner-team
+  bucket_size: 16
+
+  probes:
+      port: SET-UP-YOUR-HEALTHCHECK-PORT-HERE
+
+  kafka:
+    required_acks: -1
+    max_retries: 1
+
+#  outbox_items:
+#    outbox_item:
+#      owner: outbox-item-owner-team
+#      partition_size: 2
+#      partition_strategy: number
+#      retention: P1W
+#      retry_strategies:
+#        - exponential_backoff
+#        - compacted_log
+#      transports:
+#        kafka_producer:
+#          topic: "outbox_item_topic"
+#          kafka:
+#            required_acks: -1
+
+#  inbox_items:
+#    inbox_item:
+#      owner: inbox-item-owner-team
+#      partition_size: 2
+#      partition_strategy: number
+#      retention: P1W
+#      retry_strategies:
+#        - exponential_backoff
+#      transports:
+#        import_order:
+#          source: "kafka_consumer"
+
+development:
+  <<: *default
+
+test:
+  <<: *default
+  bucket_size: 2
+
+staging:
+  <<: *default
+
+production:
+  <<: *default
+  bucket_size: 256

data/lib/generators/outbox/item/USAGE

@@ -0,0 +1,12 @@
+Description:
+    Generates Outbox item (model, migration, patches outbox.yml, etc)
+
+Example:
+    bin/rails generate outbox:item Model::InboxItem --kind inbox
+
+    This will create:
+        app/models/model/inbox_item.rb
+        db/migrations/XXX_create_model_inbox_items.rb
+    This will modify:
+        configs/values.yml
+        config/outbox.yml

data/lib/generators/outbox/item/item_generator.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "generators/outbox"
+
+module Outbox
+  module Generators
+    class ItemGenerator < NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      class_option :kind, type: :string, desc: "Either inbox or outbox", banner: "inbox/outbox", required: true
+      class_option :skip_migration, type: :boolean, default: false, desc: "Skip creating migration"
+      class_option :skip_model, type: :boolean, default: false, desc: "Skip creating model class"
+      class_option :skip_config, type: :boolean, default: false, desc: "Skip modifying config/outbox.yml"
+      class_option :skip_values, type: :boolean, default: false, desc: "Skip modifying configs/values.yaml"
+
+      def check_kind!
+        return if options[:kind].in?(%w[inbox outbox])
+
+        raise Rails::Generators::Error, "Unknown item kind. " \
+                                        "Please specify `--kind inbox` or `--kind outbox`"
+      end
+
+      def check!
+        check_config!
+      end
+
+      def validate_item_name
+        return if file_name.underscore.match?(%r{#{options[:kind]}_item$})
+
+        continue = yes?(
+          "Warning: your item's name doesn't match the conventional" \
+          " name format (i.e. Some#{options[:kind].camelize}Item). Continue?"
+        )
+        return if continue
+
+        raise Rails::Generators::Error, "Aborting"
+      end
+
+      def create_migration
+        return if options[:skip_migration]
+
+        create_migration_file(migration_class_name, migration_table_name)
+      end
+
+      def create_model
+        return if options[:skip_model]
+
+        template "#{options[:kind]}_item.rb", File.join("app/models", "#{item_path}.rb")
+      end
+
+      def patch_config
+        return if options[:skip_config]
+
+        item_template_data = if options[:kind] == "inbox"
+          <<~RUBY
+            #{item_path}:
+              partition_size: 1
+              partition_strategy: hash
+              retention: P1W
+              max_retries: 7
+              retry_strategies:
+                - exponential_backoff
+            #  # see README to learn more about transport configuration
+            #  transports: {}
+          RUBY
+        else
+          <<~RUBY
+            #{item_path}:
+              partition_size: 1
+              partition_strategy: number
+              retention: P3D
+              max_retries: 7
+              retry_strategies:
+                - exponential_backoff
+            #  # see README to learn more about transport configuration
+            #  transports: {}
+          RUBY
+        end
+
+        add_item_to_config("#{options[:kind]}_items", item_template_data)
+      end
+
+      def patch_values
+        return if options[:skip_values]
+        return unless paas_app?
+
+        # e.g. order/inbox_item => inbox-order-inbox-items
+        deployment_name = "#{options[:kind]}-" + dasherize_item(item_path)
+
+        add_item_to_values(deployment_name.pluralize, item_path)
+      end
+    end
+  end
+end

data/lib/generators/outbox/item/templates/inbox_item.rb.tt

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+<% module_namespacing do -%>
+class <%= namespaced_item_class_name %> < Sbmt::Outbox::InboxItem
+  self.table_name = "<%= migration_table_name %>"
+end
+<% end -%>

data/lib/generators/outbox/item/templates/outbox_item.rb.tt

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+<% module_namespacing do -%>
+class <%= namespaced_item_class_name %> < Sbmt::Outbox::OutboxItem
+  self.table_name = "<%= migration_table_name %>"
+
+  private
+
+  def extra_options
+    {
+      # Kafka partition key
+      key: event_key
+    }
+  end
+end
+<% end -%>

data/lib/generators/outbox/transport/USAGE

@@ -0,0 +1,19 @@
+Description:
+    Configures a transport inside the config/outbox.yml file
+    Arguments:
+        - item name, either CamelCased or under_scored, required
+        - transport name, i.e. sbmt/kafka_producer or import_order, required
+    Options:
+        - --kind inbox/outbox - transport type, required
+        - --topic, i.e. my-topic-name, optional
+        - --event_name, i.e. order_created, affects only outbox transports, optional
+        - --source, i.e. kafka, affects only inbox transports, optional
+        - --target, i.e order, affects only inbox transports, optional
+
+Example:
+    bin/rails g outbox:transport MyOutboxItem sbmt/kafka_producer --kind outbox
+
+    The following lines will be inserted to config/outbox.yml under the `outbox_items/my_outbox_item` section:
+      transports:
+        sbmt/kafka_producer:
+          topic: "need to add a topic"

data/lib/generators/outbox/transport/templates/inbox_transport.yml.erb

@@ -0,0 +1,9 @@
+transports:
+  <%= transport.underscore %>:
+  <%- if options[:source].present? -%>
+    source: "<%= options[:source] %>"
+  <%- elsif options[:target].present? -%>
+    target: "<%= options[:target] %>"
+  <%- else -%>
+    # configure source, target and other options here
+  <%- end -%>

data/lib/generators/outbox/transport/templates/outbox_transport.yml.erb

@@ -0,0 +1,10 @@
+<%- topic = options[:topic].presence || "need to add a topic" -%>
+transports:
+  <%- if options[:event_name].present? -%>
+  - class: "<%= transport.underscore %>"
+  event_name: "<%= options[:event_name] %>"
+  topic: "<%= topic %>"
+  <%- else -%>
+  <%= transport.underscore %>:
+    topic: "<%= topic %>"
+  <%- end -%>

data/lib/generators/outbox/transport/transport_generator.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "generators/outbox"
+
+module Outbox
+  module Generators
+    class TransportGenerator < NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      argument :transport, required: true, type: :string, banner: "sbmt/kafka_producer"
+
+      class_option :kind, type: :string, desc: "Either inbox or outbox", banner: "inbox/outbox", required: true
+      class_option :topic, type: :string, banner: "my-topic-name"
+      class_option :source, type: :string, banner: "kafka"
+      class_option :target, type: :string, banner: "order"
+      class_option :event_name, type: :string, default: "", banner: "order_created"
+
+      def check_name!
+        return if options[:kind].in?(%w[inbox outbox])
+
+        raise Rails::Generators::Error, "Unknown transport type. Should be inbox or outbox"
+      end
+
+      def check!
+        check_config!
+      end
+
+      def check_item_exists!
+        return if item_exists?
+
+        raise Rails::Generators::Error, "Item `#{item_name}` does not exist in the `#{options[:kind]}_items` " \
+                                        "section of #{CONFIG_PATH}. You may insert this item by running " \
+                                        "`bin/rails g outbox:item #{item_name} --kind #{options[:kind]}`"
+      end
+
+      def insert_transport
+        data = optimize_indentation(template, 6)
+        insert_into_file CONFIG_PATH, data, after: "#{item_name.underscore}:\n", force: true
+      end
+
+      private
+
+      def item_name
+        name
+      end
+
+      def item_exists?
+        File.binread(CONFIG_PATH).match?(%r{#{options[:kind]}_items:.*#{item_name.underscore}:}m)
+      end
+
+      def template_path
+        File.join(TransportGenerator.source_root, "#{options[:kind]}_transport.yml.erb")
+      end
+
+      def template
+        ERB.new(File.read(template_path), trim_mode: "%-").result(binding)
+      end
+    end
+  end
+end

data/lib/generators/outbox.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require_relative "helpers"
+
+module Outbox
+  module Generators
+    class Base < Rails::Generators::Base
+      include Helpers::Config
+      include Helpers::Initializer
+      include Helpers::Paas
+    end
+
+    class NamedBase < Rails::Generators::NamedBase
+      include Helpers::Config
+      include Helpers::Initializer
+      include Helpers::Items
+      include Helpers::Migration
+      include Helpers::Paas
+      include Helpers::Values
+    end
+  end
+end

data/lib/sbmt/outbox/ascii_art.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module AsciiArt
+      module_function
+
+      def logo
+        <<~'TEXT'
+
+                      ______________________
+                    /\    _______  _______ \
+                    /  \  /\      \/\      \ \
+                  / _  \   \      \ \      \ \
+                  / /\   \   \      \ \      \ \
+                / /  \   \   \______\ \______\ \
+                / /    \   \  / _____/_/ _____/_ \
+              / /      \_  \  /\      \/\      \ \
+              / _\      / _  \   \      \ \      \ \
+            / /\ \    / /\   \   \      \ \      \ \
+            / /  \ \  / /  \   \   \______\ \______\ \
+          / /    \ \/_/    \   \  /      / /      /  \
+          / /      \_ /      \_  \_____________________\
+          \ \      / _\      /   /                     /
+          \ \    / /\ \    /   /  \______\ \______\  /
+            \ \  / /  \ \  /   /   /      / /      / /
+            \ \/_/    \ \/_  /   /      / /      / /
+              \  /      \_   /   /      / /      / /
+              \ \      /   /  \/______/\/______/ /
+                \ \    /   /  \______\ \______\  /
+                \ \  /   /   /      / /      / /
+                  \ \/_  /   /      / /      / /
+                  \    /   /      / /      / /
+                    \  /  \/______/\/______/ /
+                    \/_____________________/
+
+        TEXT
+      end
+
+      def shutdown
+        <<~'TEXT'
+
+                   _
+                  | |
+                  | |===( )   //////
+                  |_|   |||  | o o|
+                         ||| ( c  )                  ____
+                          ||| \= /                  ||   \_
+                           ||||||                   ||     |
+                           ||||||                ...||__/|-"
+                           ||||||             __|________|__
+                             |||             |______________|
+                             |||             || ||      || ||
+                             |||             || ||      || ||
+          -------------------|||-------------||-||------||-||-------
+                             |__>            || ||      || ||
+
+        TEXT
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/cli.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "thor"
+require "sbmt/outbox/ascii_art"
+
+module Sbmt
+  module Outbox
+    class CLI < Thor
+      def self.exit_on_failure?
+        true
+      end
+
+      default_command :start
+
+      desc "start", "Start outbox worker"
+      option :box,
+        aliases: "-b",
+        repeatable: true,
+        desc: "Outbox/Inbox processors to start in format foo_name:1,2,n bar_name:1,2,n"
+      option :concurrency,
+        aliases: "-c",
+        type: :numeric,
+        default: 10,
+        desc: "Number of threads"
+      def start
+        load_environment
+
+        worker = Sbmt::Outbox::Worker.new(
+          boxes: format_boxes(options[:box]),
+          concurrency: options[:concurrency]
+        )
+
+        Sbmt::Outbox.current_worker = worker
+
+        watch_signals(worker)
+
+        $stdout.puts AsciiArt.logo
+        $stdout.puts "Outbox/Inbox worker has been started"
+        $stdout.puts "Version: #{VERSION}"
+        $stdout.puts "Starting probes..."
+        Sbmt::Outbox::Probes::Probe.run_probes
+
+        worker.start
+      end
+
+      private
+
+      def load_environment
+        load(lookup_outboxfile)
+
+        require "sbmt/outbox"
+        require "sbmt/outbox/worker"
+      end
+
+      def lookup_outboxfile
+        file_path = ENV["OUTBOXFILE"] || "#{Dir.pwd}/Outboxfile"
+
+        raise "Cannot locate Outboxfile at #{file_path}" unless File.exist?(file_path)
+
+        file_path
+      end
+
+      def format_boxes(val)
+        if val.nil?
+          fetch_all_boxes
+        else
+          extract_boxes(val)
+        end
+      end
+
+      def fetch_all_boxes
+        Outbox.outbox_item_classes + Outbox.inbox_item_classes
+      end
+
+      def extract_boxes(boxes)
+        boxes.map do |name, acc|
+          item_class = Sbmt::Outbox.item_classes_by_name[name]
+          raise "Cannot locate box #{name}" unless item_class
+          item_class
+        end
+      end
+
+      def watch_signals(worker)
+        # ctrl+c
+        Signal.trap("INT") do
+          $stdout.puts AsciiArt.shutdown
+          $stdout.puts "Going to shut down..."
+          worker.stop
+        end
+
+        # normal kill with number 15
+        Signal.trap("TERM") do
+          $stdout.puts AsciiArt.shutdown
+          $stdout.puts "Going to shut down..."
+          worker.stop
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/database_switcher.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class DatabaseSwitcher
+      def self.use_slave
+        yield
+      end
+
+      def self.use_master
+        yield
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/engine.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "rails/engine"
+
+module Sbmt
+  module Outbox
+    class Engine < Rails::Engine
+      isolate_namespace Sbmt::Outbox
+
+      config.outbox = ActiveSupport::OrderedOptions.new.tap do |c|
+        c.active_record_base_class = "ApplicationRecord"
+        c.active_job_base_class = "ApplicationJob"
+        c.error_tracker = "Sbmt::Outbox::ErrorTracker"
+        c.outbox_item_classes = []
+        c.inbox_item_classes = []
+        c.paths = []
+        c.redis = {url: ENV.fetch("REDIS_URL", "redis://127.0.0.1:6379")}
+        c.process_items = ActiveSupport::OrderedOptions.new.tap do |c|
+          c.general_timeout = 120
+          c.cutoff_timeout = 60
+          c.batch_size = 200
+        end
+        c.worker = ActiveSupport::OrderedOptions.new.tap do |c|
+          c.rate_limit = 20
+          c.rate_interval = 60
+          c.shuffle_jobs = true
+        end
+        c.database_switcher = "Sbmt::Outbox::DatabaseSwitcher"
+        c.batch_process_middlewares = []
+        c.item_process_middlewares = []
+        c.create_item_middlewares = []
+
+        if defined?(::Sentry)
+          c.batch_process_middlewares.push("Sbmt::Outbox::Middleware::Sentry::TracingBatchProcessMiddleware")
+          c.item_process_middlewares.push("Sbmt::Outbox::Middleware::Sentry::TracingItemProcessMiddleware")
+        end
+      end
+
+      rake_tasks do
+        load "sbmt/outbox/tasks/retry_failed_items.rake"
+        load "sbmt/outbox/tasks/delete_failed_items.rake"
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/error_tracker.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class ErrorTracker
+      class << self
+        def error(message, params = {})
+          unless defined?(Sentry)
+            Outbox.logger.log_error(message, params)
+            return
+          end
+
+          Sentry.with_scope do |scope|
+            scope.set_contexts(contexts: params)
+
+            if message.is_a?(Exception)
+              Sentry.capture_exception(message, level: :error)
+            else
+              Sentry.capture_message(message, level: :error)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/errors.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class Error < StandardError
+    end
+
+    class ConfigError < Error
+    end
+
+    class DatabaseError < Error
+    end
+  end
+end

data/lib/sbmt/outbox/instrumentation/open_telemetry_loader.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "opentelemetry"
+require "opentelemetry-common"
+require "opentelemetry-instrumentation-base"
+
+require_relative "../middleware/open_telemetry/tracing_create_item_middleware"
+require_relative "../middleware/open_telemetry/tracing_item_process_middleware"
+
+module Sbmt
+  module Outbox
+    module Instrumentation
+      class OpenTelemetryLoader < ::OpenTelemetry::Instrumentation::Base
+        install do |_config|
+          require_dependencies
+
+          ::Sbmt::Outbox.config.create_item_middlewares.push("Sbmt::Outbox::Middleware::OpenTelemetry::TracingCreateItemMiddleware")
+          ::Sbmt::Outbox.config.item_process_middlewares.push("Sbmt::Outbox::Middleware::OpenTelemetry::TracingItemProcessMiddleware")
+        end
+
+        present do
+          true
+        end
+
+        private
+
+        def require_dependencies
+          require_relative "../middleware/open_telemetry/tracing_create_item_middleware"
+          require_relative "../middleware/open_telemetry/tracing_item_process_middleware"
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/logger.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    class Logger
+      delegate :logger, to: :Rails
+
+      def log_info(message, **params)
+        with_tags(**params) do
+          logger.info(message)
+        end
+      end
+
+      def log_error(message, **params)
+        with_tags(**params) do
+          logger.error(message)
+        end
+      end
+
+      def log_success(message, **params)
+        log_info(message, status: "success", **params)
+      end
+
+      def log_failure(message, **params)
+        log_error(message, status: "failure", **params)
+      end
+
+      def with_tags(**params)
+        logger.tagged(**params) do
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/outbox/middleware/builder.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module Outbox
+    module Middleware
+      class Builder
+        def initialize(middlewares)
+          middlewares.each { |middleware| stack << middleware }
+        end
+
+        def call(...)
+          Runner.new(stack.dup).call(...)
+        end
+
+        private
+
+        def stack
+          @stack ||= []
+        end
+      end
+    end
+  end
+end