sbmt-kafka_producer 2.0.0

data/lib/generators/kafka_producer/producer/producer_generator.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module KafkaProducer
+  module Generators
+    class ProducerGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      argument :producer_type, type: :string, banner: "sync/async"
+      argument :topic, type: :string, banner: "topic"
+
+      def insert_producer_class
+        template "producer.rb.erb", "app/producers/#{file_path}_producer.rb"
+      end
+    end
+  end
+end

data/lib/generators/kafka_producer/producer/templates/producer.rb.erb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+<%- module_namespacing do -%>
+class <%= "#{name.classify}Producer" %> < Sbmt::KafkaProducer::BaseProducer
+  option :topic, default: -> { "<%= topic %>" }
+
+  def publish(payload, **options)
+    <%= ("#{producer_type}_publish") %>(payload, options)
+  end
+end
+<%- end -%>

data/lib/sbmt/kafka_producer/base_producer.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    class BaseProducer
+      extend Dry::Initializer
+
+      option :client, default: -> { KafkaClientFactory.default_client }
+      option :topic
+
+      def sync_publish!(payload, options = {})
+        around_publish do
+          client.produce_sync(payload: payload, **options.merge(topic: topic))
+        end
+        true
+      end
+
+      def sync_publish(payload, options = {})
+        sync_publish!(payload, options)
+        true
+      rescue WaterDrop::Errors::ProduceError => e
+        log_error(e)
+        false
+      end
+
+      def async_publish!(payload, options = {})
+        around_publish do
+          client.produce_async(payload: payload, **options.merge(topic: topic))
+        end
+        true
+      end
+
+      def async_publish(payload, options = {})
+        async_publish!(payload, options)
+        true
+      rescue WaterDrop::Errors::ProduceError => e
+        log_error(e)
+        false
+      end
+
+      private
+
+      def logger
+        ::Sbmt::KafkaProducer.logger
+      end
+
+      def around_publish
+        with_sentry_transaction { yield }
+      end
+
+      def with_sentry_transaction
+        return yield unless defined?(::Sentry)
+        return yield unless ::Sentry.initialized?
+
+        transaction = ::Sentry.start_transaction(
+          name: "Karafka/#{self.class.name}",
+          op: "kafka-producer"
+        )
+
+        # Tracing is disabled by config
+        return yield unless transaction
+
+        result = nil
+        transaction.with_child_span do |span|
+          span.set_data(:topic, topic)
+          result = yield
+        end
+
+        transaction.finish
+        result
+      end
+
+      def ignore_kafka_errors?
+        config.ignore_kafka_error.to_s == "true"
+      end
+
+      def log_error(error)
+        return true if ignore_kafka_errors?
+
+        logger.error "KAFKA ERROR: #{format_exception_error(error)}\n#{error.backtrace.join("\n")}"
+        ErrorTracker.error(error)
+      end
+
+      def format_exception_error(error)
+        text = "#{format_exception_error(error.cause)}. " if with_cause?(error)
+
+        if error.respond_to?(:message)
+          "#{text}#{error.class.name} #{error.message}"
+        else
+          "#{text}#{error}"
+        end
+      end
+
+      def with_cause?(error)
+        error.respond_to?(:cause) && error.cause.present?
+      end
+
+      def config
+        Config::Producer
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/config/auth.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    module Config
+      class Auth < Dry::Struct
+        transform_keys(&:to_sym)
+
+        AVAILABLE_AUTH_KINDS = %w[plaintext sasl_plaintext].freeze
+        DEFAULT_AUTH_KIND = "plaintext"
+
+        AVAILABLE_SASL_MECHANISMS = %w[PLAIN SCRAM-SHA-256 SCRAM-SHA-512].freeze
+        DEFAULT_SASL_MECHANISM = "SCRAM-SHA-512"
+
+        attribute :kind, Sbmt::KafkaProducer::Types::Strict::String
+          .default(DEFAULT_AUTH_KIND)
+          .enum(*AVAILABLE_AUTH_KINDS)
+        attribute? :sasl_mechanism, Sbmt::KafkaProducer::Types::Strict::String
+          .default(DEFAULT_SASL_MECHANISM)
+          .enum(*AVAILABLE_SASL_MECHANISMS)
+        attribute? :sasl_username, Sbmt::KafkaProducer::Types::Strict::String
+        attribute? :sasl_password, Sbmt::KafkaProducer::Types::Strict::String
+
+        def to_kafka_options
+          ensure_options_are_valid
+
+          opts = {}
+
+          case kind
+          when "sasl_plaintext"
+            opts.merge!(
+              "security.protocol": kind,
+              "sasl.mechanism": sasl_mechanism,
+              "sasl.username": sasl_username,
+              "sasl.password": sasl_password
+            )
+          when "plaintext"
+            opts[:"security.protocol"] = kind
+          else
+            raise Anyway::Config::ValidationError, "unknown auth kind: #{kind}"
+          end
+
+          opts.symbolize_keys
+        end
+
+        private
+
+        def ensure_options_are_valid
+          raise Anyway::Config::ValidationError, "unknown auth kind: #{kind}" unless AVAILABLE_AUTH_KINDS.include?(kind)
+
+          case kind
+          when "sasl_plaintext"
+            raise Anyway::Config::ValidationError, "sasl_username is required for #{kind} auth kind" if sasl_username.blank?
+            raise Anyway::Config::ValidationError, "sasl_password is required for #{kind} auth kind" if sasl_password.blank?
+            raise Anyway::Config::ValidationError, "sasl_mechanism is required for #{kind} auth kind" if sasl_mechanism.blank?
+            raise Anyway::Config::ValidationError, "invalid sasl_mechanism for #{kind} auth kind, available options are: [#{AVAILABLE_SASL_MECHANISMS.join(",")}]" unless AVAILABLE_SASL_MECHANISMS.include?(sasl_mechanism)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/config/kafka.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    module Config
+      class Kafka < Dry::Struct
+        transform_keys(&:to_sym)
+
+        # srv1:port1,srv2:port2,...
+        SERVERS_REGEXP = /^[a-z\d.\-:]+(,[a-z\d.\-:]+)*$/.freeze
+
+        attribute :servers, Sbmt::KafkaProducer::Types::String.constrained(format: SERVERS_REGEXP)
+
+        # defaults are rdkafka's
+        # see https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
+        attribute :connect_timeout, Sbmt::KafkaProducer::Types::Coercible::Integer.optional.default(1)
+        attribute :ack_timeout, Sbmt::KafkaProducer::Types::Coercible::Integer.optional.default(1)
+        attribute :required_acks, Sbmt::KafkaProducer::Types::Coercible::Integer.optional.default(-1)
+        attribute :max_retries, Sbmt::KafkaProducer::Types::Coercible::Integer.optional.default(2)
+        attribute :retry_backoff, Sbmt::KafkaProducer::Types::Coercible::Integer.optional.default(1)
+
+        attribute :kafka_config, Sbmt::KafkaProducer::Types::ConfigAttrs.optional.default({}.freeze)
+
+        def to_kafka_options
+          kafka_config.merge(
+            "bootstrap.servers": servers,
+            "socket.connection.setup.timeout.ms": connect_timeout.to_f * 1000,
+            "request.timeout.ms": ack_timeout.to_f * 1000,
+            "request.required.acks": required_acks,
+            "message.send.max.retries": max_retries,
+            "retry.backoff.ms": retry_backoff.to_f * 1000
+          ).symbolize_keys
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/config/producer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    module Config
+      class Producer < Anyway::Config
+        class << self
+          # Make it possible to access a singleton config instance
+          # via class methods (i.e., without explicitly calling `instance`)
+          delegate_missing_to :instance
+
+          def coerce_to(struct)
+            lambda do |raw_attrs|
+              struct.new(**raw_attrs)
+            rescue Dry::Types::SchemaError => e
+              raise_validation_error "cannot parse #{struct}: #{e.message}"
+            end
+          end
+
+          private
+
+          # Returns a singleton config instance
+          def instance
+            @instance ||= new
+          end
+        end
+
+        config_name :kafka_producer
+
+        attr_config :ignore_kafka_error, :deliver, :wait_on_queue_full,
+          :max_payload_size, :max_wait_timeout, :wait_timeout,
+          :wait_on_queue_full_timeout,
+          auth: {}, kafka: {},
+          logger_class: "::Sbmt::KafkaProducer::Logger",
+          metrics_listener_class: "::Sbmt::KafkaProducer::Instrumentation::YabedaMetricsListener"
+
+        coerce_types ignore_kafka_error: :boolean,
+          deliver: :boolean, wait_on_queue_full: :boolean,
+          max_payload_size: :integer, max_wait_timeout: :integer,
+          wait_timeout: :float, wait_on_queue_full_timeout: :float
+        coerce_types kafka: coerce_to(Kafka)
+        coerce_types auth: coerce_to(Auth)
+
+        def to_kafka_options
+          kafka.to_kafka_options
+            .merge(auth.to_kafka_options)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/error_tracker.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    class ErrorTracker
+      class << self
+        def error(arr)
+          logging(:error, arr)
+        end
+
+        private
+
+        def logging(level, arr)
+          return unless defined?(::Sentry)
+
+          sentry_logging(level, arr) if ::Sentry.initialized?
+        end
+
+        def sentry_logging(level, arr)
+          Sentry.with_scope do |_scope|
+            if arr.is_a?(Exception)
+              Sentry.capture_exception(arr, level: level)
+            else
+              Sentry.capture_message(arr, level: level)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/instrumentation/open_telemetry_loader.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "opentelemetry"
+require "opentelemetry-common"
+require "opentelemetry-instrumentation-base"
+
+require_relative "open_telemetry_tracer"
+
+module Sbmt
+  module KafkaProducer
+    module Instrumentation
+      class OpenTelemetryLoader < ::OpenTelemetry::Instrumentation::Base
+        install do |_config|
+          OpenTelemetryTracer.enabled = true
+        end
+
+        present do
+          defined?(OpenTelemetryTracer)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/instrumentation/open_telemetry_tracer.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    module Instrumentation
+      class OpenTelemetryTracer
+        class << self
+          def enabled?
+            !!@enabled
+          end
+
+          attr_writer :enabled
+        end
+
+        def enabled?
+          self.class.enabled?
+        end
+
+        def call(message)
+          return message unless enabled?
+
+          topic = message[:topic]
+          attributes = {
+            "messaging.system" => "kafka",
+            "messaging.destination" => topic,
+            "messaging.destination_kind" => "topic"
+          }
+
+          message_key = extract_message_key(message[:key])
+          attributes["messaging.kafka.message_key"] = message_key if message_key
+
+          message[:headers] ||= {}
+
+          tracer.in_span("#{topic} publish", attributes: attributes, kind: :producer) do
+            ::OpenTelemetry.propagation.inject(message[:headers])
+          end
+
+          message
+        end
+
+        private
+
+        def tracer
+          ::Sbmt::KafkaProducer::Instrumentation::OpenTelemetryLoader.instance.tracer
+        end
+
+        def extract_message_key(key)
+          # skip encode if already valid utf8
+          return key if key.nil? || (key.encoding == Encoding::UTF_8 && key.valid_encoding?)
+
+          key.encode(Encoding::UTF_8)
+        rescue Encoding::UndefinedConversionError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/instrumentation/tracing_middleware.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    module Instrumentation
+      class TracingMiddleware < ::WaterDrop::Middleware
+        def initialize
+          super
+
+          append(OpenTelemetryTracer.new) if defined?(OpenTelemetryTracer)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/instrumentation/yabeda_metrics_listener.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    module Instrumentation
+      class YabedaMetricsListener
+        DEFAULT_CLIENT = {client: "waterdrop"}.freeze
+        def on_statistics_emitted(event)
+          # https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
+          stats = event.payload[:statistics]
+          broker_stats = stats["brokers"]
+
+          report_broker_stats(broker_stats)
+        end
+
+        def on_error_occurred(event)
+          tags = {topic: event[:topic]}.merge!(DEFAULT_CLIENT) if event.payload.include?(:topic)
+
+          case event[:type]
+          when "message.produce_sync", "message.produce_async"
+            Yabeda.kafka_producer.produce_errors
+              .increment(produce_base_tags(event))
+          when "librdkafka.dispatch_error"
+            Yabeda.kafka_producer.deliver_errors
+              .increment(tags)
+          end
+        end
+
+        %i[produced_sync produced_async].each do |event_scope|
+          define_method(:"on_message_#{event_scope}") do |event|
+            Yabeda.kafka_producer.produced_messages
+              .increment(produce_base_tags(event))
+
+            Yabeda.kafka_producer.message_size
+              .measure(produce_base_tags(event), event[:message].to_s.bytesize)
+
+            Yabeda.kafka_producer.deliver_latency
+              .measure(produce_base_tags(event), event[:time])
+          end
+        end
+
+        def on_message_buffered(event)
+          Yabeda.kafka_producer.buffer_size
+            .measure(DEFAULT_CLIENT, event[:buffer].size)
+        end
+
+        def on_message_acknowledged(event)
+          tag = {topic: event[:topic]}.merge!(DEFAULT_CLIENT)
+
+          Yabeda.kafka_producer.deliver_messages
+            .increment(tag)
+        end
+
+        private
+
+        def produce_base_tags(event)
+          {
+            client: DEFAULT_CLIENT[:client],
+            topic: event[:message][:topic]
+          }
+        end
+
+        def report_broker_stats(brokers)
+          brokers.each_value do |broker_statistics|
+            # Skip bootstrap nodes
+            next if broker_statistics["nodeid"] == -1
+
+            broker_tags = {
+              client: DEFAULT_CLIENT[:client],
+              broker: broker_statistics["nodename"]
+            }
+
+            Yabeda.kafka_api.calls
+              .increment(broker_tags, by: broker_statistics["tx"])
+            Yabeda.kafka_api.latency
+              .measure(broker_tags, broker_statistics["rtt"]["avg"])
+            Yabeda.kafka_api.request_size
+              .measure(broker_tags, broker_statistics["txbytes"])
+            Yabeda.kafka_api.response_size
+              .measure(broker_tags, broker_statistics["rxbytes"])
+            Yabeda.kafka_api.errors
+              .increment(broker_tags, by: broker_statistics["txerrs"] + broker_statistics["rxerrs"])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/kafka_client_factory.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    class KafkaClientFactory
+      class << self
+        def default_client
+          @default_client ||= ConnectionPool::Wrapper.new do
+            WaterDrop::Producer.new do |config|
+              configure_client(config)
+            end
+          end
+        end
+
+        def build(kafka = {})
+          return default_client if kafka.empty?
+
+          ConnectionPool::Wrapper.new do
+            WaterDrop::Producer.new do |config|
+              configure_client(config, kafka)
+            end
+          end
+        end
+
+        private
+
+        def configure_client(kafka_config, kafka_options = {})
+          kafka_config.logger = config.logger_class.classify.constantize.new
+          kafka_config.kafka = config.to_kafka_options.merge(custom_kafka_config(kafka_options)).symbolize_keys
+
+          kafka_config.middleware = Instrumentation::TracingMiddleware.new
+
+          kafka_config.deliver = config.deliver if config.deliver.present?
+          kafka_config.wait_on_queue_full = config.wait_on_queue_full if config.wait_on_queue_full.present?
+          kafka_config.max_payload_size = config.max_payload_size if config.max_payload_size.present?
+          kafka_config.max_wait_timeout = config.max_wait_timeout if config.max_wait_timeout.present?
+          kafka_config.wait_timeout = config.wait_timeout if config.wait_timeout.present?
+          kafka_config.wait_on_queue_full_timeout = config.wait_on_queue_full_timeout if config.wait_on_queue_full_timeout.present?
+
+          kafka_config.monitor.subscribe(config.metrics_listener_class.classify.constantize.new)
+        end
+
+        def custom_kafka_config(kafka_options)
+          result = {}
+
+          result["socket.connection.setup.timeout.ms"] = kafka_options["connect_timeout"].to_f * 1000 if kafka_options.key?("connect_timeout")
+          result["request.timeout.ms"] = kafka_options["ack_timeout"].to_f * 1000 if kafka_options.key?("ack_timeout")
+          result["request.required.acks"] = kafka_options["required_acks"] if kafka_options.key?("required_acks")
+          result["message.send.max.retries"] = kafka_options["max_retries"] if kafka_options.key?("max_retries")
+          result["retry.backoff.ms"] = kafka_options["retry_backoff"].to_f * 1000 if kafka_options.key?("retry_backoff")
+
+          result
+        end
+
+        def config
+          Config::Producer
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/logger.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    class Logger
+      delegate :logger, to: :Rails
+
+      %i[
+        debug
+        info
+        warn
+        error
+        fatal
+      ].each do |log_level|
+        define_method log_level do |*args|
+          logger.send(log_level, *args)
+        end
+      end
+
+      def add(...)
+        logger.add(...)
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/outbox_producer.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    class OutboxProducer < Sbmt::KafkaProducer::BaseProducer
+      def call(outbox_item, payload)
+        sync_publish!(payload, **outbox_item.options)
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/outbox_transport_factory.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    class OutboxTransportFactory
+      class << self
+        def build(topic:, kafka: {})
+          OutboxProducer.new(topic: topic, client: KafkaClientFactory.build(kafka))
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/railtie.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    class Railtie < Rails::Railtie
+      initializer "sbmt_kafka_producer_yabeda.configure_rails_initialization" do
+        YabedaConfigurer.configure
+      end
+
+      initializer "sbmt_kafka_producer_opentelemetry_init.configure_rails_initialization",
+        after: "opentelemetry.configure" do
+        require "sbmt/kafka_producer/instrumentation/open_telemetry_loader" if defined?(::OpenTelemetry)
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_producer/testing/configure_producer_client.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+RSpec.configure do |config|
+  config.before(:each) do
+    allow(Sbmt::KafkaProducer::KafkaClientFactory)
+      .to receive(:default_client)
+      .and_return(instance_double(WaterDrop::Producer, {produce_sync: true, produce_async: true}))
+
+    allow(Sbmt::KafkaProducer::KafkaClientFactory)
+      .to receive(:build)
+      .and_return(instance_double(WaterDrop::Producer, {produce_sync: true, produce_async: true}))
+  end
+end

data/lib/sbmt/kafka_producer/testing.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require "rspec"
+
+Dir["#{__dir__}/testing/*.rb"].sort.each { |f| require f }

data/lib/sbmt/kafka_producer/types.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    module Types
+      include Dry.Types
+
+      ConfigAttrs = Dry::Types["hash"].constructor { |hsh| hsh.deep_symbolize_keys }
+      ConfigProducer = Types.Constructor(Config::Producer)
+    end
+  end
+end

data/lib/sbmt/kafka_producer/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaProducer
+    VERSION = "2.0.0"
+  end
+end