sbmt-kafka_consumer 2.0.0

data/lib/sbmt/kafka_consumer/instrumentation/logger_listener.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Instrumentation
+      class LoggerListener < Karafka::Instrumentation::LoggerListener
+        include ListenerHelper
+        CUSTOM_ERROR_TYPES = %w[consumer.base.consume_one consumer.inbox.consume_one].freeze
+
+        def on_error_occurred(event)
+          type = event[:type]
+          error = event[:error]
+
+          # catch here only consumer-specific errors
+          # and let default handler to process other
+          return super unless CUSTOM_ERROR_TYPES.include?(type)
+
+          tags = {}
+          tags[:status] = event[:status] if type == "consumer.inbox.consume_one"
+
+          logger.tagged(
+            type: type,
+            **tags
+          ) do
+            logger.error(error_message(error))
+            log_backtrace(error)
+          end
+        end
+
+        # BaseConsumer events
+        def on_consumer_consumed_one(event)
+          logger.info("Successfully consumed message in #{event.payload[:time]} ms")
+        end
+
+        # InboxConsumer events
+        def on_consumer_inbox_consumed_one(event)
+          logger.tagged(status: event[:status]) do
+            logger.info("Successfully consumed message with uuid: #{event[:message_uuid]} in #{event.payload[:time]} ms")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/instrumentation/open_telemetry_loader.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "opentelemetry"
+require "opentelemetry-common"
+require "opentelemetry-instrumentation-base"
+
+require_relative "open_telemetry_tracer"
+
+module Sbmt
+  module KafkaConsumer
+    module Instrumentation
+      class OpenTelemetryLoader < OpenTelemetry::Instrumentation::Base
+        install do |_config|
+          OpenTelemetryTracer.enabled = true
+        end
+
+        present do
+          defined?(OpenTelemetryTracer)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/instrumentation/open_telemetry_tracer.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require_relative "tracer"
+
+module Sbmt
+  module KafkaConsumer
+    module Instrumentation
+      class OpenTelemetryTracer < ::Sbmt::KafkaConsumer::Instrumentation::Tracer
+        class << self
+          def enabled?
+            !!@enabled
+          end
+
+          attr_writer :enabled
+        end
+
+        def enabled?
+          self.class.enabled?
+        end
+
+        def trace(&block)
+          return handle_consumed_one(&block) if @event_id == "consumer.consumed_one"
+          return handle_inbox_consumed_one(&block) if @event_id == "consumer.inbox.consumed_one"
+          return handle_error(&block) if @event_id == "error.occurred"
+
+          yield
+        end
+
+        def handle_consumed_one
+          return yield unless enabled?
+
+          consumer = @payload[:caller]
+          message = @payload[:message]
+
+          parent_context = ::OpenTelemetry.propagation.extract(message.headers, getter: ::OpenTelemetry::Context::Propagation.text_map_getter)
+          span_context = ::OpenTelemetry::Trace.current_span(parent_context).context
+          links = [::OpenTelemetry::Trace::Link.new(span_context)] if span_context.valid?
+
+          ::OpenTelemetry::Context.with_current(parent_context) do
+            tracer.in_span("consume #{message.topic}", links: links, attributes: consumer_attrs(consumer, message), kind: :consumer) do
+              yield
+            end
+          end
+        end
+
+        def handle_inbox_consumed_one
+          return yield unless enabled?
+
+          inbox_name = @payload[:inbox_name]
+          event_name = @payload[:event_name]
+          status = @payload[:status]
+
+          inbox_attributes = {
+            "inbox.inbox_name" => inbox_name,
+            "inbox.event_name" => event_name,
+            "inbox.status" => status
+          }.compact
+
+          tracer.in_span("inbox #{inbox_name} process", attributes: inbox_attributes, kind: :consumer) do
+            yield
+          end
+        end
+
+        def handle_error
+          return yield unless enabled?
+
+          current_span = OpenTelemetry::Trace.current_span
+          current_span&.status = OpenTelemetry::Trace::Status.error
+
+          yield
+        end
+
+        private
+
+        def tracer
+          ::Sbmt::KafkaConsumer::Instrumentation::OpenTelemetryLoader.instance.tracer
+        end
+
+        def consumer_attrs(consumer, message)
+          attributes = {
+            "messaging.system" => "kafka",
+            "messaging.destination" => message.topic,
+            "messaging.destination_kind" => "topic",
+            "messaging.kafka.consumer_group" => consumer.topic.consumer_group.id,
+            "messaging.kafka.partition" => message.partition,
+            "messaging.kafka.offset" => message.offset
+          }
+
+          message_key = extract_message_key(message.key)
+          attributes["messaging.kafka.message_key"] = message_key if message_key
+
+          attributes.compact
+        end
+
+        def extract_message_key(key)
+          # skip encode if already valid utf8
+          return key if key.nil? || (key.encoding == Encoding::UTF_8 && key.valid_encoding?)
+
+          key.encode(Encoding::UTF_8)
+        rescue Encoding::UndefinedConversionError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/instrumentation/readiness_listener.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Instrumentation
+      class ReadinessListener
+        include ListenerHelper
+        include KafkaConsumer::Probes::Probe
+
+        def initialize
+          setup_subscription
+        end
+
+        def on_app_running(_event)
+          @ready = true
+        end
+
+        def on_app_stopping(_event)
+          @ready = false
+        end
+
+        def probe(_env)
+          ready? ? probe_ok(ready: true) : probe_error(ready: false)
+        end
+
+        private
+
+        def ready?
+          @ready
+        end
+
+        def setup_subscription
+          Karafka::App.monitor.subscribe(self)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/instrumentation/sentry_tracer.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "sentry-ruby"
+require_relative "tracer"
+
+module Sbmt
+  module KafkaConsumer
+    module Instrumentation
+      class SentryTracer < ::Sbmt::KafkaConsumer::Instrumentation::Tracer
+        CONSUMER_ERROR_TYPES = %w[
+          consumer.base.consume_one
+          consumer.inbox.consume_one
+        ].freeze
+
+        def trace(&block)
+          return handle_consumed_one(&block) if @event_id == "consumer.consumed_one"
+          return handle_error(&block) if @event_id == "error.occurred"
+
+          yield
+        end
+
+        def handle_consumed_one
+          return yield unless ::Sentry.initialized?
+
+          consumer = @payload[:caller]
+          message = @payload[:message]
+          trace_id = @payload[:trace_id]
+
+          scope, transaction = start_transaction(trace_id, consumer, message)
+
+          begin
+            yield
+          rescue
+            finish_transaction(transaction, 500)
+            raise
+          end
+
+          finish_transaction(transaction, 200)
+          scope.clear
+        end
+
+        def handle_error
+          return yield unless ::Sentry.initialized?
+
+          exception = @payload[:error]
+          return yield unless exception.respond_to?(:message)
+
+          ::Sentry.with_scope do |scope|
+            if detailed_logging_enabled?
+              message = @payload[:message]
+              if message.present?
+                contexts = {
+                  payload: message_payload(message),
+                  metadata: message.metadata
+                }
+                scope.set_contexts(contexts: contexts)
+              end
+            end
+            ::Sentry.capture_exception(exception)
+          end
+
+          yield
+        end
+
+        private
+
+        def start_transaction(trace_id, consumer, message)
+          scope = ::Sentry.get_current_scope
+          scope.set_tags(trace_id: trace_id, topic: message.topic, offset: message.offset)
+          scope.set_transaction_name("Sbmt/KafkaConsumer/#{consumer.class.name}")
+
+          transaction = ::Sentry.start_transaction(name: scope.transaction_name, op: "kafka-consumer")
+
+          scope.set_span(transaction) if transaction
+
+          [scope, transaction]
+        end
+
+        def finish_transaction(transaction, status)
+          return unless transaction
+
+          transaction.set_http_status(status)
+          transaction.finish
+        end
+
+        def detailed_logging_enabled?
+          consumer = @payload[:caller]
+          event_type = @payload[:type]
+
+          CONSUMER_ERROR_TYPES.include?(event_type) && consumer.send(:log_payload?)
+        end
+
+        def message_payload(message)
+          message.payload
+        rescue => _ex
+          # payload triggers deserialization error
+          # so in that case we return raw_payload
+          message.raw_payload
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/instrumentation/tracer.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Instrumentation
+      class Tracer
+        def initialize(event_id, payload)
+          @event_id = event_id
+          @payload = payload
+        end
+
+        def trace(&block)
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/instrumentation/tracing_monitor.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Instrumentation
+      class TracingMonitor < ChainableMonitor
+        def initialize
+          tracers = []
+          tracers << OpenTelemetryTracer if defined?(OpenTelemetryTracer)
+          tracers << SentryTracer if defined?(SentryTracer)
+
+          super(tracers)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/instrumentation/yabeda_metrics_listener.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Instrumentation
+      class YabedaMetricsListener
+        include ListenerHelper
+
+        delegate :logger, to: ::Sbmt::KafkaConsumer
+
+        def on_statistics_emitted(event)
+          # statistics.emitted is being executed in the main rdkafka thread
+          # so we have to do it in async way to prevent thread's hang issues
+          report_rdkafka_stats(event)
+        end
+
+        def on_consumer_consumed(event)
+          # batch processed
+          consumer = event[:caller]
+
+          Yabeda.kafka_consumer.batch_size
+            .measure(
+              consumer_base_tags(consumer),
+              consumer.messages.count
+            )
+
+          Yabeda.kafka_consumer.process_batch_latency
+            .measure(
+              consumer_base_tags(consumer),
+              time_elapsed_sec(event)
+            )
+
+          Yabeda.kafka_consumer.time_lag
+            .set(
+              consumer_base_tags(consumer),
+              consumer.messages.metadata.consumption_lag
+            )
+        end
+
+        def on_consumer_consumed_one(event)
+          # one message processed by any consumer
+
+          consumer = event[:caller]
+          Yabeda.kafka_consumer.process_messages
+            .increment(consumer_base_tags(consumer))
+          Yabeda.kafka_consumer.process_message_latency
+            .measure(
+              consumer_base_tags(consumer),
+              time_elapsed_sec(event)
+            )
+        end
+
+        def on_consumer_inbox_consumed_one(event)
+          # one message processed by InboxConsumer
+          Yabeda
+            .kafka_consumer
+            .inbox_consumes
+            .increment(consumer_inbox_tags(event))
+        end
+
+        def on_error_occurred(event)
+          caller = event[:caller]
+
+          return unless caller.respond_to?(:messages)
+
+          # caller is a BaseConsumer subclass
+          case event[:type]
+          when "consumer.revoked.error"
+            Yabeda.kafka_consumer.leave_group_errors
+              .increment(consumer_base_tags(caller))
+          when "consumer.consume.error"
+            Yabeda.kafka_consumer.process_batch_errors
+              .increment(consumer_base_tags(caller))
+          when "consumer.base.consume_one"
+            Yabeda.kafka_consumer.process_message_errors
+              .increment(consumer_base_tags(caller))
+          when "consumer.inbox.consume_one"
+            Yabeda.kafka_consumer.inbox_consumes
+              .increment(consumer_inbox_tags(event))
+          end
+        end
+
+        private
+
+        def consumer_base_tags(consumer)
+          {
+            client: Karafka::App.config.client_id,
+            group_id: consumer.topic.consumer_group.id,
+            topic: consumer.messages.metadata.topic,
+            partition: consumer.messages.metadata.partition
+          }
+        end
+
+        def consumer_inbox_tags(event)
+          caller = event[:caller]
+
+          consumer_base_tags(caller)
+            .merge(inbox_tags(event))
+        end
+
+        def report_rdkafka_stats(event, async: true)
+          thread = Thread.new do
+            # https://github.com/confluentinc/librdkafka/blob/master/STATISTICS.md
+            stats = event.payload[:statistics]
+            consumer_group_id = event.payload[:consumer_group_id]
+            consumer_group_stats = stats["cgrp"]
+            broker_stats = stats["brokers"]
+            topic_stats = stats["topics"]
+
+            report_broker_stats(broker_stats)
+            report_consumer_group_stats(consumer_group_id, consumer_group_stats)
+            report_topic_stats(consumer_group_id, topic_stats)
+          rescue => e
+            logger.error("exception happened while reporting rdkafka metrics: #{e.message}")
+            logger.error(e.backtrace&.join("\n"))
+          end
+
+          thread.join unless async
+        end
+
+        def report_broker_stats(brokers)
+          brokers.each_value do |broker_statistics|
+            # Skip bootstrap nodes
+            next if broker_statistics["nodeid"] == -1
+
+            broker_tags = {
+              client: Karafka::App.config.client_id,
+              broker: broker_statistics["nodename"]
+            }
+
+            Yabeda.kafka_api.calls
+              .increment(broker_tags, by: broker_statistics["tx"])
+            Yabeda.kafka_api.latency
+              .measure(broker_tags, broker_statistics["rtt"]["avg"])
+            Yabeda.kafka_api.request_size
+              .measure(broker_tags, broker_statistics["txbytes"])
+            Yabeda.kafka_api.response_size
+              .measure(broker_tags, broker_statistics["rxbytes"])
+            Yabeda.kafka_api.errors
+              .increment(broker_tags, by: broker_statistics["txerrs"] + broker_statistics["rxerrs"])
+          end
+        end
+
+        def report_consumer_group_stats(group_id, group_stats)
+          return if group_stats.blank?
+
+          cg_tags = {
+            client: Karafka::App.config.client_id,
+            group_id: group_id,
+            state: group_stats["state"]
+          }
+
+          Yabeda.kafka_consumer.consumer_group_rebalances
+            .increment(cg_tags, by: group_stats["rebalance_cnt"])
+        end
+
+        def report_topic_stats(group_id, topic_stats)
+          return if topic_stats.blank?
+
+          topic_stats.each do |topic_name, topic_values|
+            topic_values["partitions"].each do |partition_name, partition_statistics|
+              next if partition_name == "-1"
+
+              # Skip until lag info is available
+              offset_lag = partition_statistics["consumer_lag"]
+              next if offset_lag == -1
+
+              Yabeda.kafka_consumer.offset_lag
+                .set({
+                  client: Karafka::App.config.client_id,
+                  group_id: group_id,
+                  topic: topic_name,
+                  partition: partition_name
+                },
+                  offset_lag)
+            end
+          end
+        end
+
+        def time_elapsed_sec(event)
+          (event.payload[:time] || 0) / 1000.0
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/probes/host.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Probes
+      class Host
+        class << self
+          def run_async
+            config = Sbmt::KafkaConsumer::Config.new
+            if config.probes[:port] == config.metrics[:port]
+              start_on_single_port(config)
+            else
+              start_on_different_ports(config)
+            end
+          end
+
+          private
+
+          def health_check_app(config)
+            ::HttpHealthCheck::RackApp.configure do |c|
+              c.logger Rails.logger unless Rails.env.production?
+
+              liveness = config[:liveness]
+              if liveness[:enabled]
+                c.probe liveness[:path], Sbmt::KafkaConsumer::Instrumentation::LivenessListener.new(
+                  timeout_sec: liveness[:timeout]
+                )
+              end
+
+              readiness = config[:readiness]
+              if readiness[:enabled]
+                c.probe readiness[:path], Sbmt::KafkaConsumer::Instrumentation::ReadinessListener.new
+              end
+            end
+          end
+
+          def start_on_single_port(config)
+            app = health_check_app(config.probes[:endpoints])
+            middlewares = defined?(Yabeda) ? {::Yabeda::Prometheus::Exporter => {path: config.metrics[:path]}} : {}
+            start_webrick(app, middlewares: middlewares, port: config.probes[:port])
+          end
+
+          def start_on_different_ports(config)
+            ::HttpHealthCheck.run_server_async(
+              port: config.probes[:port],
+              rack_app: health_check_app(config.probes[:endpoints])
+            )
+            if defined?(Yabeda)
+              start_webrick(
+                Yabeda::Prometheus::Mmap::Exporter::NOT_FOUND_HANDLER,
+                middlewares: {::Yabeda::Prometheus::Exporter => {path: config.metrics[:path]}},
+                port: config.metrics[:port]
+              )
+            end
+          end
+
+          def start_webrick(app, middlewares:, port:)
+            Thread.new do
+              ::Rack::Handler::WEBrick.run(
+                ::Rack::Builder.new do
+                  middlewares.each do |middleware, options|
+                    use middleware, **options
+                  end
+                  run app
+                end,
+                Host: "0.0.0.0",
+                Port: port
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/probes/probe.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Sbmt
+  module KafkaConsumer
+    module Probes
+      module Probe
+        HEADERS = {"Content-Type" => "application/json"}.freeze
+
+        def call(env)
+          with_error_handler { probe(env) }
+        end
+
+        def meta
+          {}
+        end
+
+        def probe_ok(extra_meta = {})
+          [200, HEADERS, [meta.merge(extra_meta).to_json]]
+        end
+
+        def probe_error(extra_meta = {})
+          [500, HEADERS, [meta.merge(extra_meta).to_json]]
+        end
+
+        def with_error_handler
+          yield
+        rescue => error
+          probe_error(error_class: error.class.name, error_message: error.message)
+        end
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/railtie.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+
+module Sbmt
+  module KafkaConsumer
+    class Railtie < Rails::Railtie
+      initializer "sbmt_kafka_consumer_yabeda.configure_rails_initialization" do
+        YabedaConfigurer.configure
+      end
+
+      # it must be consistent with sbmt_karafka initializers' name
+      initializer "sbmt_kafka_consumer_karafka_init.configure_rails_initialization",
+        before: "karafka.require_karafka_boot_file" do
+        # skip loading native karafka.rb, because we want custom init process
+        Karafka.instance_eval do
+          def boot_file; false; end
+        end
+      end
+
+      initializer "sbmt_kafka_consumer_opentelemetry_init.configure_rails_initialization",
+        after: "opentelemetry.configure" do
+        require "sbmt/kafka_consumer/instrumentation/open_telemetry_loader" if defined?(::OpenTelemetry)
+      end
+
+      config.after_initialize do
+        require "sbmt/kafka_consumer/instrumentation/sentry_tracer" if defined?(::Sentry)
+      end
+    end
+  end
+end

data/lib/sbmt/kafka_consumer/routing/karafka_v1_consumer_mapper.rb

@@ -0,0 +1,12 @@
+module Sbmt
+  module KafkaConsumer
+    module Routing
+      class KarafkaV1ConsumerMapper < Karafka::Routing::ConsumerMapper
+        def call(raw_consumer_group_name)
+          client_id = ActiveSupport::Inflector.underscore(Karafka::App.config.client_id).tr("/", "_")
+          "#{client_id}_#{raw_consumer_group_name}"
+        end
+      end
+    end
+  end
+end