sbmt-kafka_consumer 2.4.1 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d328765bacde07e4c62f2a9844bb22ab61192ee180d2837a4fdfd1a5e4258dd
-  data.tar.gz: a7b441d6e0a6d2aa29b1e1e3e8474cc0076463b00b56697b8b3cccc204567a14
+  metadata.gz: 167a1f001609eaac9e7cfb33fbd19422a89711d099dedd4b4bdc96b63731562a
+  data.tar.gz: fe081b40a7c1d2462b95874bb9c089b7604db7c7b15e7717ce904462b90383e3
 SHA512:
-  metadata.gz: 61ecdb8df804c1fa008bb1421037f1f96c062d3ec3fb1ac3034ef0a45bfa9641e2bfef4b9bee364597781436e852661a3e04375a3c9236199e557c5abce521d5
-  data.tar.gz: 4eb7926ffb74c3423c0d83a0066ba33d35e6f0f722e93c2dff9f4f953dc7c5988953e5cd55ca5ed71084ee3834c4a8530c940f550a1be88d140c01b82102bc9f
+  metadata.gz: de1d6a79ee8fa4eb55830b186037b17b5d900cb6fba4b19556dbc78d62da7d9d8a70dd1b5c829a99dfb88ea60b78ed49ec61c75b00d377311af693167924f4b8
+  data.tar.gz: d2fd8c167da6f471d00a5cad5542fdc0cd5815a3f0680a1bd9404203ce982bddc3b3d50ce524448113cdf97f9d5cebc56c2e23820c5d53a242f2a98e698d60fd

data/CHANGELOG.md

@@ -13,6 +13,22 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
+## [2.6.0] - 2024-07-01
+
+### Added
+
+- Added instrumentation for methods `process_message` and `mark_as_consumed!`
+
+### Fixed
+
+- From `do_consume(message)` to `yield`
+
+## [2.5.0] - 2024-06-24
+
+### Added
+
+- Added option `max_error_count` for liveness probes, which is triggered when `librdkafka.error`
+
 ## [2.4.1] - 2024-06-15
 
 ### Fixed

data/README.md

@@ -92,6 +92,7 @@ default: &default
         enabled: true
         path: "/liveness"
         timeout: 15
+        max_error_count: 15 # default 10
   metrics: # optional section
     port: 9090
     path: "/metrics"

data/lib/sbmt/kafka_consumer/base_consumer.rb

@@ -52,7 +52,7 @@ module Sbmt
             "consumer.consumed_one",
             caller: self, message: message, trace_id: trace_id
           ) do
-            do_consume(message)
+            yield
           rescue SkipUndeserializableMessage => ex
             instrument_error(ex, message)
             logger.warn("skipping undeserializable message: #{ex.message}")
@@ -87,6 +87,23 @@ module Sbmt
         end
       end
 
+      def with_common_instrumentation(name, message)
+        @trace_id = SecureRandom.base58
+
+        logger.tagged(
+          trace_id: trace_id
+        ) do
+          ::Sbmt::KafkaConsumer.monitor.instrument(
+            "consumer.#{name}",
+            caller: self,
+            message: message,
+            trace_id: trace_id
+          ) do
+            yield
+          end
+        end
+      end
+
       def do_consume(message)
         log_message(message) if log_payload?
 
@@ -94,9 +111,13 @@ module Sbmt
         # so we trigger it explicitly to catch undeserializable message early
         message.payload
 
-        call_middlewares(message, middlewares) { process_message(message) }
+        with_common_instrumentation("process_message", message) do
+          call_middlewares(message, middlewares) { process_message(message) }
+        end
 
-        mark_as_consumed!(message)
+        with_common_instrumentation("mark_as_consumed", message) do
+          mark_as_consumed!(message)
+        end
       end
 
       def skip_on_error

data/lib/sbmt/kafka_consumer/config/probes/liveness_probe.rb

@@ -8,4 +8,5 @@ class Sbmt::KafkaConsumer::Config::Probes::LivenessProbe < Dry::Struct
     .optional
     .default("/liveness")
   attribute :timeout, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(10)
+  attribute :max_error_count, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(10)
 end

data/lib/sbmt/kafka_consumer/instrumentation/base_monitor.rb

@@ -10,6 +10,8 @@ module Sbmt
           consumer.consumed_one
           consumer.inbox.consumed_one
           consumer.consumed_batch
+          consumer.process_message
+          consumer.mark_as_consumed
         ].freeze
 
         def initialize

data/lib/sbmt/kafka_consumer/instrumentation/liveness_listener.rb

@@ -7,9 +7,14 @@ module Sbmt
         include ListenerHelper
         include KafkaConsumer::Probes::Probe
 
-        def initialize(timeout_sec: 10)
+        ERROR_TYPE = "Liveness probe error"
+
+        def initialize(timeout_sec: 10, max_error_count: 10)
           @consumer_groups = Karafka::App.routes.map(&:name)
           @timeout_sec = timeout_sec
+          @max_error_count = max_error_count
+          @error_count = 0
+          @error_backtrace = nil
           @polls = {}
 
           setup_subscription
@@ -18,9 +23,14 @@ module Sbmt
         def probe(_env)
           now = current_time
           timed_out_polls = select_timed_out_polls(now)
-          return probe_ok groups: meta_from_polls(polls, now) if timed_out_polls.empty?
 
-          probe_error failed_groups: meta_from_polls(timed_out_polls, now)
+          if timed_out_polls.empty? && @error_count < @max_error_count
+            probe_ok groups: meta_from_polls(polls, now) if timed_out_polls.empty?
+          elsif @error_count >= @max_error_count
+            probe_error error_type: ERROR_TYPE, failed_librdkafka: {error_count: @error_count, error_backtrace: @error_backtrace}
+          else
+            probe_error error_type: ERROR_TYPE, failed_groups: meta_from_polls(timed_out_polls, now)
+          end
         end
 
         def on_connection_listener_fetch_loop(event)
@@ -28,6 +38,16 @@ module Sbmt
           polls[consumer_group.name] = current_time
         end
 
+        def on_error_occurred(event)
+          type = event[:type]
+
+          return unless type == "librdkafka.error"
+          error = event[:error]
+
+          @error_backtrace ||= (error.backtrace || []).join("\n")
+          @error_count += 1
+        end
+
         private
 
         attr_reader :polls, :timeout_sec, :consumer_groups

data/lib/sbmt/kafka_consumer/instrumentation/logger_listener.rb

@@ -32,6 +32,14 @@ module Sbmt
           logger.info("Successfully consumed message in #{event.payload[:time]} ms")
         end
 
+        def on_consumer_mark_as_consumed(event)
+          logger.info("Processing message in #{event.payload[:time]} ms")
+        end
+
+        def on_consumer_process_message(event)
+          logger.info("Commit offset in #{event.payload[:time]} ms")
+        end
+
         # InboxConsumer events
         def on_consumer_inbox_consumed_one(event)
           logger.tagged(status: event[:status]) do

data/lib/sbmt/kafka_consumer/instrumentation/open_telemetry_tracer.rb

@@ -6,6 +6,11 @@ module Sbmt
   module KafkaConsumer
     module Instrumentation
       class OpenTelemetryTracer < ::Sbmt::KafkaConsumer::Instrumentation::Tracer
+        CONSUMED_EVENTS = %w[
+          consumer.process_message
+          consumer.mark_as_consumed
+        ].freeze
+
         class << self
           def enabled?
             !!@enabled
@@ -22,6 +27,7 @@ module Sbmt
           return handle_consumed_one(&block) if @event_id == "consumer.consumed_one"
           return handle_consumed_batch(&block) if @event_id == "consumer.consumed_batch"
           return handle_inbox_consumed_one(&block) if @event_id == "consumer.inbox.consumed_one"
+          return handle_common_event(&block) if CONSUMED_EVENTS.include?(@event_id)
           return handle_error(&block) if @event_id == "error.occurred"
 
           yield
@@ -79,6 +85,16 @@ module Sbmt
           end
         end
 
+        def handle_common_event(&block)
+          return yield unless enabled?
+
+          if @payload[:inbox_name].present?
+            handle_inbox_consumed_one(&block)
+          else
+            handle_consumed_one(&block)
+          end
+        end
+
         def handle_error
           return yield unless enabled?
 

data/lib/sbmt/kafka_consumer/instrumentation/sentry_tracer.rb

@@ -13,8 +13,14 @@ module Sbmt
           consumer.inbox.consume_one
         ].freeze
 
+        EVENTS = %w[
+          consumer.consumed_one
+          consumer.process_message
+          consumer.mark_as_consumed
+        ].freeze
+
         def trace(&block)
-          return handle_consumed_one(&block) if @event_id == "consumer.consumed_one"
+          return handle_consumed_one(&block) if EVENTS.include?(@event_id)
           return handle_consumed_batch(&block) if @event_id == "consumer.consumed_batch"
           return handle_error(&block) if @event_id == "error.occurred"
 

data/lib/sbmt/kafka_consumer/probes/host.rb

@@ -23,7 +23,7 @@ module Sbmt
               liveness = config[:liveness]
               if liveness[:enabled]
                 c.probe liveness[:path], Sbmt::KafkaConsumer::Instrumentation::LivenessListener.new(
-                  timeout_sec: liveness[:timeout]
+                  timeout_sec: liveness[:timeout], max_error_count: liveness[:max_error_count]
                 )
               end
 

data/lib/sbmt/kafka_consumer/probes/probe.rb

@@ -19,6 +19,7 @@ module Sbmt
         end
 
         def probe_error(extra_meta = {})
+          KafkaConsumer.logger.error("probe error meta: #{meta.merge(extra_meta).inspect}")
           [500, HEADERS, [meta.merge(extra_meta).to_json]]
         end
 

data/lib/sbmt/kafka_consumer/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module KafkaConsumer
-    VERSION = "2.4.1"
+    VERSION = "2.6.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-kafka_consumer
 version: !ruby/object:Gem::Version
-  version: 2.4.1
+  version: 2.6.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-06-17 00:00:00.000000000 Z
+date: 2024-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -570,7 +570,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: Ruby gem for consuming Kafka messages