sbmt-kafka_consumer 2.4.0 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3aaf5a3d0cab0fe477ab5af189b754b642c403036eec6f7329818645a1dff362
-  data.tar.gz: f198d12579b3437f903fff1c7be33246c5121b1b37bb735e861c7c0f9cbb0314
+  metadata.gz: 6e41656d16c2a1b741868c2d079e4e83c3df397e5e7491680e417738231afe71
+  data.tar.gz: '09117d5de7194d45bf44235f731adfd4e6f60cc18d6c613c1fe2a3800048a89c'
 SHA512:
-  metadata.gz: 468be419229e1096fbae7a0892607c33c8913f686478bca5d8ed8816ac83c8932be7683eedbfbc8eeb55225578dd74707c7bc256947691e65fa19d7142fdb095
-  data.tar.gz: 105b7fdd0b37927b92e81819035442705e9a636828469f47b61ce5bb67850db2f1c921e5d45b9bcadedb89eaceb6e929a3fde903672517a9aae5810e68ef0627
+  metadata.gz: 0ce2e28e58baf91e4e1612d02c37767399e643ed99c78f26adfcae1e23e77aab8f1af8bda64c6f91b16456e069b24f7db53812843347924b124b9e890d8f4e69
+  data.tar.gz: 3447e5c1c06c3f636d4c5f37070a9f349714aea586710ba7c87d2cd3248ba5ad686bfca28d1550ec0af86d18cd556315945596c7b7c069fa31a4d0db2e1425b9

data/CHANGELOG.md

@@ -13,6 +13,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
+## [2.5.0] - 2024-06-24
+
+### Added
+
+- Added option `max_error_count` for liveness probes, which is triggered when `librdkafka.error`
+
+## [2.4.1] - 2024-06-15
+
+### Fixed
+
+- Fixed display of metrics `kafka_api_calls`  and `kafka_api_errors`
+
 ## [2.4.0] - 2024-06-06
 
 ### Added

data/README.md

@@ -92,6 +92,7 @@ default: &default
         enabled: true
         path: "/liveness"
         timeout: 15
+        max_error_count: 15 # default 10
   metrics: # optional section
     port: 9090
     path: "/metrics"

data/lib/sbmt/kafka_consumer/config/probes/liveness_probe.rb

@@ -8,4 +8,5 @@ class Sbmt::KafkaConsumer::Config::Probes::LivenessProbe < Dry::Struct
     .optional
     .default("/liveness")
   attribute :timeout, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(10)
+  attribute :max_error_count, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(10)
 end

data/lib/sbmt/kafka_consumer/instrumentation/liveness_listener.rb

@@ -7,9 +7,14 @@ module Sbmt
         include ListenerHelper
         include KafkaConsumer::Probes::Probe
 
-        def initialize(timeout_sec: 10)
+        ERROR_TYPE = "Liveness probe error"
+
+        def initialize(timeout_sec: 10, max_error_count: 10)
           @consumer_groups = Karafka::App.routes.map(&:name)
           @timeout_sec = timeout_sec
+          @max_error_count = max_error_count
+          @error_count = 0
+          @error_backtrace = nil
           @polls = {}
 
           setup_subscription
@@ -18,9 +23,14 @@ module Sbmt
         def probe(_env)
           now = current_time
           timed_out_polls = select_timed_out_polls(now)
-          return probe_ok groups: meta_from_polls(polls, now) if timed_out_polls.empty?
 
-          probe_error failed_groups: meta_from_polls(timed_out_polls, now)
+          if timed_out_polls.empty? && @error_count < @max_error_count
+            probe_ok groups: meta_from_polls(polls, now) if timed_out_polls.empty?
+          elsif @error_count >= @max_error_count
+            probe_error error_type: ERROR_TYPE, failed_librdkafka: {error_count: @error_count, error_backtrace: @error_backtrace}
+          else
+            probe_error error_type: ERROR_TYPE, failed_groups: meta_from_polls(timed_out_polls, now)
+          end
         end
 
         def on_connection_listener_fetch_loop(event)
@@ -28,6 +38,16 @@ module Sbmt
           polls[consumer_group.name] = current_time
         end
 
+        def on_error_occurred(event)
+          type = event[:type]
+
+          return unless type == "librdkafka.error"
+          error = event[:error]
+
+          @error_backtrace ||= (error.backtrace || []).join("\n")
+          @error_count += 1
+        end
+
         private
 
         attr_reader :polls, :timeout_sec, :consumer_groups

data/lib/sbmt/kafka_consumer/instrumentation/yabeda_metrics_listener.rb

@@ -129,7 +129,7 @@ module Sbmt
             }
 
             Yabeda.kafka_api.calls
-              .increment(broker_tags, by: broker_statistics["tx"])
+              .increment(broker_tags, by: broker_statistics["tx_d"])
             Yabeda.kafka_api.latency
               .measure(broker_tags, broker_statistics["rtt"]["avg"])
             Yabeda.kafka_api.request_size
@@ -137,7 +137,7 @@ module Sbmt
             Yabeda.kafka_api.response_size
               .measure(broker_tags, broker_statistics["rxbytes"])
             Yabeda.kafka_api.errors
-              .increment(broker_tags, by: broker_statistics["txerrs"] + broker_statistics["rxerrs"])
+              .increment(broker_tags, by: broker_statistics["txerrs_d"] + broker_statistics["rxerrs_d"])
           end
         end
 

data/lib/sbmt/kafka_consumer/probes/host.rb

@@ -23,7 +23,7 @@ module Sbmt
               liveness = config[:liveness]
               if liveness[:enabled]
                 c.probe liveness[:path], Sbmt::KafkaConsumer::Instrumentation::LivenessListener.new(
-                  timeout_sec: liveness[:timeout]
+                  timeout_sec: liveness[:timeout], max_error_count: liveness[:max_error_count]
                 )
               end
 

data/lib/sbmt/kafka_consumer/probes/probe.rb

@@ -19,6 +19,7 @@ module Sbmt
         end
 
         def probe_error(extra_meta = {})
+          KafkaConsumer.logger.error("probe error meta: #{meta.merge(extra_meta).inspect}")
           [500, HEADERS, [meta.merge(extra_meta).to_json]]
         end
 

data/lib/sbmt/kafka_consumer/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module KafkaConsumer
-    VERSION = "2.4.0"
+    VERSION = "2.5.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-kafka_consumer
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.5.0
 platform: ruby
 authors:
 - Sbermarket Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-06-07 00:00:00.000000000 Z
+date: 2024-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -570,7 +570,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Ruby gem for consuming Kafka messages