sbmt-kafka_consumer 3.4.2 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46ea6a25a613913774156f44f2c81c402b647f3009e83900bdf8e46440c310fd
-  data.tar.gz: 2ee4be3a0439595064d837023fcbcd47a96077ac3e4928599ecdf5cb93b8fb08
+  metadata.gz: 872a886b138e7956051e347be06915f7c9153abd50076ae2ae77e0fc83672e48
+  data.tar.gz: 651678b88fe6f8fde83ec4c086dc7bbfcc5edd5ff1dfd214b151bca76f8b3783
 SHA512:
-  metadata.gz: 1069c06f8d3f8a9195c0b6405042d78131631e9dc391affb1feb1895efe6f4b95b657ca3194d852d80f0c4592cb7cf2d820dbadcd944ab21f2c26ab270c8dfa3
-  data.tar.gz: f75958623481cd4885825e3f8bf61808adbd52ff6d22b8eb19bd049bb0d6a1e2bb4f36e093ba69db0cc657eec694f287c0843d44ae4493b1432cbbfc38fe51bb
+  metadata.gz: 2b60b4a2a5f84f9e3f6bc7602e352d14a373f49c9bab3d30784259e8a41e01e4d5d4324257a283cec6f3a23169ff47824fbf5c702acab891214c1ba806f2da90
+  data.tar.gz: 2ad55a703d2a6e264d1ceca17335d0a9a6c38250eba00d7c4b8dc4848df1a83442e6c6de1e59916be7ac0de9f3d99114b3c1b0fc0dda24501b8f05d38c867b8f

data/.rubocop.yml

@@ -2,10 +2,12 @@ inherit_mode:
   merge:
     - Exclude
 
-require:
+plugins:
   - rubocop-performance
   - rubocop-rails
   - rubocop-rspec
+
+require:
   - standard
 
 inherit_gem:
@@ -22,9 +24,6 @@ AllCops:
   Exclude:
     - spec/internal/pkg/**/*
 
-RSpec/FilePath:
-  Enabled: false
-
 RSpec/VerifiedDoubles:
   Exclude:
     - spec/**/*_spec.rb

data/CHANGELOG.md

@@ -13,6 +13,23 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
+## [3.5.1] - 2025-04-18
+
+### Fixed
+
+- Rename `yabeda_kafka_consumer_consumer_group_rebalances` gauge to `yabeda_kafka_consumer_group_rebalances`
+- Change type of `yabeda_kafka_consumer_group_rebalances` from `Counter` to `Gauge`
+
+## [3.5.0] - 2025-04-17
+
+### Added
+
+- Increase default timeout to 300 seconds for better accommodation of temporary delays.
+- Make liveness probe listener thread-safe using a mutex.
+- Use monotonic clock for more accurate time measurements in liveness checks.
+- Simplify liveness status checking logic.
+- Support Rack v3
+
 ## [3.4.2] - 2025-04-02
 
 ### Fixed

data/README.md

@@ -91,7 +91,7 @@ default: &default
       liveness:
         enabled: true
         path: "/liveness"
-        timeout: 15
+        timeout: 300
         max_error_count: 15 # default 10
   metrics: # optional section
     port: 9090
@@ -157,7 +157,7 @@ auth:
 The `servers` key is required and should be in rdkafka format: without `kafka://` prefix, for example: `srv1:port1,srv2:port2,...`.
 
 The `kafka_config` section may contain any [rdkafka option](https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md). Also, `kafka_options` may be redefined for each topic.
-Please note that the `partition.assignment.strategy` option within kafka_options is not supported for topics; instead, use the global option partition_assignment_strategy. 
+Please note that the `partition.assignment.strategy` option within kafka_options is not supported for topics; instead, use the global option partition_assignment_strategy.
 
 ### `consumer_groups` config section
 
@@ -275,7 +275,7 @@ To process messages in batches, you need to add the `process_batch` method in th
 # app/consumers/some_consumer.rb
 class SomeConsumer < Sbmt::KafkaConsumer::BaseConsumer
   def process_batch(messages)
-    # some code 
+    # some code
   end
 end
 ```

data/lib/sbmt/kafka_consumer/config/probes/liveness_probe.rb

@@ -7,6 +7,6 @@ class Sbmt::KafkaConsumer::Config::Probes::LivenessProbe < Dry::Struct
   attribute :path, Sbmt::KafkaConsumer::Types::Strict::String
     .optional
     .default("/liveness")
-  attribute :timeout, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(10)
+  attribute :timeout, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(300)
   attribute :max_error_count, Sbmt::KafkaConsumer::Types::Coercible::Integer.optional.default(10)
 end

data/lib/sbmt/kafka_consumer/instrumentation/liveness_listener.rb

@@ -9,33 +9,35 @@ module Sbmt
 
         ERROR_TYPE = "Liveness probe error"
 
-        def initialize(timeout_sec: 10, max_error_count: 10)
-          @consumer_groups = Karafka::App.routes.map(&:name)
-          @timeout_sec = timeout_sec
+        def initialize(timeout_sec: 300, max_error_count: 10)
+          @timeout_sec = timeout_sec * 1000
           @max_error_count = max_error_count
           @error_count = 0
           @error_backtrace = nil
           @polls = {}
-
+          @mutex = Mutex.new
           setup_subscription
         end
 
         def probe(_env)
-          now = current_time
-          timed_out_polls = select_timed_out_polls(now)
+          now = monotonic_now
+          has_timed_out_polls = polls.values.any? { |tick| (now - tick) > timeout_sec }
 
-          if timed_out_polls.empty? && @error_count < @max_error_count
-            probe_ok groups: meta_from_polls(polls, now) if timed_out_polls.empty?
+          if !has_timed_out_polls && @error_count < @max_error_count
+            probe_ok timed_out_polls: false, errors_count: @error_count
           elsif @error_count >= @max_error_count
-            probe_error error_type: ERROR_TYPE, failed_librdkafka: {error_count: @error_count, error_backtrace: @error_backtrace}
+            probe_error error_type: ERROR_TYPE, timed_out_polls: false, error_count: @error_count, error_backtrace: @error_backtrace
           else
-            probe_error error_type: ERROR_TYPE, failed_groups: meta_from_polls(timed_out_polls, now)
+            probe_error error_type: ERROR_TYPE, timed_out_polls: true, errors_count: @error_count, polls: polls
           end
         end
 
         def on_connection_listener_fetch_loop(event)
-          consumer_group = event.payload[:subscription_group].consumer_group
-          polls[consumer_group.name] = current_time
+          now = monotonic_now
+          KafkaConsumer.logger.debug("on_connection_listener_fetch_loop: now=#{now}, thread_id=#{thread_id}")
+          mutex.synchronize do
+            polls[thread_id] = monotonic_now
+          end
         end
 
         def on_error_occurred(event)
@@ -50,36 +52,14 @@ module Sbmt
 
         private
 
-        attr_reader :polls, :timeout_sec, :consumer_groups
+        attr_reader :polls, :timeout_sec, :mutex
 
-        def current_time
-          Time.now.utc
+        def monotonic_now
+          ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :float_millisecond)
         end
 
-        def select_timed_out_polls(now)
-          raise "consumer_groups are empty. Please set them up" if consumer_groups.empty?
-
-          consumer_groups.each_with_object({}) do |group, hash|
-            last_poll_at = polls[group]
-            next if last_poll_at && last_poll_at + timeout_sec >= now
-
-            hash[group] = last_poll_at
-          end
-        end
-
-        def meta_from_polls(polls, now)
-          polls.each_with_object({}) do |(group, last_poll_at), hash|
-            if last_poll_at.nil?
-              hash[group] = {had_poll: false}
-              next
-            end
-
-            hash[group] = {
-              had_poll: true,
-              last_poll_at: last_poll_at,
-              seconds_since_last_poll: (now - last_poll_at).to_i
-            }
-          end
+        def thread_id
+          Thread.current.object_id
         end
 
         def setup_subscription

data/lib/sbmt/kafka_consumer/instrumentation/yabeda_metrics_listener.rb

@@ -150,8 +150,8 @@ module Sbmt
             state: group_stats["state"]
           }
 
-          Yabeda.kafka_consumer.consumer_group_rebalances
-            .increment(cg_tags, by: group_stats["rebalance_cnt"])
+          Yabeda.kafka_consumer.group_rebalances
+            .set(cg_tags, group_stats["rebalance_cnt"])
         end
 
         def report_topic_stats(group_id, topic_stats)

data/lib/sbmt/kafka_consumer/probes/host.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "rack"
+require "rackup/handler/webrick" if Gem::Version.new(::Rack.release) >= Gem::Version.new("3")
+
 module Sbmt
   module KafkaConsumer
     module Probes
@@ -14,6 +17,12 @@ module Sbmt
             end
           end
 
+          def webrick
+            return ::Rack::Handler::WEBrick if Gem::Version.new(::Rack.release) < Gem::Version.new("3")
+
+            ::Rackup::Handler::WEBrick
+          end
+
           private
 
           def health_check_app(config)
@@ -56,7 +65,7 @@ module Sbmt
 
           def start_webrick(app, middlewares:, port:)
             Thread.new do
-              ::Rack::Handler::WEBrick.run(
+              webrick.run(
                 ::Rack::Builder.new do
                   middlewares.each do |middleware, options|
                     use middleware, **options

data/lib/sbmt/kafka_consumer/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module KafkaConsumer
-    VERSION = "3.4.2"
+    VERSION = "3.5.1"
   end
 end

data/lib/sbmt/kafka_consumer/yabeda_configurer.rb

@@ -30,7 +30,7 @@ module Sbmt
           end
 
           group :kafka_consumer do
-            counter :consumer_group_rebalances,
+            gauge :group_rebalances,
               tags: %i[client group_id state],
               comment: "Consumer group rebalances"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-kafka_consumer
 version: !ruby/object:Gem::Version
-  version: 3.4.2
+  version: 3.5.1
 platform: ruby
 authors:
 - Kuper Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-04-02 00:00:00.000000000 Z
+date: 2025-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -565,7 +565,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.5.21
 signing_key:
 specification_version: 4
 summary: Ruby gem for consuming Kafka messages