sbmt-kafka_consumer 3.3.2 → 3.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7918960487cca5e2c992726b0ff8b431e855abb11775645ea5562de36aea17e1
-  data.tar.gz: 526a797e57595ad90658b21c3a419b2e21884918c66669e922e10f0cfe11d696
+  metadata.gz: af93b22c9aee1651414e9403f9de60ffbf2c2d85288a1b5f104f9b7b3399d405
+  data.tar.gz: 3e5a22909901810fee2becfd11a6564575a1c801311eac7e3a47f904bd30a600
 SHA512:
-  metadata.gz: 529347ea9e8e886e442f8bbde28ce1a801691a200f735936aa032364418d35799a3c601df28e30c897cec71049f5e4ab146cfeaf014257c41715e0c818d9ded8
-  data.tar.gz: 3d1a36773274f20fedb611c644cac723826216d4b730027f0267cf5cd3ced59df111e7d5dc33c61416d399ecb528003f03416479e384a4f185a59ea40ee03388
+  metadata.gz: 91720a7822537c66443713a12b8039e1525c09e9c5fc25aa55d91246081b566c91512a51a3595207ab137089db08468810cab9ad1d94fd32a63c97fcc6aef35b
+  data.tar.gz: 6378ea65496f58f5d9dd1543b9d90a181fef5ff6f3ea254f940298b83c4a8deb299de286beacc8ea258a153dea3d135566a2378f0510857ef515da6b34948ab3

data/CHANGELOG.md

@@ -13,6 +13,22 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Fixed
 
+## [3.4.1] - 2025-03-06
+
+### Fixed
+- reset consumer offset lag metric to zero after cg rebalance to avoid reporting stale metrics
+
+## [3.4.0] - 2025-01-27
+
+### Added
+
+- Added support for warning log level in LoggerListener error handling
+- Consider duplicate messages as warnings in logs
+
+### Changed
+
+### Fixed
+
 ## [3.3.2] - 2024-12-27
 
 ### Fixed
@@ -32,20 +48,20 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - inheritance of kafka settings for topic
 
 ### Fixed
- 
+
 - support `karafka 2.4.12`
 - properly report metrics for consumer offset lag when partition fetch assignment was lost
 
 ## [3.2.2] - 2024-09-23
 
 ### Fixed
- 
+
 - log OTEL `trace_id`
 
 ## [3.2.1] - 2024-09-20
 
 ### Fixed
- 
+
 - Limit Karafka version to less than 2.4.12 since `bootstrap.servers` has been made required
 
 ## [3.2.0] - 2024-09-17

data/lib/sbmt/kafka_consumer/base_consumer.rb

@@ -138,7 +138,8 @@ module Sbmt
           error: error,
           caller: self,
           message: message,
-          type: "consumer.base.consume_one"
+          type: "consumer.base.consume_one",
+          log_level: :error
         )
       end
 

data/lib/sbmt/kafka_consumer/inbox_consumer.rb

@@ -56,7 +56,7 @@ module Sbmt
         item = result.success
         item.track_metrics_after_consume if item.respond_to?(:track_metrics_after_consume)
       rescue ActiveRecord::RecordNotUnique
-        instrument_error("Skipped duplicate message for #{inbox_name}, message_uuid: #{message_uuid(message)}", message, "duplicate")
+        instrument_warn("Skipped duplicate message for #{inbox_name}, message_uuid: #{message_uuid(message)}", message, "duplicate")
       rescue => ex
         if skip_on_error
           logger.warn("skipping unprocessable message for #{inbox_name}, message_uuid: #{message_uuid(message)}")
@@ -108,7 +108,7 @@ module Sbmt
         inbox_item_class.box_name
       end
 
-      def instrument_error(error, message, status = "failure")
+      def instrument_error(error, message, status = "failure", log_level: :error)
         ::Sbmt::KafkaConsumer.monitor.instrument(
           "error.occurred",
           error: error,
@@ -117,9 +117,14 @@ module Sbmt
           inbox_name: inbox_name,
           event_name: event_name,
           status: status,
-          type: "consumer.inbox.consume_one"
+          type: "consumer.inbox.consume_one",
+          log_level: log_level
         )
       end
+
+      def instrument_warn(*args, **kwargs)
+        instrument_error(*args, **kwargs, log_level: :warn)
+      end
     end
   end
 end

data/lib/sbmt/kafka_consumer/instrumentation/logger_listener.rb

@@ -6,6 +6,7 @@ module Sbmt
       class LoggerListener < Karafka::Instrumentation::LoggerListener
         include ListenerHelper
         CUSTOM_ERROR_TYPES = %w[consumer.base.consume_one consumer.inbox.consume_one].freeze
+        VALID_LOG_LEVELS = %i[error warn].freeze
 
         def on_error_occurred(event)
           type = event[:type]
@@ -23,7 +24,12 @@ module Sbmt
             stacktrace: log_backtrace(error),
             **tags
           ) do
-            logger.error(error_message(error))
+            log_level = event[:log_level] || :error
+            if VALID_LOG_LEVELS.include?(log_level)
+              logger.public_send(log_level, error_message(error))
+            else
+              raise "Invalid log level #{log_level}"
+            end
           end
         end
 

data/lib/sbmt/kafka_consumer/instrumentation/yabeda_metrics_listener.rb

@@ -165,7 +165,10 @@ module Sbmt
               offset_lag = partition_statistics["consumer_lag"]
               next if offset_lag == -1
 
-              next unless partition_owned?(partition_statistics)
+              unless partition_owned?(partition_statistics)
+                # reset offset lag after cg rebalance
+                offset_lag = 0
+              end
 
               Yabeda.kafka_consumer.offset_lag
                 .set({

data/lib/sbmt/kafka_consumer/version.rb

@@ -2,6 +2,6 @@
 
 module Sbmt
   module KafkaConsumer
-    VERSION = "3.3.2"
+    VERSION = "3.4.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sbmt-kafka_consumer
 version: !ruby/object:Gem::Version
-  version: 3.3.2
+  version: 3.4.1
 platform: ruby
 authors:
 - Kuper Ruby-Platform Team
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-12-27 00:00:00.000000000 Z
+date: 2025-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -565,7 +565,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Ruby gem for consuming Kafka messages