karafka 2.0.9 → 2.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee9c609249fea5e996d9506bd3e04435dbcc189addcd6383a092b080e776f525
-  data.tar.gz: 6ed354b21361966f1988bb90b29cea14d842fb02299f12bd30ab4ec57eaf8cc1
+  metadata.gz: 00fdec9b351cb2da897d141a0dbb97816d7b0e53d7548b86d1ff48c5a8d5af3a
+  data.tar.gz: 514964e409f4904932ee9e7cc1aa27094820d2cbe875c75ad49c82ce41b8afbf
 SHA512:
-  metadata.gz: a20a1bf2d2b86fcd63bf2e036d535c1d7aa4d06943cc00a414851e8f0a0054054eff621b67bfd355456ac93f1561931aec598b43c0ca535ceaa57c9d94957378
-  data.tar.gz: '05996101b929a143926508a0afb69e6e6b09de04fd088dd06c83cba122efc9cfecdd7586892c80ab5409964737dd4a74b0ecea0f8a30ff1dfbf032f6bb289288'
+  metadata.gz: ba9281c9bfdb3a7a2a4c4121c26d1debb4bd39af07a190172a15f34705ee65bd9a1c8ce2e3773ccb3fe749dc83025bf96e0dde31664f1c6541e5923cad87b7a5
+  data.tar.gz: 95f150e80a77f68c16f0ecce849403a7afdcaf50f67bdaa1471074173f8b27b5a6b66d8a5cda8c3dc39106d8153ba843a35c9123883dc803b7bb05ec2c1f47c3

data/.github/workflows/ci.yml

@@ -52,9 +52,14 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          # We run it against the oldest and the newest of a given major to make sure, that there
+          # are no syntax-sugars that we would use that were introduced down the road
           - '3.1'
+          - '3.1.0'
           - '3.0'
+          - '3.0.0'
           - '2.7'
+          - '2.7.0'
         include:
           - ruby: '3.1'
             coverage: 'true'

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Karafka framework changelog
 
+## 2.0.11 (2022-09-29)
+- Report early on errors related to network and on max poll interval being exceeded to indicate critical problems that will be retries but may mean some underlying problems in the system.
+- Fix support of Ruby 2.7.0 to 2.7.2 (#1045)
+
+## 2.0.10 (2022-09-23)
+- Improve error recovery by delegating the recovery to the existing `librdkafka` instance.
+
 ## 2.0.9 (2022-09-22)
 - Fix Singleton not visible when used in PORO (#1034)
 - Divide pristine specs into pristine and poro. Pristine will still have helpers loaded, poro will have nothing.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.9)
+    karafka (2.0.11)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)
@@ -61,14 +61,13 @@ GEM
     thor (1.2.1)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
-    waterdrop (2.4.1)
+    waterdrop (2.4.2)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.10)
       zeitwerk (~> 2.3)
     zeitwerk (2.6.0)
 
 PLATFORMS
-  arm64-darwin
   x86_64-linux
 
 DEPENDENCIES

data/bin/integrations

@@ -81,7 +81,7 @@ class Scenario
 
   # @return [Boolean] any spec that is not a regular one should not run in parallel with others
   def linear?
-    !type == :regular
+    type != :regular
   end
 
   # @return [Boolean] did this scenario finished or is it still running

data/lib/karafka/connection/client.rb

@@ -15,7 +15,7 @@ module Karafka
       attr_reader :name
 
       # How many times should we retry polling in case of a failure
-      MAX_POLL_RETRIES = 10
+      MAX_POLL_RETRIES = 20
 
       private_constant :MAX_POLL_RETRIES
 
@@ -188,6 +188,7 @@ module Karafka
         tpl = topic_partition_list(topic, partition) || @paused_tpls[topic][partition]
 
         return unless tpl
+
         # If we did not have it, it means we never paused this partition, thus no resume should
         # happen in the first place
         return unless @paused_tpls[topic].delete(partition)
@@ -317,7 +318,7 @@ module Karafka
         Rdkafka::Consumer::TopicPartitionList.new({ topic => [rdkafka_partition] })
       end
 
-      # Performs a single poll operation.
+      # Performs a single poll operation and handles retries and error
       #
       # @param timeout [Integer] timeout for a single poll
       # @return [Rdkafka::Consumer::Message, nil] fetched message or nil if nothing polled
@@ -330,44 +331,45 @@ module Karafka
 
         @kafka.poll(timeout)
       rescue ::Rdkafka::RdkafkaError => e
-        # We return nil, so we do not restart until running the whole loop
-        # This allows us to run revocation jobs and other things and we will pick up new work
-        # next time after dispatching all the things that are needed
-        #
-        # If we would retry here, the client reset would become transparent and we would not have
-        # a chance to take any actions
-        early_return = false
+        early_report = false
 
+        # There are retryable issues on which we want to report fast as they are source of
+        # problems and can mean some bigger system instabilities
+        # Those are mainly network issues and exceeding the max poll interval
+        # We want to report early on max poll interval exceeding because it may mean that the
+        # underlying processing is taking too much time and it is not LRJ
         case e.code
         when :max_poll_exceeded # -147
-          reset
-          early_return = true
-        when :transport # -195
-          reset
-          early_return = true
-        when :not_coordinator # 16
-          reset
-          early_return = true
+          early_report = true
         when :network_exception # 13
-          early_return = true
-        when :rebalance_in_progress # -27
-          early_return = true
-        when :coordinator_load_in_progress # 14
-          early_return = true
-        when :unknown_topic_or_part
-          # This is expected and temporary until rdkafka catches up with metadata
-          early_return = true
+          early_report = true
+        when :transport # -195
+          early_report = true
+        end
+
+        retryable = time_poll.attempts <= MAX_POLL_RETRIES && time_poll.retryable?
+
+        if early_report || !retryable
+          Karafka.monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            type: 'connection.client.poll.error'
+          )
         end
 
-        raise if time_poll.attempts > MAX_POLL_RETRIES
-        raise unless time_poll.retryable?
+        raise unless retryable
+
+        # Most of the errors can be safely ignored as librdkafka will recover from them
+        # @see https://github.com/edenhill/librdkafka/issues/1987#issuecomment-422008750
+        # @see https://github.com/edenhill/librdkafka/wiki/Error-handling
 
         time_poll.checkpoint
         time_poll.backoff
 
-        # On unknown errors we do our best to retry and handle them before raising unless we
-        # decide to early return
-        early_return ? nil : retry
+        # poll may not only return message but also can run callbacks and if they changed,
+        # despite the errors we need to delegate to the other app parts
+        @rebalance_manager.changed? ? nil : retry
       end
 
       # Builds a new rdkafka consumer instance based on the subscription group configuration

data/lib/karafka/instrumentation/logger_listener.rb

@@ -153,6 +153,11 @@ module Karafka
         when 'librdkafka.error'
           error "librdkafka internal error occurred: #{error}"
           error details
+        # Those will only occur when retries in the client fail and when they did not stop after
+        # backoffs
+        when 'connection.client.poll.error'
+          error "Data polling error occurred: #{error}"
+          error details
         else
           # This should never happen. Please contact the maintainers
           raise Errors::UnsupportedCaseError, event

data/lib/karafka/pro/base_consumer.rb

@@ -92,7 +92,7 @@ module Karafka
           # If this is not a long-running job there is nothing for us to do here
           return unless topic.long_running_job?
 
-          seek(coordinator.seek_offset)
+          seek(coordinator.seek_offset) unless revoked?
 
           resume
         else

data/lib/karafka/routing/consumer_group.rb

@@ -21,7 +21,7 @@ module Karafka
       #   We add it to make a multi-system development easier for people that don't use
       #   kafka and don't understand the concept of consumer groups.
       def initialize(name)
-        @name = name
+        @name = name.to_s
         @id = Karafka::App.config.consumer_mapper.call(name)
         @topics = Topics.new([])
       end
@@ -52,7 +52,7 @@ module Karafka
       def subscription_group=(name, &block)
         # We cast it here, so the routing supports symbol based but that's anyhow later on
         # validated as a string
-        self.current_subscription_group_name = name.to_s
+        self.current_subscription_group_name = name
 
         Proxy.new(self, &block)
 

data/lib/karafka/routing/proxy.rb

@@ -14,17 +14,22 @@ module Karafka
         instance_eval(&block)
       end
 
-      # Translates the no "=" DSL of routing into elements assignments on target
-      # @param method_name [Symbol] name of the missing method
-      def method_missing(method_name, ...)
-        return super unless respond_to_missing?(method_name)
+      # Ruby 2.7.0 to 2.7.2 do not have arg forwarding, so we fallback to the old way
+      arg_forwarding = RUBY_VERSION < '3.0' ? '*args, &block' : '...'
+
+      class_eval <<~RUBY, __FILE__, __LINE__ + 1
+        # Translates the no "=" DSL of routing into elements assignments on target
+        # @param method_name [Symbol] name of the missing method
+        def method_missing(method_name, #{arg_forwarding})
+          return super unless respond_to_missing?(method_name)
 
-        if @target.respond_to?(:"#{method_name}=")
-          @target.public_send(:"#{method_name}=", ...)
-        else
-          @target.public_send(method_name, ...)
+          if @target.respond_to?(:"\#{method_name}=")
+            @target.public_send(:"\#{method_name}=", #{arg_forwarding})
+          else
+            @target.public_send(method_name, #{arg_forwarding})
+          end
         end
-      end
+      RUBY
 
       # Tells whether or not a given element exists on the target
       # @param method_name [Symbol] name of the missing method

data/lib/karafka/serialization/json/deserializer.rb

@@ -10,6 +10,7 @@ module Karafka
         # @param message [Karafka::Messages::Message] Message object that we want to deserialize
         # @return [Hash] hash with deserialized JSON data
         def call(message)
+          # nil payload can be present for example for tombstone messages
           message.raw_payload.nil? ? nil : ::JSON.parse(message.raw_payload)
         end
       end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.9'
+  VERSION = '2.0.11'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.9
+  version: 2.0.11
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-09-22 00:00:00.000000000 Z
+date: 2022-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core