karafka 2.0.9 → 2.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee9c609249fea5e996d9506bd3e04435dbcc189addcd6383a092b080e776f525
-  data.tar.gz: 6ed354b21361966f1988bb90b29cea14d842fb02299f12bd30ab4ec57eaf8cc1
+  metadata.gz: f78f7cb985880d9172961be96386a0ebd37735831915f0cb9b9b46c832d2e9a9
+  data.tar.gz: a7f5a27cb3a6f0fa5185f32e5df7615f8a6013ef0018c85fb16c87de346f9362
 SHA512:
-  metadata.gz: a20a1bf2d2b86fcd63bf2e036d535c1d7aa4d06943cc00a414851e8f0a0054054eff621b67bfd355456ac93f1561931aec598b43c0ca535ceaa57c9d94957378
-  data.tar.gz: '05996101b929a143926508a0afb69e6e6b09de04fd088dd06c83cba122efc9cfecdd7586892c80ab5409964737dd4a74b0ecea0f8a30ff1dfbf032f6bb289288'
+  metadata.gz: 5b6fed517f69a2bd84b16e82d266251dd4b7564d52f466745f3a93be9106f866ec658581c38b1a0913dd9e126a17cf4b1f5e85c888447c6e1e4b790453e7c87e
+  data.tar.gz: 25cfa154028dd12519d7d4b72606242a4058c9fabd1655ba727a9d73a905a942cd80374e2f7357f410a098b74b623966fa840b1c6725f71d630bf3ce92db187e

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Karafka framework changelog
 
+## 2.0.10 (2022-09-23)
+- Improve error recovery by delegating the recovery to the existing `librdkafka` instance.
+
 ## 2.0.9 (2022-09-22)
 - Fix Singleton not visible when used in PORO (#1034)
 - Divide pristine specs into pristine and poro. Pristine will still have helpers loaded, poro will have nothing.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.9)
+    karafka (2.0.10)
       karafka-core (>= 2.0.2, < 3.0.0)
       rdkafka (>= 0.12)
       thor (>= 0.20)

data/bin/integrations

@@ -81,7 +81,7 @@ class Scenario
 
   # @return [Boolean] any spec that is not a regular one should not run in parallel with others
   def linear?
-    !type == :regular
+    type != :regular
   end
 
   # @return [Boolean] did this scenario finished or is it still running

data/lib/karafka/connection/client.rb

@@ -15,7 +15,7 @@ module Karafka
       attr_reader :name
 
       # How many times should we retry polling in case of a failure
-      MAX_POLL_RETRIES = 10
+      MAX_POLL_RETRIES = 20
 
       private_constant :MAX_POLL_RETRIES
 
@@ -330,44 +330,26 @@ module Karafka
 
         @kafka.poll(timeout)
       rescue ::Rdkafka::RdkafkaError => e
-        # We return nil, so we do not restart until running the whole loop
-        # This allows us to run revocation jobs and other things and we will pick up new work
-        # next time after dispatching all the things that are needed
-        #
-        # If we would retry here, the client reset would become transparent and we would not have
-        # a chance to take any actions
-        early_return = false
+        # Most of the errors can be safely ignored as librdkafka will recover from them
+        # @see https://github.com/edenhill/librdkafka/issues/1987#issuecomment-422008750
+        # @see https://github.com/edenhill/librdkafka/wiki/Error-handling
+        if time_poll.attempts > MAX_POLL_RETRIES || !time_poll.retryable?
+          Karafka.monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            type: 'connection.client.poll.error'
+          )
 
-        case e.code
-        when :max_poll_exceeded # -147
-          reset
-          early_return = true
-        when :transport # -195
-          reset
-          early_return = true
-        when :not_coordinator # 16
-          reset
-          early_return = true
-        when :network_exception # 13
-          early_return = true
-        when :rebalance_in_progress # -27
-          early_return = true
-        when :coordinator_load_in_progress # 14
-          early_return = true
-        when :unknown_topic_or_part
-          # This is expected and temporary until rdkafka catches up with metadata
-          early_return = true
+          raise
         end
 
-        raise if time_poll.attempts > MAX_POLL_RETRIES
-        raise unless time_poll.retryable?
-
         time_poll.checkpoint
         time_poll.backoff
 
-        # On unknown errors we do our best to retry and handle them before raising unless we
-        # decide to early return
-        early_return ? nil : retry
+        # poll may not only return message but also can run callbacks and if they changed,
+        # despite the errors we need to delegate to the other app parts
+        @rebalance_manager.changed? ? nil : retry
       end
 
       # Builds a new rdkafka consumer instance based on the subscription group configuration

data/lib/karafka/instrumentation/logger_listener.rb

@@ -153,6 +153,11 @@ module Karafka
         when 'librdkafka.error'
           error "librdkafka internal error occurred: #{error}"
           error details
+        # Those will only occur when retries in the client fail and when they did not stop after
+        # backoffs
+        when 'connection.client.poll.error'
+          error "Data polling error occurred: #{error}"
+          error details
         else
           # This should never happen. Please contact the maintainers
           raise Errors::UnsupportedCaseError, event

data/lib/karafka/pro/base_consumer.rb

@@ -92,7 +92,7 @@ module Karafka
           # If this is not a long-running job there is nothing for us to do here
           return unless topic.long_running_job?
 
-          seek(coordinator.seek_offset)
+          seek(coordinator.seek_offset) unless revoked?
 
           resume
         else

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.0.9'
+  VERSION = '2.0.10'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.0.9
+  version: 2.0.10
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-09-22 00:00:00.000000000 Z
+date: 2022-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core