karafka 2.3.0.alpha2 → 2.3.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 07bebe70b6697a90d04154dcbfe5837f5bcfaf934073f91bbfc9e8939d9d1a6c
-  data.tar.gz: 13e41b276eee5142b55eb6908c8b9292bd8f802f470fb4e9bdc1f812dbd50189
+  metadata.gz: b07678937b0dc9c4154c90ccea2e9bbbc25aa2669461d937e859b77e134ddc48
+  data.tar.gz: a2e3eee8a9351a789dc33a1f78a48899a007da29b81205b4383e75a9fa0cc13c
 SHA512:
-  metadata.gz: fef2cbded4409d951cf7e752f25b7db016052cc5f2a77c54ccf7e8778fbf44edba74cd1b04cf82caf341cf05beda552af5c4cbd994a510c54ad1d3c4e561fd17
-  data.tar.gz: 00bf893d7c6f29e559530585c40ba721ab3a9f3ce5906cd7c5b6948f79725d6d99423444d9ec7e6f8593f9780aec3375e400986ed8478dfab2cbbabb5807a85d
+  metadata.gz: d2331d328849f4ffb65aa5deb921bf8d12d732f06b30acc5b17ef1fb844d5da23166dbcd27a6b53a6d57d76c6a7d508e9c5479e064a73aa6e0c2eaed69597c8a
+  data.tar.gz: da8afa92e4b5fc552319b15322b3b3e883c2ec5c2b46ec887d098d34f7a0fbd7a7408558a89d781699eb521309c8fe69d2df7e6a1f46de19259451fb3050e391

data/CHANGELOG.md

@@ -14,6 +14,7 @@
 - [Enhancement] Provide an `:independent` configuration to DLQ allowing to reset pause count track on each marking as consumed when retrying.
 - [Enhancement] Remove no longer needed shutdown patches for `librdkafka` improving multi-sg shutdown times for `cooperative-sticky`.
 - [Enhancement] Allow for parallel closing of connections from independent consumer groups.
+- [Enhancement] Provide recovery flow for cases where DLQ dispatch would fail.
 - [Change] Make `Kubernetes::LivenessListener` not start until Karafka app starts running.
 - [Change] Remove the legacy "inside of topics" way of defining subscription groups names
 - [Change] Update supported instrumentation to report on `#tick`.

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    karafka (2.3.0.alpha2)
-      karafka-core (>= 2.3.0.alpha1, < 2.4.0)
+    karafka (2.3.0.rc1)
+      karafka-core (>= 2.3.0.rc1, < 2.4.0)
       waterdrop (>= 2.6.12, < 3.0.0)
       zeitwerk (~> 2.3)
 
@@ -23,7 +23,7 @@ GEM
       mutex_m
       tzinfo (~> 2.0)
     base64 (0.2.0)
-    bigdecimal (3.1.5)
+    bigdecimal (3.1.6)
     byebug (11.1.3)
     concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
@@ -39,7 +39,7 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.3.0.alpha1)
+    karafka-core (2.3.0.rc1)
       karafka-rdkafka (>= 0.14.7, < 0.15.0)
     karafka-rdkafka (0.14.7)
       ffi (~> 1.15)
@@ -52,11 +52,11 @@ GEM
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     mini_portile2 (2.8.5)
-    minitest (5.20.0)
+    minitest (5.21.2)
     mutex_m (0.2.0)
     rack (3.0.8)
     rake (13.1.0)
-    roda (3.75.0)
+    roda (3.76.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -100,4 +100,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.5.3
+   2.5.4

data/config/locales/errors.yml

@@ -75,6 +75,7 @@ en:
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false
       dead_letter_queue.independent_format: needs to be either true or false
+      dead_letter_queue.transactional_format: needs to be either true or false
       active_format: needs to be either true or false
       declaratives.partitions_format: needs to be more or equal to 1
       declaratives.active_format: needs to be true

data/karafka.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.3.0.alpha1', '< 2.4.0'
+  spec.add_dependency 'karafka-core', '>= 2.3.0.rc1', '< 2.4.0'
   spec.add_dependency 'waterdrop', '>= 2.6.12', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 

data/lib/karafka/base_consumer.rb

@@ -79,10 +79,25 @@ module Karafka
     end
 
     # @private
+    #
     # @note This should not be used by the end users as it is part of the lifecycle of things but
     #   not as part of the public api.
+    #
+    # @note We handle and report errors here because of flows that could fail. For example a DLQ
+    #   flow could fail if it was not able to dispatch the DLQ message. Other "non-user" based
+    #   flows do not interact with external systems and their errors are expected to bubble up
     def on_after_consume
       handle_after_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        seek_offset: coordinator.seek_offset,
+        type: 'consumer.after_consume.error'
+      )
+
+      retry_after_pause
     end
 
     # Can be used to run code prior to scheduling of idle execution

data/lib/karafka/instrumentation/logger_listener.rb

@@ -248,7 +248,10 @@ module Karafka
           error "Consumer on shutdown failed due to an error: #{error}"
           error details
         when 'consumer.tick.error'
-          error "Consumer tick failed due to an error: #{error}"
+          error "Consumer on tick failed due to an error: #{error}"
+          error details
+        when 'consumer.after_consume.error'
+          error "Consumer on after_consume failed due to an error: #{error}"
           error details
         when 'worker.process.error'
           fatal "Worker processing failed due to an error: #{error}"

data/lib/karafka/pro/processing/strategies/default.rb

@@ -107,8 +107,23 @@ module Karafka
           # allow to mark offsets inside of the transaction. If the transaction is initialized
           # only from the consumer, the offset will be stored in a regular fashion.
           #
+          # @param active_producer [WaterDrop::Producer] alternative producer instance we may want
+          #   to use. It is useful when we have connection pool or any other selective engine for
+          #   managing multiple producers. If not provided, default producer taken from `#producer`
+          #   will be used.
+          #
           # @param block [Proc] code that we want to run in a transaction
-          def transaction(&block)
+          #
+          # @note Please note, that if you provide the producer, it will reassign the producer of
+          #   the consumer for the transaction time. This means, that in case you would even
+          #   accidentally refer to `Consumer#producer` from other threads, it will contain the
+          #   reassigned producer and not the initially used/assigned producer. It is done that
+          #   way, so the message producing aliases operate from within transactions and since the
+          #   producer in transaction is locked, it will prevent other threads from using it.
+          def transaction(active_producer = producer, &block)
+            default_producer = producer
+            self.producer = active_producer
+
             transaction_started = false
 
             # Prevent from nested transactions. It would not make any sense
@@ -138,6 +153,8 @@ module Karafka
               marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
             end
           ensure
+            self.producer = default_producer
+
             if transaction_started
               @_transaction_marked.clear
               @_in_transaction = false

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -84,9 +84,7 @@ module Karafka
                 else
                   # We reset the pause to indicate we will now consider it as "ok".
                   coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
+                  dispatch_if_needed_and_mark_as_consumed
                   pause(coordinator.seek_offset, nil, false)
                 end
               end
@@ -133,6 +131,25 @@ module Karafka
               )
             end
 
+            # Dispatches the message to the DLQ (when needed and when applicable based on settings)
+            #   and marks this message as consumed for non MOM flows.
+            #
+            # If producer is transactional and config allows, uses transaction to do that
+            def dispatch_if_needed_and_mark_as_consumed
+              skippable_message, = find_skippable_message
+
+              dispatch = lambda do
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                mark_as_consumed(skippable_message)
+              end
+
+              if dispatch_in_a_transaction?
+                transaction { dispatch.call }
+              else
+                dispatch.call
+              end
+            end
+
             # @param skippable_message [Array<Karafka::Messages::Message>]
             # @return [Hash] dispatch DLQ message
             def build_dlq_message(skippable_message)
@@ -168,6 +185,13 @@ module Karafka
             def dispatch_to_dlq?
               topic.dead_letter_queue.topic
             end
+
+            # @return [Boolean] should we use a transaction to move the data to the DLQ.
+            #   We can do it only when producer is transactional and configuration for DLQ
+            #   transactional dispatches is not set to false.
+            def dispatch_in_a_transaction?
+              producer.transactional? && topic.dead_letter_queue.transactional?
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -46,9 +46,9 @@ module Karafka
                   retry_after_pause
                 else
                   coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
+
+                  dispatch_if_needed_and_mark_as_consumed
+
                   pause(coordinator.seek_offset, nil, false)
                 end
               end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -60,9 +60,8 @@ module Karafka
 
                   return resume if revoked?
 
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
+                  dispatch_if_needed_and_mark_as_consumed
+
                   pause(coordinator.seek_offset, nil, false)
                 end
               end

data/lib/karafka/pro/processing/strategies/dlq/lrj.rb

@@ -49,9 +49,8 @@ module Karafka
 
                   return resume if revoked?
 
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
+                  dispatch_if_needed_and_mark_as_consumed
+
                   pause(coordinator.seek_offset, nil, false)
                 end
               end

data/lib/karafka/routing/features/dead_letter_queue/config.rb

@@ -13,10 +13,13 @@ module Karafka
           :topic,
           # Should retries be handled collectively on a batch or independently per message
           :independent,
+          # Move to DLQ and mark as consumed in transactional mode (if applicable)
+          :transactional,
           keyword_init: true
         ) do
           alias_method :active?, :active
           alias_method :independent?, :independent
+          alias_method :transactional?, :transactional
         end
       end
     end

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -20,6 +20,7 @@ module Karafka
               required(:active) { |val| [true, false].include?(val) }
               required(:independent) { |val| [true, false].include?(val) }
               required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:transactional) { |val| [true, false].include?(val) }
             end
 
             # Validate topic name only if dlq is active

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -16,17 +16,21 @@ module Karafka
           #   if we do not want to move it anywhere and just skip
           # @param independent [Boolean] needs to be true in order for each marking as consumed
           #   in a retry flow to reset the errors counter
+          # @param transactional [Boolean] if applicable, should transaction be used to move
+          #   given message to the dead-letter topic and mark it as consumed.
           # @return [Config] defined config
           def dead_letter_queue(
             max_retries: DEFAULT_MAX_RETRIES,
             topic: nil,
-            independent: false
+            independent: false,
+            transactional: true
           )
             @dead_letter_queue ||= Config.new(
               active: !topic.nil?,
               max_retries: max_retries,
               topic: topic,
-              independent: independent
+              independent: independent,
+              transactional: transactional
             )
           end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.3.0.alpha2'
+  VERSION = '2.3.0.rc1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.3.0.alpha2
+  version: 2.3.0.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-01-17 00:00:00.000000000 Z
+date: 2024-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.alpha1
+        version: 2.3.0.rc1
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.alpha1
+        version: 2.3.0.rc1
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0