karafka 2.3.0.alpha1 → 2.3.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de7ea23762cefa19d5f3620e92a39f0030cd8ff78f318c92f30c494d79b78163
-  data.tar.gz: 775cfbd40d181036004dcf72dbcb84394dc8367bed0f6d69812f2324dc179d6f
+  metadata.gz: b07678937b0dc9c4154c90ccea2e9bbbc25aa2669461d937e859b77e134ddc48
+  data.tar.gz: a2e3eee8a9351a789dc33a1f78a48899a007da29b81205b4383e75a9fa0cc13c
 SHA512:
-  metadata.gz: d68a4122a35afad517e4280b94f6f3d7cb3cab94fb37c11729e5e5c7a7aca082a7a272a52ff09a86e2f55ad0e078e234c88be79ce3b730527a2f6e7629ef259c
-  data.tar.gz: aa2ddb108cc39caa8ad5c95a86d07006b5be374647e703414a7761ffd5c333010d7e53b9fd2c42216780e35f00639d8cf80126c291a59d0585424077840cc6b5
+  metadata.gz: d2331d328849f4ffb65aa5deb921bf8d12d732f06b30acc5b17ef1fb844d5da23166dbcd27a6b53a6d57d76c6a7d508e9c5479e064a73aa6e0c2eaed69597c8a
+  data.tar.gz: da8afa92e4b5fc552319b15322b3b3e883c2ec5c2b46ec887d098d34f7a0fbd7a7408558a89d781699eb521309c8fe69d2df7e6a1f46de19259451fb3050e391

data/CHANGELOG.md

@@ -1,6 +1,7 @@
 # Karafka framework changelog
 
 ## 2.3.0 (Unreleased)
+- **[Feature]** Introduce Exactly-Once Semantics within consumers `#transaction` block (Pro)
 - **[Feature]** Provide ability to multiplex subscription groups (Pro)
 - **[Feature]** Provide `Karafka::Admin::Acl` for Kafka ACL management via the Admin APIs.
 - **[Feature]** Periodic Jobs (Pro)
@@ -13,6 +14,7 @@
 - [Enhancement] Provide an `:independent` configuration to DLQ allowing to reset pause count track on each marking as consumed when retrying.
 - [Enhancement] Remove no longer needed shutdown patches for `librdkafka` improving multi-sg shutdown times for `cooperative-sticky`.
 - [Enhancement] Allow for parallel closing of connections from independent consumer groups.
+- [Enhancement] Provide recovery flow for cases where DLQ dispatch would fail.
 - [Change] Make `Kubernetes::LivenessListener` not start until Karafka app starts running.
 - [Change] Remove the legacy "inside of topics" way of defining subscription groups names
 - [Change] Update supported instrumentation to report on `#tick`.

data/Gemfile.lock

@@ -1,18 +1,18 @@
 PATH
   remote: .
   specs:
-    karafka (2.3.0.alpha1)
-      karafka-core (>= 2.3.0.alpha1, < 2.4.0)
+    karafka (2.3.0.rc1)
+      karafka-core (>= 2.3.0.rc1, < 2.4.0)
       waterdrop (>= 2.6.12, < 3.0.0)
       zeitwerk (~> 2.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.1.2)
-      activesupport (= 7.1.2)
+    activejob (7.1.3)
+      activesupport (= 7.1.3)
       globalid (>= 0.3.6)
-    activesupport (7.1.2)
+    activesupport (7.1.3)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -23,9 +23,9 @@ GEM
       mutex_m
       tzinfo (~> 2.0)
     base64 (0.2.0)
-    bigdecimal (3.1.5)
+    bigdecimal (3.1.6)
     byebug (11.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
     diff-lcs (1.5.0)
     docile (1.4.0)
@@ -39,7 +39,7 @@ GEM
       activesupport (>= 6.1)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    karafka-core (2.3.0.alpha1)
+    karafka-core (2.3.0.rc1)
       karafka-rdkafka (>= 0.14.7, < 0.15.0)
     karafka-rdkafka (0.14.7)
       ffi (~> 1.15)
@@ -52,11 +52,11 @@ GEM
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     mini_portile2 (2.8.5)
-    minitest (5.20.0)
+    minitest (5.21.2)
     mutex_m (0.2.0)
     rack (3.0.8)
     rake (13.1.0)
-    roda (3.75.0)
+    roda (3.76.0)
       rack
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
@@ -100,4 +100,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.5.3
+   2.5.4

data/bin/integrations

@@ -28,7 +28,7 @@ ROOT_PATH = Pathname.new(File.expand_path(File.join(File.dirname(__FILE__), '../
 CONCURRENCY = ENV.key?('CI') ? 5 : Etc.nprocessors * 3
 
 # How may bytes do we want to keep from the stdout in the buffer for when we need to print it
-MAX_BUFFER_OUTPUT = 51_200
+MAX_BUFFER_OUTPUT = 102_400
 
 # Abstraction around a single test scenario execution process
 class Scenario

data/config/locales/errors.yml

@@ -75,6 +75,7 @@ en:
       dead_letter_queue.topic_format: 'needs to be a string with a Kafka accepted format'
       dead_letter_queue.active_format: needs to be either true or false
       dead_letter_queue.independent_format: needs to be either true or false
+      dead_letter_queue.transactional_format: needs to be either true or false
       active_format: needs to be either true or false
       declaratives.partitions_format: needs to be more or equal to 1
       declaratives.active_format: needs to be true

data/karafka.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
     without having to focus on things that are not your business domain.
   DESC
 
-  spec.add_dependency 'karafka-core', '>= 2.3.0.alpha1', '< 2.4.0'
+  spec.add_dependency 'karafka-core', '>= 2.3.0.rc1', '< 2.4.0'
   spec.add_dependency 'waterdrop', '>= 2.6.12', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 

data/lib/karafka/base_consumer.rb

@@ -79,10 +79,25 @@ module Karafka
     end
 
     # @private
+    #
     # @note This should not be used by the end users as it is part of the lifecycle of things but
     #   not as part of the public api.
+    #
+    # @note We handle and report errors here because of flows that could fail. For example a DLQ
+    #   flow could fail if it was not able to dispatch the DLQ message. Other "non-user" based
+    #   flows do not interact with external systems and their errors are expected to bubble up
     def on_after_consume
       handle_after_consume
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        seek_offset: coordinator.seek_offset,
+        type: 'consumer.after_consume.error'
+      )
+
+      retry_after_pause
     end
 
     # Can be used to run code prior to scheduling of idle execution

data/lib/karafka/errors.rb

@@ -76,5 +76,9 @@ module Karafka
 
     # Raised when we want to un-pause listener that was not paused
     InvalidListenerPauseError = Class.new(BaseError)
+
+    # Raised in transactions when we attempt to store offset for a partition that we have lost
+    # This does not affect producer only transactions, hence we raise it only on offset storage
+    AssignmentLostError = Class.new(BaseError)
   end
 end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -248,7 +248,10 @@ module Karafka
           error "Consumer on shutdown failed due to an error: #{error}"
           error details
         when 'consumer.tick.error'
-          error "Consumer tick failed due to an error: #{error}"
+          error "Consumer on tick failed due to an error: #{error}"
+          error details
+        when 'consumer.after_consume.error'
+          error "Consumer on after_consume failed due to an error: #{error}"
           error details
         when 'worker.process.error'
           fatal "Worker processing failed due to an error: #{error}"

data/lib/karafka/pro/connection/manager.rb

@@ -24,18 +24,9 @@ module Karafka
       # @note Manager operations relate to consumer groups and not subscription groups. Since
       #   cluster operations can cause consumer group wide effects, we always apply only one
       #   change on a consumer group.
-      #
-      # @note Since we collect statistical data from listeners and this happens in a background
-      #   thread, we need to make sure we lock not to have race conditions with expired data
-      #   eviction.
       class Manager < Karafka::Connection::Manager
         include Core::Helpers::Time
 
-        # How long should we keep stale stats before evicting them completely
-        EVICTION_DELAY = 5 * 60 * 1_000
-
-        private_constant :EVICTION_DELAY
-
         # How long should we wait after a rebalance before doing anything on a consumer group
         #
         # @param scale_delay [Integer] How long should we wait before making any changes. Any
@@ -50,7 +41,6 @@ module Karafka
               state: '',
               join_state: '',
               state_age: 0,
-              state_age_sync: monotonic_now,
               changed_at: monotonic_now
             }
           end
@@ -65,6 +55,9 @@ module Karafka
         def register(listeners)
           @listeners = listeners
 
+          # Preload all the keys into the hash so we never add keys to changes but just change them
+          listeners.each { |listener| @changes[listener.subscription_group.id] }
+
           in_sg_families do |first_subscription_group, sg_listeners|
             multiplexing = first_subscription_group.multiplexing
 
@@ -86,25 +79,22 @@ module Karafka
         # @note Please note that while we collect here per subscription group, we use those metrics
         #   collectively on a whole consumer group. This reduces the friction.
         def notice(subscription_group_id, statistics)
-          @mutex.synchronize do
-            times = []
-            # stateage is in microseconds
-            # We monitor broker changes to make sure we do not introduce extra friction
-            times << statistics['brokers'].values.map { |stats| stats['stateage'] }.min / 1_000
-            times << statistics['cgrp']['rebalance_age']
-            times << statistics['cgrp']['stateage']
-
-            # Keep the previous change age for changes that were triggered by us
-            previous_changed_at = @changes[subscription_group_id][:changed_at]
-
-            @changes[subscription_group_id] = {
-              state_age: times.min,
-              changed_at: previous_changed_at,
-              join_state: statistics['cgrp']['join_state'],
-              state: statistics['cgrp']['state'],
-              state_age_sync: monotonic_now
-            }
-          end
+          times = []
+          # stateage is in microseconds
+          # We monitor broker changes to make sure we do not introduce extra friction
+          times << statistics['brokers'].values.map { |stats| stats['stateage'] }.min / 1_000
+          times << statistics['cgrp']['rebalance_age']
+          times << statistics['cgrp']['stateage']
+
+          # Keep the previous change age for changes that were triggered by us
+          previous_changed_at = @changes[subscription_group_id][:changed_at]
+
+          @changes[subscription_group_id].merge!(
+            state_age: times.min,
+            changed_at: previous_changed_at,
+            join_state: statistics['cgrp']['join_state'],
+            state: statistics['cgrp']['state']
+          )
         end
 
         # Shuts down all the listeners when it is time (including moving to quiet) or rescales
@@ -158,8 +148,6 @@ module Karafka
         #
         # We always run scaling down and up because it may be applicable to different CGs
         def rescale
-          evict
-
           scale_down
           scale_up
         end
@@ -232,23 +220,11 @@ module Karafka
           end
         end
 
-        # Removes states that are no longer being reported for stopped/pending listeners
-        def evict
-          @mutex.synchronize do
-            @changes.delete_if do |_, details|
-              monotonic_now - details[:state_age_sync] >= EVICTION_DELAY
-            end
-          end
-        end
-
         # Indicates, that something has changed on a subscription group. We consider every single
         # change we make as a change to the setup as well.
         # @param subscription_group_id [String]
         def touch(subscription_group_id)
-          @mutex.synchronize do
-            @changes[subscription_group_id][:changed_at] = 0
-            @changes[subscription_group_id][:state_age_sync] = monotonic_now
-          end
+          @changes[subscription_group_id][:changed_at] = 0
         end
 
         # @param sg_listeners [Array<Listener>] listeners from one multiplexed sg
@@ -257,17 +233,10 @@ module Karafka
         #   are also stable. This is a strong indicator that no rebalances or other operations are
         #   happening at a given moment.
         def stable?(sg_listeners)
-          # If none of listeners has changes reported it means we did not yet start collecting
-          # metrics about any of them and at least one must be present. We do not consider it
-          # stable in such case as we still are waiting for metrics.
-          return false if sg_listeners.none? do |sg_listener|
-            @changes.key?(sg_listener.subscription_group.id)
-          end
-
           sg_listeners.all? do |sg_listener|
-            # Not all SGs may be started initially or may be stopped, we ignore them here as they
-            # are irrelevant from the point of view of establishing stability
-            next true unless @changes.key?(sg_listener.subscription_group.id)
+            # If a listener is not active, we do not take it into consideration when looking at
+            # the stability data
+            next true unless sg_listener.active?
 
             state = @changes[sg_listener.subscription_group.id]
 

data/lib/karafka/pro/processing/strategies/default.rb

@@ -27,6 +27,21 @@ module Karafka
           # Apply strategy for a non-feature based flow
           FEATURES = %i[].freeze
 
+          # Allows to set offset metadata that will be used with the upcoming marking as consumed
+          # as long as a different offset metadata was not used. After it was used either via
+          # `#mark_as_consumed` or `#mark_as_consumed!` it will be set back to `nil`. It is done
+          # that way to provide the end user with ability to influence metadata on the non-user
+          # initiated markings in complex flows.
+          #
+          # @param offset_metadata [String, nil] metadata we want to store with the upcoming
+          #   marking as consumed
+          #
+          # @note Please be aware, that offset metadata set this way will be passed to any marking
+          #   as consumed even if it was not user initiated. For example in the DLQ flow.
+          def store_offset_metadata(offset_metadata)
+            @_current_offset_metadata = offset_metadata
+          end
+
           # Marks message as consumed in an async way.
           #
           # @param message [Messages::Message] last successfully processed message.
@@ -38,7 +53,7 @@ module Karafka
           #   processing another message. In case like this we do not pause on the message we've
           #   already processed but rather at the next one. This applies to both sync and async
           #   versions of this method.
-          def mark_as_consumed(message, offset_metadata = nil)
+          def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
             if @_in_transaction
               mark_in_transaction(message, offset_metadata, true)
             else
@@ -54,6 +69,8 @@ module Karafka
             end
 
             true
+          ensure
+            @_current_offset_metadata = nil
           end
 
           # Marks message as consumed in a sync way.
@@ -62,7 +79,7 @@ module Karafka
           # @param offset_metadata [String, nil] offset metadata string or nil if nothing
           # @return [Boolean] true if we were able to mark the offset, false otherwise.
           #   False indicates that we were not able and that we have lost the partition.
-          def mark_as_consumed!(message, offset_metadata = nil)
+          def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
             if @_in_transaction
               mark_in_transaction(message, offset_metadata, false)
             else
@@ -79,6 +96,8 @@ module Karafka
             end
 
             true
+          ensure
+            @_current_offset_metadata = nil
           end
 
           # Starts producer transaction, saves the transaction context for transactional marking
@@ -88,8 +107,23 @@ module Karafka
           # allow to mark offsets inside of the transaction. If the transaction is initialized
           # only from the consumer, the offset will be stored in a regular fashion.
           #
+          # @param active_producer [WaterDrop::Producer] alternative producer instance we may want
+          #   to use. It is useful when we have connection pool or any other selective engine for
+          #   managing multiple producers. If not provided, default producer taken from `#producer`
+          #   will be used.
+          #
           # @param block [Proc] code that we want to run in a transaction
-          def transaction(&block)
+          #
+          # @note Please note, that if you provide the producer, it will reassign the producer of
+          #   the consumer for the transaction time. This means, that in case you would even
+          #   accidentally refer to `Consumer#producer` from other threads, it will contain the
+          #   reassigned producer and not the initially used/assigned producer. It is done that
+          #   way, so the message producing aliases operate from within transactions and since the
+          #   producer in transaction is locked, it will prevent other threads from using it.
+          def transaction(active_producer = producer, &block)
+            default_producer = producer
+            self.producer = active_producer
+
             transaction_started = false
 
             # Prevent from nested transactions. It would not make any sense
@@ -119,6 +153,8 @@ module Karafka
               marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
             end
           ensure
+            self.producer = default_producer
+
             if transaction_started
               @_transaction_marked.clear
               @_in_transaction = false
@@ -134,6 +170,7 @@ module Karafka
           #   transaction state synchronization usage as within transaction it is always sync)
           def mark_in_transaction(message, offset_metadata, async)
             raise Errors::TransactionRequiredError unless @_in_transaction
+            raise Errors::AssignmentLostError if revoked?
 
             producer.transaction_mark_as_consumed(
               client,

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -35,7 +35,7 @@ module Karafka
             # @see `Strategies::Default#mark_as_consumed` for more details
             # @param message [Messages::Message]
             # @param offset_metadata [String, nil]
-            def mark_as_consumed(message, offset_metadata = nil)
+            def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
               return super unless retrying?
               return super unless topic.dead_letter_queue.independent?
               return false unless super
@@ -43,6 +43,8 @@ module Karafka
               coordinator.pause_tracker.reset
 
               true
+            ensure
+              @_current_offset_metadata = nil
             end
 
             # Override of the standard `#mark_as_consumed!`. Resets the pause tracker count in case
@@ -51,7 +53,7 @@ module Karafka
             # @see `Strategies::Default#mark_as_consumed!` for more details
             # @param message [Messages::Message]
             # @param offset_metadata [String, nil]
-            def mark_as_consumed!(message, offset_metadata = nil)
+            def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
               return super unless retrying?
               return super unless topic.dead_letter_queue.independent?
               return false unless super
@@ -59,6 +61,8 @@ module Karafka
               coordinator.pause_tracker.reset
 
               true
+            ensure
+              @_current_offset_metadata = nil
             end
 
             # When we encounter non-recoverable message, we skip it and go on with our lives
@@ -80,9 +84,7 @@ module Karafka
                 else
                   # We reset the pause to indicate we will now consider it as "ok".
                   coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
+                  dispatch_if_needed_and_mark_as_consumed
                   pause(coordinator.seek_offset, nil, false)
                 end
               end
@@ -129,6 +131,25 @@ module Karafka
               )
             end
 
+            # Dispatches the message to the DLQ (when needed and when applicable based on settings)
+            #   and marks this message as consumed for non MOM flows.
+            #
+            # If producer is transactional and config allows, uses transaction to do that
+            def dispatch_if_needed_and_mark_as_consumed
+              skippable_message, = find_skippable_message
+
+              dispatch = lambda do
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                mark_as_consumed(skippable_message)
+              end
+
+              if dispatch_in_a_transaction?
+                transaction { dispatch.call }
+              else
+                dispatch.call
+              end
+            end
+
             # @param skippable_message [Array<Karafka::Messages::Message>]
             # @return [Hash] dispatch DLQ message
             def build_dlq_message(skippable_message)
@@ -164,6 +185,13 @@ module Karafka
             def dispatch_to_dlq?
               topic.dead_letter_queue.topic
             end
+
+            # @return [Boolean] should we use a transaction to move the data to the DLQ.
+            #   We can do it only when producer is transactional and configuration for DLQ
+            #   transactional dispatches is not set to false.
+            def dispatch_in_a_transaction?
+              producer.transactional? && topic.dead_letter_queue.transactional?
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -46,9 +46,9 @@ module Karafka
                   retry_after_pause
                 else
                   coordinator.pause_tracker.reset
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
+
+                  dispatch_if_needed_and_mark_as_consumed
+
                   pause(coordinator.seek_offset, nil, false)
                 end
               end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -60,9 +60,8 @@ module Karafka
 
                   return resume if revoked?
 
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
+                  dispatch_if_needed_and_mark_as_consumed
+
                   pause(coordinator.seek_offset, nil, false)
                 end
               end

data/lib/karafka/pro/processing/strategies/dlq/lrj.rb

@@ -49,9 +49,8 @@ module Karafka
 
                   return resume if revoked?
 
-                  skippable_message, = find_skippable_message
-                  dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                  mark_as_consumed(skippable_message)
+                  dispatch_if_needed_and_mark_as_consumed
+
                   pause(coordinator.seek_offset, nil, false)
                 end
               end

data/lib/karafka/pro/processing/strategies/vp/default.rb

@@ -33,7 +33,7 @@ module Karafka
             # @note This virtual offset management uses a regular default marking API underneath.
             #   We do not alter the "real" marking API, as VPs are just one of many cases we want
             #   to support and we do not want to impact them with collective offsets management
-            def mark_as_consumed(message, offset_metadata = nil)
+            def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
               if @_in_transaction && !collapsed?
                 mark_in_transaction(message, offset_metadata, true)
               elsif collapsed?
@@ -55,11 +55,13 @@ module Karafka
                   manager.markable? ? super(*manager.markable) : revoked?
                 end
               end
+            ensure
+              @_current_offset_metadata = nil
             end
 
             # @param message [Karafka::Messages::Message] blocking marks message as consumed
             # @param offset_metadata [String, nil]
-            def mark_as_consumed!(message, offset_metadata = nil)
+            def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
               if @_in_transaction && !collapsed?
                 mark_in_transaction(message, offset_metadata, false)
               elsif collapsed?
@@ -73,6 +75,8 @@ module Karafka
                   manager.markable? ? super(*manager.markable) : revoked?
                 end
               end
+            ensure
+              @_current_offset_metadata = nil
             end
 
             # Stores the next offset for processing inside of the transaction when collapsed and
@@ -93,6 +97,8 @@ module Karafka
             #   transaction state synchronization usage as within transaction it is always sync)
             def mark_in_transaction(message, offset_metadata, async)
               raise Errors::TransactionRequiredError unless @_in_transaction
+              # Prevent from attempts of offset storage when we no longer own the assignment
+              raise Errors::AssignmentLostError if revoked?
 
               return super if collapsed?
 

data/lib/karafka/routing/features/dead_letter_queue/config.rb

@@ -13,10 +13,13 @@ module Karafka
           :topic,
           # Should retries be handled collectively on a batch or independently per message
           :independent,
+          # Move to DLQ and mark as consumed in transactional mode (if applicable)
+          :transactional,
           keyword_init: true
         ) do
           alias_method :active?, :active
           alias_method :independent?, :independent
+          alias_method :transactional?, :transactional
         end
       end
     end

data/lib/karafka/routing/features/dead_letter_queue/contracts/topic.rb

@@ -20,6 +20,7 @@ module Karafka
               required(:active) { |val| [true, false].include?(val) }
               required(:independent) { |val| [true, false].include?(val) }
               required(:max_retries) { |val| val.is_a?(Integer) && val >= 0 }
+              required(:transactional) { |val| [true, false].include?(val) }
             end
 
             # Validate topic name only if dlq is active

data/lib/karafka/routing/features/dead_letter_queue/topic.rb

@@ -16,17 +16,21 @@ module Karafka
           #   if we do not want to move it anywhere and just skip
           # @param independent [Boolean] needs to be true in order for each marking as consumed
           #   in a retry flow to reset the errors counter
+          # @param transactional [Boolean] if applicable, should transaction be used to move
+          #   given message to the dead-letter topic and mark it as consumed.
           # @return [Config] defined config
           def dead_letter_queue(
             max_retries: DEFAULT_MAX_RETRIES,
             topic: nil,
-            independent: false
+            independent: false,
+            transactional: true
           )
             @dead_letter_queue ||= Config.new(
               active: !topic.nil?,
               max_retries: max_retries,
               topic: topic,
-              independent: independent
+              independent: independent,
+              transactional: transactional
             )
           end
 

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.3.0.alpha1'
+  VERSION = '2.3.0.rc1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.3.0.alpha1
+  version: 2.3.0.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-01-15 00:00:00.000000000 Z
+date: 2024-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.alpha1
+        version: 2.3.0.rc1
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0
@@ -53,7 +53,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.0.alpha1
+        version: 2.3.0.rc1
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.4.0

metadata.gz.sig

@@ -1 +1,3 @@
-�ߚډ����bPԻg����;����r�ѽʚ�t⋈t�XV1P���q�ʝ��یmB�sw�=��ly�'�L����~̱��G>������l}P�,���"��f;�1{N*E��B�.i�ݨ|nj6��f)TB�C�&x^8�H���h;�1A�YD�p�3z���*	߮�z7���k�r�Z)�<�3^x|�����+VP�\fq�R��ֳ1�1��J� �\��>
�\�\��O���nk;�L�ƃ¾���OJ<��E�z};�Xu%Pe���`Q}8(6�[bD6�+E�B���d*S�b	�?gg��՜v�r�b7i�K/��rLs��ͪ(��qy�K�����S��Ÿ���&�����|�#��r$#_�Au�^%l�b�
+h!O��R�o�Z�%Q�sp�K�X�v77�������a��F�(=�yB��L�K��V�G�3� ��k���[L^��zGг(�.� թ���.m$�$x�s�5�M#m���@��)R+Ė��l2j_�͚Mγ�W<��̾	̤'���B+v��a���Q����2�}v�uC^�o�m'�F��&v���
����g����#�#���i��%�|�7���CA�������;���i����#��=�D�ׂG�t��m!�<�������<���O��d�^�lkY�b�R�
4x
+8:��I*¼h��y�i��ix���>pէ>�����,��#��-]�t���/q0M�t��W��k�=P��5;�N@!H)2I�j��6�\Mu�L
+&