karafka 2.4.0 → 2.4.18

data/lib/karafka/pro/processing/strategies/default.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -54,18 +46,30 @@ module Karafka
           #   already processed but rather at the next one. This applies to both sync and async
           #   versions of this method.
           def mark_as_consumed(message, offset_metadata = @_current_offset_metadata)
+            # If we are inside a transaction than we can just mark as consumed within it
             if @_in_transaction
               mark_in_transaction(message, offset_metadata, true)
+            elsif @_in_transaction_marked
+              mark_in_memory(message)
             else
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
-              return true if coordinator.seek_offset.nil?
+              return true if seek_offset.nil?
               # Ignore earlier offsets than the one we already committed
-              return true if coordinator.seek_offset > message.offset
+              return true if seek_offset > message.offset
               return false if revoked?
-              return revoked? unless client.mark_as_consumed(message, offset_metadata)
 
-              coordinator.seek_offset = message.offset + 1
+              # If we are not inside a transaction but this is a transactional topic, we mark with
+              # artificially created transaction
+              stored = if producer.transactional?
+                         mark_with_transaction(message, offset_metadata, true)
+                       else
+                         client.mark_as_consumed(message, offset_metadata)
+                       end
+
+              return revoked? unless stored
+
+              self.seek_offset = message.offset + 1
             end
 
             true
@@ -82,17 +86,27 @@ module Karafka
           def mark_as_consumed!(message, offset_metadata = @_current_offset_metadata)
             if @_in_transaction
               mark_in_transaction(message, offset_metadata, false)
+            elsif @_in_transaction_marked
+              mark_in_memory(message)
             else
               # seek offset can be nil only in case `#seek` was invoked with offset reset request
               # In case like this we ignore marking
-              return true if coordinator.seek_offset.nil?
+              return true if seek_offset.nil?
               # Ignore earlier offsets than the one we already committed
-              return true if coordinator.seek_offset > message.offset
+              return true if seek_offset > message.offset
               return false if revoked?
 
-              return revoked? unless client.mark_as_consumed!(message, offset_metadata)
+              # If we are not inside a transaction but this is a transactional topic, we mark with
+              # artificially created transaction
+              stored = if producer.transactional?
+                         mark_with_transaction(message, offset_metadata, false)
+                       else
+                         client.mark_as_consumed!(message, offset_metadata)
+                       end
 
-              coordinator.seek_offset = message.offset + 1
+              return revoked? unless stored
+
+              self.seek_offset = message.offset + 1
             end
 
             true
@@ -112,7 +126,7 @@ module Karafka
           #   managing multiple producers. If not provided, default producer taken from `#producer`
           #   will be used.
           #
-          # @param block [Proc] code that we want to run in a transaction
+          # @yield code that we want to run in a transaction
           #
           # @note Please note, that if you provide the producer, it will reassign the producer of
           #   the consumer for the transaction time. This means, that in case you would even
@@ -120,37 +134,57 @@ module Karafka
           #   reassigned producer and not the initially used/assigned producer. It is done that
           #   way, so the message producing aliases operate from within transactions and since the
           #   producer in transaction is locked, it will prevent other threads from using it.
-          def transaction(active_producer = producer, &block)
-            default_producer = producer
-            self.producer = active_producer
-
-            transaction_started = false
-
-            # Prevent from nested transactions. It would not make any sense
-            raise Errors::TransactionAlreadyInitializedError if @_in_transaction
-
-            transaction_started = true
-            @_transaction_marked = []
-            @_in_transaction = true
-
-            producer.transaction(&block)
-
-            @_in_transaction = false
-
-            # This offset is already stored in transaction but we set it here anyhow because we
-            # want to make sure our internal in-memory state is aligned with the transaction
-            #
-            # @note We never need to use the blocking `#mark_as_consumed!` here because the offset
-            #   anyhow was already stored during the transaction
-            #
-            # @note In theory we could only keep reference to the most recent marking and reject
-            #   others. We however do not do it for two reasons:
-            #   - User may have non standard flow relying on some alternative order and we want to
-            #     mimic this
-            #   - Complex strategies like VPs can use this in VPs to mark in parallel without
-            #     having to redefine the transactional flow completely
-            @_transaction_marked.each do |marking|
-              marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
+          def transaction(active_producer = producer)
+            default_producer = nil
+            transaction_started = nil
+
+            monitor.instrument('consumer.consuming.transaction', caller: self) do
+              default_producer = producer
+              self.producer = active_producer
+
+              transaction_started = false
+
+              # Prevent from nested transactions. It would not make any sense
+              raise Errors::TransactionAlreadyInitializedError if @_in_transaction
+
+              transaction_started = true
+              @_transaction_marked = []
+              @_in_transaction = true
+              @_in_transaction_marked = false
+
+              producer.transaction do
+                yield
+
+                # Ensure this transaction is rolled back if we have lost the ownership of this
+                # transaction. We do it only for transactions that contain offset management as for
+                # producer only, this is not relevant.
+                raise Errors::AssignmentLostError if @_in_transaction_marked && revoked?
+              end
+
+              @_in_transaction = false
+
+              # This offset is already stored in transaction but we set it here anyhow because we
+              # want to make sure our internal in-memory state is aligned with the transaction
+              #
+              # @note We never need to use the blocking `#mark_as_consumed!` here because the
+              #   offset anyhow was already stored during the transaction
+              #
+              # @note Since the offset could have been already stored in Kafka (could have because
+              #   you can have transactions without marking), we use the `@_in_transaction_marked`
+              #   state to decide if we need to dispatch the offset via client at all
+              #   (if post transaction, then we do not have to)
+              #
+              # @note In theory we could only keep reference to the most recent marking and reject
+              #   others. We however do not do it for two reasons:
+              #   - User may have non standard flow relying on some alternative order and we want
+              #     to mimic this
+              #   - Complex strategies like VPs can use this in VPs to mark in parallel without
+              #     having to redefine the transactional flow completely
+              @_transaction_marked.each do |marking|
+                marking.pop ? mark_as_consumed(*marking) : mark_as_consumed!(*marking)
+              end
+
+              true
             end
           ensure
             self.producer = default_producer
@@ -158,6 +192,7 @@ module Karafka
             if transaction_started
               @_transaction_marked.clear
               @_in_transaction = false
+              @_in_transaction_marked = false
             end
           end
 
@@ -178,13 +213,60 @@ module Karafka
               offset_metadata
             )
 
+            @_in_transaction_marked = true
             @_transaction_marked ||= []
             @_transaction_marked << [message, offset_metadata, async]
           end
 
+          # @private
+          # @param message [Messages::Message] message we want to commit inside of a transaction
+          # @param offset_metadata [String, nil] offset metadata or nil if none
+          # @param async [Boolean] should we mark in async or sync way (applicable only to post
+          #   transaction state synchronization usage as within transaction it is always sync)
+          # @return [Boolean] false if marking failed otherwise true
+          def mark_with_transaction(message, offset_metadata, async)
+            # This flag is used by VPs to differentiate between user initiated transactions and
+            # post-execution system transactions.
+            @_transaction_internal = true
+
+            transaction do
+              mark_in_transaction(message, offset_metadata, async)
+            end
+
+            true
+          # We handle both cases here because this is a private API for internal usage and we want
+          # the post-user code execution marking with transactional producer to result in a
+          # boolean state of marking for further framework flow. This is a normalization to make it
+          # behave the same way as it would behave with a non-transactional one
+          rescue ::Rdkafka::RdkafkaError, Errors::AssignmentLostError
+            false
+          ensure
+            @_transaction_internal = false
+          end
+
+          # Marks the current state only in memory as the offset marking has already happened
+          # using the producer transaction
+          # @param message [Messages::Message] last successfully processed message.
+          # @return [Boolean] true if all good, false if we lost assignment and no point in marking
+          def mark_in_memory(message)
+            # seek offset can be nil only in case `#seek` was invoked with offset reset request
+            # In case like this we ignore marking
+            return true if seek_offset.nil?
+            # Ignore earlier offsets than the one we already committed
+            return true if seek_offset > message.offset
+            return false if revoked?
+
+            # If we have already marked this successfully in a transaction that was running
+            # we should not mark it again with the client offset delegation but instead we should
+            # just align the in-memory state
+            self.seek_offset = message.offset + 1
+
+            true
+          end
+
           # No actions needed for the standard flow here
           def handle_before_schedule_consume
-            Karafka.monitor.instrument('consumer.before_schedule_consume', caller: self)
+            monitor.instrument('consumer.before_schedule_consume', caller: self)
 
             nil
           end
@@ -203,8 +285,8 @@ module Karafka
             # This can happen primarily when an LRJ job gets to the internal worker queue and
             # this partition is revoked prior processing.
             unless revoked?
-              Karafka.monitor.instrument('consumer.consume', caller: self)
-              Karafka.monitor.instrument('consumer.consumed', caller: self) do
+              monitor.instrument('consumer.consume', caller: self)
+              monitor.instrument('consumer.consumed', caller: self) do
                 consume
               end
             end
@@ -250,8 +332,8 @@ module Karafka
               coordinator.revoke
             end
 
-            Karafka.monitor.instrument('consumer.revoke', caller: self)
-            Karafka.monitor.instrument('consumer.revoked', caller: self) do
+            monitor.instrument('consumer.revoke', caller: self)
+            monitor.instrument('consumer.revoked', caller: self) do
               revoked
             end
           ensure
@@ -260,15 +342,15 @@ module Karafka
 
           # No action needed for the tick standard flow
           def handle_before_schedule_tick
-            Karafka.monitor.instrument('consumer.before_schedule_tick', caller: self)
+            monitor.instrument('consumer.before_schedule_tick', caller: self)
 
             nil
           end
 
           # Runs the consumer `#tick` method with reporting
           def handle_tick
-            Karafka.monitor.instrument('consumer.tick', caller: self)
-            Karafka.monitor.instrument('consumer.ticked', caller: self) do
+            monitor.instrument('consumer.tick', caller: self)
+            monitor.instrument('consumer.ticked', caller: self) do
               tick
             end
           ensure

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -90,7 +82,7 @@ module Karafka
             #   information if this message was from marked offset or figured out via mom flow
             def find_skippable_message
               skippable_message = messages.find do |msg|
-                coordinator.marked? && msg.offset == coordinator.seek_offset
+                coordinator.marked? && msg.offset == seek_offset
               end
 
               # If we don't have the message matching the last comitted offset, it means that
@@ -119,7 +111,7 @@ module Karafka
               )
 
               # Notify about dispatch on the events bus
-              Karafka.monitor.instrument(
+              monitor.instrument(
                 'dead_letter_queue.dispatched',
                 caller: self,
                 message: skippable_message
@@ -135,7 +127,12 @@ module Karafka
 
               dispatch = lambda do
                 dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
-                mark_dispatched_to_dlq(skippable_message)
+
+                if mark_after_dispatch?
+                  mark_dispatched_to_dlq(skippable_message)
+                else
+                  self.seek_offset = skippable_message.offset + 1
+                end
               end
 
               if dispatch_in_a_transaction?
@@ -154,11 +151,12 @@ module Karafka
                 topic: topic.dead_letter_queue.topic,
                 key: original_partition,
                 payload: skippable_message.raw_payload,
-                headers: skippable_message.headers.merge(
+                headers: skippable_message.raw_headers.merge(
                   'original_topic' => topic.name,
                   'original_partition' => original_partition,
                   'original_offset' => skippable_message.offset.to_s,
                   'original_consumer_group' => topic.consumer_group.id,
+                  'original_key' => skippable_message.raw_key.to_s,
                   'original_attempts' => attempt.to_s
                 )
               }
@@ -192,6 +190,16 @@ module Karafka
               producer.transactional? && topic.dead_letter_queue.transactional?
             end
 
+            # @return [Boolean] should we mark given message as consumed after dispatch.
+            #  For default non MOM strategies if user did not explicitly tell us not to, we mark
+            #  it. Default is `nil`, which means `true` in this case. If user provided alternative
+            #  value, we go with it.
+            def mark_after_dispatch?
+              return true if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+              topic.dead_letter_queue.mark_after_dispatch
+            end
+
             # Runs the DLQ strategy and based on it it performs certain operations
             #
             # In case of `:skip` and `:dispatch` will run the exact flow provided in a block
@@ -218,7 +226,7 @@ module Karafka
               coordinator.pause_tracker.reset
 
               # Always backoff after DLQ dispatch even on skip to prevent overloads on errors
-              pause(coordinator.seek_offset, nil, false)
+              pause(seek_offset, nil, false)
             end
 
             # Marks message that went to DLQ (if applicable) based on the requested method

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -48,7 +40,7 @@ module Karafka
                   if coordinator.filtered? && !revoked?
                     handle_post_filtering
                   elsif !revoked? && !coordinator.manual_seek?
-                    seek(coordinator.seek_offset, false)
+                    seek(seek_offset, false)
                     resume
                   else
                     resume

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -55,7 +47,11 @@ module Karafka
                     skippable_message, _marked = find_skippable_message
                     dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                    coordinator.seek_offset = skippable_message.offset + 1
+                    if mark_after_dispatch?
+                      mark_dispatched_to_dlq(skippable_message)
+                    else
+                      self.seek_offset = skippable_message.offset + 1
+                    end
                   end
                 end
               end

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/dlq/ftr_lrj_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -46,11 +38,27 @@ module Karafka
                     skippable_message, _marked = find_skippable_message
                     dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                    coordinator.seek_offset = skippable_message.offset + 1
+                    if mark_after_dispatch?
+                      mark_dispatched_to_dlq(skippable_message)
+                    else
+                      self.seek_offset = skippable_message.offset + 1
+                    end
                   end
                 end
               end
             end
+
+            # @return [Boolean] should we mark given message as consumed after dispatch. For
+            #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
+            #  is `nil`, which means `false` in this case. If user provided alternative value, we
+            #  go with it.
+            #
+            # @note Please note, this is the opposite behavior than in case of AOM strategies.
+            def mark_after_dispatch?
+              return false if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+              topic.dead_letter_queue.mark_after_dispatch
+            end
           end
         end
       end

data/lib/karafka/pro/processing/strategies/dlq/ftr_mom_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/dlq/ftr_vp.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/strategies/dlq/lrj.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -39,7 +31,7 @@ module Karafka
 
                   mark_as_consumed(last_group_message) unless revoked?
                   # We should not overwrite user manual seel request with our seek
-                  seek(coordinator.seek_offset, false) unless revoked? || coordinator.manual_seek?
+                  seek(seek_offset, false) unless revoked? || coordinator.manual_seek?
 
                   resume
                 else

data/lib/karafka/pro/processing/strategies/dlq/lrj_mom.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -49,11 +41,27 @@ module Karafka
                     skippable_message, _marked = find_skippable_message
                     dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
 
-                    coordinator.seek_offset = skippable_message.offset + 1
+                    if mark_after_dispatch?
+                      mark_dispatched_to_dlq(skippable_message)
+                    else
+                      self.seek_offset = skippable_message.offset + 1
+                    end
                   end
                 end
               end
             end
+
+            # @return [Boolean] should we mark given message as consumed after dispatch. For
+            #  MOM strategies if user did not explicitly tell us to mark, we do not mark. Default
+            #  is `nil`, which means `false` in this case. If user provided alternative value, we
+            #  go with it.
+            #
+            # @note Please note, this is the opposite behavior than in case of AOM strategies.
+            def mark_after_dispatch?
+              return false if topic.dead_letter_queue.mark_after_dispatch.nil?
+
+              topic.dead_letter_queue.mark_after_dispatch
+            end
           end
         end
       end