karafka 2.0.27 → 2.0.28

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.0.27)
+    karafka (2.0.28)
       karafka-core (>= 2.0.9, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.9, < 3.0.0)
@@ -10,10 +10,10 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (7.0.4)
-      activesupport (= 7.0.4)
+    activejob (7.0.4.1)
+      activesupport (= 7.0.4.1)
       globalid (>= 0.3.6)
-    activesupport (7.0.4)
+    activesupport (7.0.4.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -25,7 +25,7 @@ GEM
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
     ffi (1.15.5)
-    globalid (1.0.0)
+    globalid (1.0.1)
       activesupport (>= 5.0)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
@@ -48,7 +48,7 @@ GEM
     rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.2)
+    rspec-mocks (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
     rspec-support (3.12.0)

data/config/locales/pro_errors.yml

@@ -5,7 +5,7 @@ en:
       virtual_partitions.max_partitions_format: needs to be equal or more than 1
       manual_offset_management_not_with_virtual_partitions: cannot be used together with Virtual Partitions
       long_running_job.active_format: needs to be either true or false
-      dead_letter_queue_not_with_virtual_partitions: cannot be used together with Virtual Partitions
+      dead_letter_queue_with_virtual_partitions: when using Dead Letter Queue with Virtual Partitions, at least one retry is required.
 
     config:
       encryption.active_format: 'needs to be either true or false'

data/lib/karafka/connection/listener.rb

@@ -267,7 +267,7 @@ module Karafka
           # Start work coordination for this topic partition
           coordinator.start(messages)
 
-          @partitioner.call(topic, messages) do |group_id, partition_messages|
+          @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
             # Count the job we're going to create here
             coordinator.increment
             executor = @executors.find_or_create(topic, partition, group_id)

data/lib/karafka/pro/active_job/consumer.rb

@@ -36,8 +36,9 @@ module Karafka
             )
 
             # We cannot mark jobs as done after each if there are virtual partitions. Otherwise
-            # this could create random markings
-            next if topic.virtual_partitions?
+            # this could create random markings.
+            # The exception here is the collapsed state where we can move one after another
+            next if topic.virtual_partitions? && !collapsed?
 
             mark_as_consumed(message)
           end

data/lib/karafka/pro/processing/collapser.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Manages the collapse of virtual partitions
+      # Since any non-virtual partition is actually a virtual partition of size one, we can use
+      # it in a generic manner without having to distinguish between those cases.
+      #
+      # We need to have notion of the offset until we want to collapse because upon pause and retry
+      # rdkafka may purge the buffer. This means, that we may end up with smaller or bigger
+      # (different) dataset and without tracking the end of collapse, there would be a chance for
+      # things to flicker. Tracking allows us to ensure, that collapse is happening until all the
+      # messages from the corrupted batch are processed.
+      class Collapser
+        # When initialized, nothing is collapsed
+        def initialize
+          @collapsed = false
+          @until_offset = -1
+          @mutex = Mutex.new
+        end
+
+        # @return [Boolean] Should we collapse into a single consumer
+        def collapsed?
+          @collapsed
+        end
+
+        # Collapse until given offset. Until given offset is encountered or offset bigger than that
+        # we keep collapsing.
+        # @param offset [Integer] offset until which we keep the collapse
+        def collapse_until!(offset)
+          @mutex.synchronize do
+            # We check it here in case after a pause and re-fetch we would get less messages and
+            # one of them would cause an error. We do not want to overwrite the offset here unless
+            # it is bigger.
+            @until_offset = offset if offset > @until_offset
+          end
+        end
+
+        # Sets the collapse state based on the first collective offset that we are going to process
+        # and makes the decision whether or not we need to still keep the collapse.
+        # @param first_offset [Integer] first offset from a collective batch
+        def refresh!(first_offset)
+          @mutex.synchronize do
+            @collapsed = first_offset < @until_offset
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/coordinator.rb

@@ -23,6 +23,7 @@ module Karafka
 
           @executed = []
           @flow_lock = Mutex.new
+          @collapser = Collapser.new
         end
 
         # Starts the coordination process
@@ -31,12 +32,28 @@ module Karafka
         def start(messages)
           super
 
+          @collapser.refresh!(messages.first.offset)
+
           @mutex.synchronize do
             @executed.clear
             @last_message = messages.last
           end
         end
 
+        # Sets the consumer failure status and additionally starts the collapse until
+        #
+        # @param consumer [Karafka::BaseConsumer] consumer that failed
+        # @param error [StandardError] error from the failure
+        def failure!(consumer, error)
+          super
+          @collapser.collapse_until!(@last_message.offset + 1)
+        end
+
+        # @return [Boolean] are we in a collapsed state at the moment
+        def collapsed?
+          @collapser.collapsed?
+        end
+
         # @return [Boolean] is the coordinated work finished or not
         def finished?
           @running_jobs.zero?

data/lib/karafka/pro/processing/partitioner.rb

@@ -18,15 +18,29 @@ module Karafka
       class Partitioner < ::Karafka::Processing::Partitioner
         # @param topic [String] topic name
         # @param messages [Array<Karafka::Messages::Message>] karafka messages
+        # @param coordinator [Karafka::Pro::Processing::Coordinator] processing coordinator that
+        #   will be used with those messages
         # @yieldparam [Integer] group id
         # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
-        def call(topic, messages)
+        def call(topic, messages, coordinator)
           ktopic = @subscription_group.topics.find(topic)
 
-          # We only partition work if we have a virtual partitioner and more than one thread to
-          # process the data. With one thread it is not worth partitioning the work as the work
-          # itself will be assigned to one thread (pointless work)
-          if ktopic.virtual_partitions? && ktopic.virtual_partitions.max_partitions > 1
+          # We only partition work if we have:
+          # - a virtual partitioner
+          # - more than one thread to process the data
+          # - collective is not collapsed via coordinator
+          #
+          # With one thread it is not worth partitioning the work as the work itself will be
+          # assigned to one thread (pointless work)
+          #
+          # We collapse the partitioning on errors because we "regain" full ordering on a batch
+          # that potentially contains the data that caused the error.
+          #
+          # This is great because it allows us to run things without the parallelization that adds
+          # a bit of uncertainty and allows us to use DLQ and safely skip messages if needed.
+          if ktopic.virtual_partitions? &&
+             ktopic.virtual_partitions.max_partitions > 1 &&
+             !coordinator.collapsed?
             # We need to reduce it to the max concurrency, so the group_id is not a direct effect
             # of the end user action. Otherwise the persistence layer for consumers would cache
             # it forever and it would cause memory leaks

data/lib/karafka/pro/processing/strategies/aj_dlq_lrj_mom_vp.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # DLQ enabled
+        # Long-Running Job enabled
+        # Manual offset management enabled
+        # Virtual Partitions enabled
+        #
+        # This case is a bit of special. Please see the `AjDlqMom` for explanation on how the
+        # offset management works in this case.
+        module AjDlqLrjMomVp
+          include AjDlqMomVp
+          include AjLrjMom
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            long_running_job
+            manual_offset_management
+            dead_letter_queue
+            virtual_partitions
+          ].freeze
+
+          # This strategy is pretty much as non VP one because of the collapse
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                return if revoked?
+                return if Karafka::App.stopping?
+
+                # Since we have VP here we do not commit intermediate offsets and need to commit
+                # them here. We do commit in collapsed mode but this is generalized.
+                mark_as_consumed(last_group_message)
+
+                seek(coordinator.seek_offset) unless revoked?
+
+                resume
+              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                retry_after_pause
+              else
+                coordinator.pause_tracker.reset
+                skippable_message = find_skippable_message
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                mark_as_consumed(skippable_message)
+                pause(coordinator.seek_offset, nil, false)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_dlq_mom_vp.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # ActiveJob enabled
+        # Manual offset management enabled
+        # Virtual Partitions enabled
+        module AjDlqMomVp
+          include Dlq
+          include Vp
+          include Default
+
+          # Features for this strategy
+          FEATURES = %i[
+            active_job
+            dead_letter_queue
+            manual_offset_management
+            virtual_partitions
+          ].freeze
+
+          # Flow including moving to DLQ in the collapsed mode
+          def handle_after_consume
+            coordinator.on_finished do |last_group_message|
+              if coordinator.success?
+                coordinator.pause_tracker.reset
+
+                # When this is an ActiveJob running via Pro with virtual partitions, we cannot mark
+                # intermediate jobs as processed not to mess up with the ordering.
+                # Only when all the jobs are processed and we did not loose the partition
+                # assignment and we are not stopping (Pro ActiveJob has an early break) we can
+                # commit offsets on this as only then we can be sure, that all the jobs were
+                # processed.
+                # For a non virtual partitions case, the flow is regular and state is marked after
+                # each successfully processed job
+                return if revoked?
+                return if Karafka::App.stopping?
+
+                mark_as_consumed(last_group_message)
+              elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
+                retry_after_pause
+              else
+                # Here we are in a collapsed state, hence we can apply the same logic as AjDlqMom
+                coordinator.pause_tracker.reset
+                skippable_message = find_skippable_message
+                dispatch_to_dlq(skippable_message) if dispatch_to_dlq?
+                mark_as_consumed(skippable_message)
+                pause(coordinator.seek_offset, nil, false)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/aj_lrj_mom_vp.rb

@@ -21,6 +21,7 @@ module Karafka
         # Virtual Partitions enabled
         module AjLrjMomVp
           include Default
+          include Vp
 
           # Features for this strategy
           FEATURES = %i[

data/lib/karafka/pro/processing/strategies/aj_mom_vp.rb

@@ -19,6 +19,7 @@ module Karafka
         # Manual offset management enabled
         # Virtual Partitions enabled
         module AjMomVp
+          include Vp
           include Default
 
           # Features for this strategy
@@ -42,6 +43,9 @@ module Karafka
                 # processed.
                 # For a non virtual partitions case, the flow is regular and state is marked after
                 # each successfully processed job
+                #
+                # We can mark and we do mark intermediate jobs in the collapsed mode when running
+                # VPs
                 return if revoked?
                 return if Karafka::App.stopping?
 

data/lib/karafka/pro/processing/strategies/default.rb

@@ -53,10 +53,10 @@ module Karafka
             end
 
             # Mark job as successful
-            coordinator.consumption(self).success!
+            coordinator.success!(self)
           rescue StandardError => e
             # If failed, mark as failed
-            coordinator.consumption(self).failure!(e)
+            coordinator.failure!(self, e)
 
             # Re-raise so reported in the consumer
             raise e

data/lib/karafka/pro/processing/strategies/dlq.rb

@@ -65,13 +65,8 @@ module Karafka
           #   dispatch to DLQ
           def dispatch_to_dlq(skippable_message)
             producer.produce_async(
-              topic: topic.dead_letter_queue.topic,
-              payload: skippable_message.raw_payload,
-              key: skippable_message.partition.to_s,
-              headers: skippable_message.headers.merge(
-                'original_topic' => topic.name,
-                'original_partition' => skippable_message.partition.to_s,
-                'original_offset' => skippable_message.offset.to_s
+              build_dlq_message(
+                skippable_message
               )
             )
 
@@ -83,6 +78,35 @@ module Karafka
             )
           end
 
+          # @param skippable_message [Array<Karafka::Messages::Message>]
+          # @return [Hash] dispatch DLQ message
+          def build_dlq_message(skippable_message)
+            original_partition = skippable_message.partition.to_s
+
+            dlq_message = {
+              topic: topic.dead_letter_queue.topic,
+              key: original_partition,
+              payload: skippable_message.raw_payload,
+              headers: skippable_message.headers.merge(
+                'original_topic' => topic.name,
+                'original_partition' => original_partition,
+                'original_offset' => skippable_message.offset.to_s,
+                'original_consumer_group' => topic.consumer_group.id
+              )
+            }
+
+            # Optional method user can define in consumer to enhance the dlq message hash with
+            # some extra details if needed or to replace payload, etc
+            if respond_to?(:enhance_dlq_message, true)
+              enhance_dlq_message(
+                dlq_message,
+                skippable_message
+              )
+            end
+
+            dlq_message
+          end
+
           # @return [Boolean] should we dispatch the message to DLQ or not. When the dispatch topic
           #   is set to false, we will skip the dispatch, effectively ignoring the broken message
           #   without taking any action.

data/lib/karafka/pro/processing/strategies/dlq_lrj_vp.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Dead-Letter Queue enabled
+        # Long-Running Job enabled
+        # Virtual Partitions enabled
+        module DlqLrjVp
+          # Same flow as the Dlq Lrj because VP collapses on errors, so DlqLrj can kick in
+          include Vp
+          include DlqLrj
+
+          # Features for this strategy
+          FEATURES = %i[
+            dead_letter_queue
+            long_running_job
+            virtual_partitions
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/dlq_vp.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Strategies
+        # Dead Letter Queue enabled
+        # Virtual Partitions enabled
+        #
+        # In general because we collapse processing in virtual partitions to one on errors, there
+        # is no special action that needs to be taken because we warranty that even with VPs
+        # on errors a retry collapses into a single state.
+        module DlqVp
+          # Features for this strategy
+          FEATURES = %i[
+            dead_letter_queue
+            virtual_partitions
+          ].freeze
+
+          include Dlq
+          include Vp
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/strategies/lrj_vp.rb

@@ -19,6 +19,7 @@ module Karafka
         # Virtual Partitions enabled
         module LrjVp
           # Same flow as the standard Lrj
+          include Vp
           include Lrj
 
           # Features for this strategy

data/lib/karafka/pro/processing/strategies/vp.rb

@@ -25,6 +25,11 @@ module Karafka
           FEATURES = %i[
             virtual_partitions
           ].freeze
+
+          # @return [Boolean] is the virtual processing collapsed in the context of given consumer.
+          def collapsed?
+            coordinator.collapsed?
+          end
         end
       end
     end

data/lib/karafka/pro/routing/features/dead_letter_queue/contract.rb

@@ -26,10 +26,10 @@ module Karafka
               ).fetch('en').fetch('validations').fetch('topic')
             end
 
-            # Make sure that we don't use DLQ with VP
-            # Using those two would cause many issues because the offset within VP is not
-            # manageable so in scenarios where we would fail on the last message, we would move by
-            # one and try again and fail, and move by one and try again and fail and so on...
+            # Make sure that when we use virtual partitions with DLQ, at least one retry is set
+            # We cannot use VP with DLQ without retries as we in order to provide ordering
+            # warranties on errors with VP, we need to collapse the VPs concurrency and retry
+            # without any indeterministic work
             virtual do |data, errors|
               next unless errors.empty?
 
@@ -38,8 +38,9 @@ module Karafka
 
               next unless dead_letter_queue[:active]
               next unless virtual_partitions[:active]
+              next if dead_letter_queue[:max_retries].positive?
 
-              [[%i[dead_letter_queue], :not_with_virtual_partitions]]
+              [[%i[dead_letter_queue], :with_virtual_partitions]]
             end
           end
         end

data/lib/karafka/processing/coordinator.rb

@@ -71,18 +71,28 @@ module Karafka
         end
       end
 
-      # @param consumer [Object] karafka consumer (normal or pro)
-      # @return [Karafka::Processing::Result] result object which we can use to indicate
-      #   consumption processing state.
-      def consumption(consumer)
+      # Is all the consumption done and finished successfully for this coordinator
+      def success?
         @mutex.synchronize do
-          @consumptions[consumer] ||= Processing::Result.new
+          @running_jobs.zero? && @consumptions.values.all?(&:success?)
         end
       end
 
-      # Is all the consumption done and finished successfully for this coordinator
-      def success?
-        @mutex.synchronize { @running_jobs.zero? && @consumptions.values.all?(&:success?) }
+      # Mark given consumption on consumer as successful
+      # @param consumer [Karafka::BaseConsumer] consumer that finished successfully
+      def success!(consumer)
+        @mutex.synchronize do
+          consumption(consumer).success!
+        end
+      end
+
+      # Mark given consumption on consumer as failed
+      # @param consumer [Karafka::BaseConsumer] consumer that failed
+      # @param error [StandardError] error that occurred
+      def failure!(consumer, error)
+        @mutex.synchronize do
+          consumption(consumer).failure!(error)
+        end
       end
 
       # Marks given coordinator for processing group as revoked
@@ -113,6 +123,15 @@ module Karafka
       def manual_pause?
         @pause_tracker.paused? && @manual_pause
       end
+
+      private
+
+      # @param consumer [Object] karafka consumer (normal or pro)
+      # @return [Karafka::Processing::Result] result object which we can use to indicate
+      #   consumption processing state.
+      def consumption(consumer)
+        @consumptions[consumer] ||= Processing::Result.new
+      end
     end
   end
 end

data/lib/karafka/processing/partitioner.rb

@@ -12,9 +12,11 @@ module Karafka
 
       # @param _topic [String] topic name
       # @param messages [Array<Karafka::Messages::Message>] karafka messages
+      # @param _coordinator [Karafka::Processing::Coordinator] processing coordinator that will
+      #   be used with those messages
       # @yieldparam [Integer] group id
       # @yieldparam [Array<Karafka::Messages::Message>] karafka messages
-      def call(_topic, messages)
+      def call(_topic, messages, _coordinator)
         yield(0, messages)
       end
     end

data/lib/karafka/processing/strategies/default.rb

@@ -31,10 +31,9 @@ module Karafka
           end
 
           # Mark job as successful
-          coordinator.consumption(self).success!
+          coordinator.success!(self)
         rescue StandardError => e
-          # If failed, mark as failed
-          coordinator.consumption(self).failure!(e)
+          coordinator.failure!(self, e)
 
           # Re-raise so reported in the consumer
           raise e

data/lib/karafka/setup/config.rb

@@ -16,7 +16,6 @@ module Karafka
 
       # Defaults for kafka settings, that will be overwritten only if not present already
       KAFKA_DEFAULTS = {
-        'client.id': 'karafka',
         # We emit the statistics by default, so all the instrumentation and web-ui work out of
         # the box, without requiring users to take any extra actions aside from enabling.
         'statistics.interval.ms': 5_000
@@ -193,6 +192,9 @@ module Karafka
             config.kafka[key] = value
           end
 
+          # Use Karafka client_id as kafka client id if not set
+          config.kafka[:'client.id'] ||= config.client_id
+
           return if Karafka::App.env.production?
 
           KAFKA_DEV_DEFAULTS.each do |key, value|