karafka 2.4.0 → 2.4.18

data/lib/karafka/pro/processing/coordinators/filters_applier.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -89,6 +81,23 @@ module Karafka
             applied.map(&:cursor).compact.min_by(&:offset)
           end
 
+          # @return [Boolean] did any of the filters requested offset storage during filter
+          #   application
+          def mark_as_consumed?
+            # We can manage filtering offset only when user wanted that and there is a cursor
+            # to use
+            applied.any?(&:mark_as_consumed?) && cursor
+          end
+
+          # @return [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`
+          def marking_method
+            candidates = applied.map(&:marking_method)
+
+            return :mark_as_consumed! if candidates.include?(:mark_as_consumed!)
+
+            :mark_as_consumed
+          end
+
           private
 
           # @return [Boolean] is filtering active

data/lib/karafka/pro/processing/coordinators/virtual_offset_manager.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/executor.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/expansions_selector.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -25,6 +17,7 @@ module Karafka
           expansions = super
           expansions << Pro::Processing::Piping::Consumer
           expansions << Pro::Processing::OffsetMetadata::Consumer if topic.offset_metadata?
+          expansions << Pro::Processing::AdaptiveIterator::Consumer if topic.adaptive_iterator?
           expansions << Pro::Processing::PeriodicJob::Consumer if topic.periodic_job?
           expansions
         end

data/lib/karafka/pro/processing/filters/base.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -54,6 +46,18 @@ module Karafka
           def timeout
             0
           end
+
+          # @return [Boolean] should we use the cursor value to mark as consumed. If any of the
+          #   filters returns true, we return lowers applicable cursor value (if any)
+          def mark_as_consumed?
+            false
+          end
+
+          # @return [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`. Applicable only if
+          #   marking is requested
+          def marking_method
+            :mark_as_consumed
+          end
         end
       end
     end

data/lib/karafka/pro/processing/filters/delayer.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -33,7 +25,7 @@ module Karafka
 
             # Time on message is in seconds with ms precision, so we need to convert the ttl that
             # is in ms to this format
-            border = ::Time.now.utc - @delay / 1_000.to_f
+            border = ::Time.now.utc - @delay / 1_000.0
 
             messages.delete_if do |message|
               too_young = message.timestamp > border
@@ -52,7 +44,7 @@ module Karafka
           def timeout
             return 0 unless @cursor
 
-            timeout = (@delay / 1_000.to_f) - (::Time.now.utc - @cursor.timestamp)
+            timeout = (@delay / 1_000.0) - (::Time.now.utc - @cursor.timestamp)
 
             timeout <= 0 ? 0 : timeout * 1_000
           end

data/lib/karafka/pro/processing/filters/expirer.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/filters/inline_insights_delayer.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/filters/throttler.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/filters/virtual_limiter.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/jobs/consume_non_blocking.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -26,6 +18,8 @@ module Karafka
         # @note It needs to be working with a proper consumer that will handle the partition
         #   management. This layer of the framework knows nothing about Kafka messages consumption.
         class ConsumeNonBlocking < ::Karafka::Processing::Jobs::Consume
+          self.action = :consume
+
           # Makes this job non-blocking from the start
           # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Consume`
           def initialize(*args)

data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module Jobs
+        # Non-Blocking version of the Eofed job
+        # We use this version for LRJ topics for cases where saturated resources would not allow
+        # to run this job for extended period of time. Under such scenarios, if we would not use
+        # a non-blocking one, we would reach max.poll.interval.ms.
+        class EofedNonBlocking < ::Karafka::Processing::Jobs::Eofed
+          self.action = :eofed
+
+          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Eofed`
+          def initialize(*args)
+            super
+            @non_blocking = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs/periodic.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -18,6 +10,8 @@ module Karafka
         # Job that represents a "ticking" work. Work that we run periodically for the Periodics
         # enabled topics.
         class Periodic < ::Karafka::Processing::Jobs::Base
+          self.action = :tick
+
           # @param executor [Karafka::Pro::Processing::Executor] pro executor that is suppose to
           #   run a given job
           def initialize(executor)

data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -20,6 +12,8 @@ module Karafka
         # to run this job for extended period of time. Under such scenarios, if we would not use
         # a non-blocking one, we would reach max.poll.interval.ms.
         class PeriodicNonBlocking < Periodic
+          self.action = :tick
+
           # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Periodic`
           def initialize(*args)
             super

data/lib/karafka/pro/processing/jobs/revoked_non_blocking.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -24,6 +16,8 @@ module Karafka
         # in scenarios where there are more jobs than threads, without this being async we
         # would potentially stop polling
         class RevokedNonBlocking < ::Karafka::Processing::Jobs::Revoked
+          self.action = :revoked
+
           # Makes this job non-blocking from the start
           # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Revoked`
           def initialize(*args)

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -33,6 +25,18 @@ module Karafka
           end
         end
 
+        # @param executor [Karafka::Pro::Processing::Executor]
+        # @return [Karafka::Processing::Jobs::Eofed] eofed job for non LRJ
+        # @return [Karafka::Processing::Jobs::EofedBlocking] eofed job that is
+        #   non-blocking, so when revocation job is scheduled for LRJ it also will not block
+        def eofed(executor)
+          if executor.topic.long_running_job?
+            Jobs::EofedNonBlocking.new(executor)
+          else
+            super
+          end
+        end
+
         # @param executor [Karafka::Pro::Processing::Executor]
         # @return [Karafka::Processing::Jobs::Revoked] revocation job for non LRJ
         # @return [Karafka::Processing::Jobs::RevokedNonBlocking] revocation job that is

data/lib/karafka/pro/processing/jobs_queue.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/offset_metadata/consumer.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/offset_metadata/fetcher.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/offset_metadata/listener.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/partitioner.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -25,10 +17,13 @@ module Karafka
         def call(topic, messages, coordinator)
           ktopic = @subscription_group.topics.find(topic)
 
+          vps = ktopic.virtual_partitions
+
           # We only partition work if we have:
           # - a virtual partitioner
           # - more than one thread to process the data
           # - collective is not collapsed via coordinator
+          # - none of the partitioner executions raised an error
           #
           # With one thread it is not worth partitioning the work as the work itself will be
           # assigned to one thread (pointless work)
@@ -38,21 +33,37 @@ module Karafka
           #
           # This is great because it allows us to run things without the parallelization that adds
           # a bit of uncertainty and allows us to use DLQ and safely skip messages if needed.
-          if ktopic.virtual_partitions? &&
-             ktopic.virtual_partitions.max_partitions > 1 &&
-             !coordinator.collapsed?
-            # We need to reduce it to the max concurrency, so the group_id is not a direct effect
-            # of the end user action. Otherwise the persistence layer for consumers would cache
-            # it forever and it would cause memory leaks
-            #
-            # This also needs to be consistent because the aggregation here needs to warrant, that
-            # the same partitioned message will always be assigned to the same virtual partition.
-            # Otherwise in case of a window aggregation with VP spanning across several polls, the
-            # data could not be complete.
-            groupings = messages.group_by do |msg|
-              key = ktopic.virtual_partitions.partitioner.call(msg).to_s.sum
+          if vps.active? && vps.max_partitions > 1 && !coordinator.collapsed?
+            # If we cannot virtualize even one message from a given batch due to user errors, we
+            # reduce the whole set into one partition and emit error. This should still allow for
+            # user flow but should mitigate damages by not virtualizing
+            begin
+              groupings = messages.group_by do |msg|
+                # We need to reduce it to the max concurrency, so the group_id is not a direct
+                # effect of the end user action. Otherwise the persistence layer for consumers
+                # would cache it forever and it would cause memory leaks
+                #
+                # This also needs to be consistent because the aggregation here needs to warrant,
+                # that the same partitioned message will always be assigned to the same virtual
+                # partition. Otherwise in case of a window aggregation with VP spanning across
+                # several polls, the data could not be complete.
+                vps.reducer.call(
+                  vps.partitioner.call(msg)
+                )
+              end
+            rescue StandardError => e
+              # This should not happen. If you are seeing this it means your partitioner code
+              # failed and raised an error. We highly recommend mitigating partitioner level errors
+              # on the user side because this type of collapse should be considered a last resort
+              Karafka.monitor.instrument(
+                'error.occurred',
+                caller: self,
+                error: e,
+                messages: messages,
+                type: 'virtual_partitions.partitioner.error'
+              )
 
-              key % ktopic.virtual_partitions.max_partitions
+              groupings = { 0 => messages }
             end
 
             groupings.each do |key, messages_group|

data/lib/karafka/pro/processing/periodic_job/consumer.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/piping/consumer.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/schedulers/base.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro