karafka 2.4.0 → 2.4.18

data/lib/karafka/pro/cleaner/messages/message.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -29,15 +21,25 @@ module Karafka
             @raw_payload == false
           end
 
-          # Cleans the message payload and removes the deserialized data references
+          # Cleans the message payload, headers, key and removes the deserialized data references
           # This is useful when working with big messages that take a lot of space.
           #
           # After the message content is no longer needed, it can be removed so it does not consume
           # space anymore.
-          def clean!
+          #
+          # @param metadata [Boolean] should we also clean metadata alongside the payload. This can
+          #   be useful when working with iterator and other things that may require only metadata
+          #   available, while not payload. `true` by default.
+          #
+          # @note Cleaning of message means we also clean its metadata (headers and key)
+          # @note Metadata cleaning (headers and key) can be disabled by setting the `metadata`
+          #   argument to `false`.
+          def clean!(metadata: true)
             @deserialized = false
             @raw_payload = false
             @payload = nil
+
+            @metadata.clean! if metadata
           end
         end
       end

data/lib/karafka/pro/cleaner/messages/messages.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/cleaner/messages/metadata.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Cleaner
+      # Cleaner messages components related enhancements
+      module Messages
+        # Extensions to the message metadata that allow for granular memory control on a per
+        # message basis
+        module Metadata
+          # @return [Object] deserialized key. By default in the raw string format.
+          def key
+            cleaned? ? raise(Errors::MessageCleanedError) : super
+          end
+
+          # @return [Object] deserialized headers. By default its a hash with keys and payload
+          #   being strings
+          def headers
+            cleaned? ? raise(Errors::MessageCleanedError) : super
+          end
+
+          # @return [Boolean] true if the message metadata has been cleaned
+          def cleaned?
+            raw_headers == false
+          end
+
+          # Cleans the headers and key
+          def clean!
+            self.raw_headers = false
+            self.raw_key = false
+            @key = nil
+            @headers = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cleaner.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -28,6 +20,7 @@ module Karafka
         # @param _config [Karafka::Core::Configurable::Node] root node config
         def pre_setup(_config)
           ::Karafka::Messages::Message.prepend(Messages::Message)
+          ::Karafka::Messages::Metadata.prepend(Messages::Metadata)
           ::Karafka::Messages::Messages.prepend(Messages::Messages)
         end
 

data/lib/karafka/pro/connection/manager.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -216,6 +208,10 @@ module Karafka
               next unless multi_part_sgs_families.include?(sg_listener.subscription_group.name)
               # Skip already active connections
               next unless sg_listener.pending? || sg_listener.stopped?
+              # Ensure that the listener thread under which we operate is already stopped and
+              # is not dangling. While not likely to happen, this may protect against a
+              # case where a shutdown critical crash would case a restart of the same listener
+              next if sg_listener.alive?
 
               touch(sg_listener.subscription_group.id)
               sg_listener.start!

data/lib/karafka/pro/connection/multiplexing/listener.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/contracts/base.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/contracts/server_cli_options.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption/cipher.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption/contracts/config.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption/errors.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption/messages/middleware.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption/messages/parser.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption/setup/config.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/instrumentation/performance_tracker.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/iterator/expander.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/iterator.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/loader.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -74,8 +66,10 @@ module Karafka
         #   that need to have some special configuration stuff injected into config, etc
         def features
           [
+            Cleaner,
             Encryption,
-            Cleaner
+            RecurringTasks,
+            ScheduledMessages
           ]
         end
 

data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      # Namespace for adaptive iterator consumer components
+      module AdaptiveIterator
+        # Consumer enhancements needed to wrap the batch iterator for adaptive iterating
+        # It automatically marks as consumed, ensures that we do not reach `max.poll.interval.ms`
+        # and does other stuff to simplify user per-message processing
+        module Consumer
+          # @param args [Array] anything accepted by `Karafka::Messages::Messages#each`
+          def each(*args)
+            adi_config = topic.adaptive_iterator
+
+            tracker = Tracker.new(
+              adi_config.safety_margin,
+              coordinator.last_polled_at,
+              topic.subscription_group.kafka.fetch(:'max.poll.interval.ms')
+            )
+
+            messages.each(*args) do |message|
+              # Always stop if we've lost the assignment
+              return if revoked?
+              # No automatic marking risk when mom is enabled so we can fast stop
+              return if Karafka::App.done? && topic.manual_offset_management?
+
+              # Seek request on done will allow us to stop without marking the offset when user had
+              # the automatic offset marking. This should not be a big network traffic issue for
+              # the end user as we're stopping anyhow but should improve shutdown time
+              if tracker.enough? || Karafka::App.done?
+                # Enough means we no longer have time to process more data without polling as we
+                # risk reaching max poll interval. Instead we seek and we will poll again soon.
+                seek(message.offset, reset_offset: true)
+
+                return
+              end
+
+              tracker.track { yield(message) }
+
+              # Clean if this is what user configured
+              message.clean! if adi_config.clean_after_yielding?
+
+              public_send(adi_config.marking_method, message)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/adaptive_iterator/tracker.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Processing
+      module AdaptiveIterator
+        # Tracker is responsible for monitoring the processing of messages within the poll
+        # interval limitation.
+        # It ensures that the consumer does not exceed the maximum poll interval by tracking the
+        # processing cost and determining when to halt further processing (if needed).
+        class Tracker
+          include Karafka::Core::Helpers::Time
+
+          # Initializes a new Tracker instance.
+          #
+          # @param safety_margin [Float] The safety margin percentage (0-100) to leave as a buffer.
+          # @param last_polled_at [Float] The timestamp of the last polling in milliseconds.
+          # @param max_poll_interval_ms [Integer] The maximum poll interval time in milliseconds.
+          def initialize(
+            safety_margin,
+            last_polled_at,
+            max_poll_interval_ms
+          )
+            @safety_margin = safety_margin / 100.0 # Convert percentage to decimal
+            @last_polled_at = last_polled_at
+            @max_processing_cost = 0
+            @max_poll_interval_ms = max_poll_interval_ms
+          end
+
+          # Tracks the processing time of a block and updates the maximum processing cost.
+          #
+          # @yield Executes the block, measuring the time taken for processing.
+          def track
+            before = monotonic_now
+
+            yield
+
+            time_taken = monotonic_now - before
+
+            return unless time_taken > @max_processing_cost
+
+            @max_processing_cost = time_taken
+          end
+
+          # Determines if there is enough time left to process more messages without exceeding the
+          # maximum poll interval, considering both the safety margin and adaptive margin.
+          #
+          # @return [Boolean] Returns true if it is time to stop processing. False otherwise.
+          def enough?
+            elapsed_time_ms = monotonic_now - @last_polled_at
+            remaining_time_ms = @max_poll_interval_ms - elapsed_time_ms
+
+            safety_margin_ms = @max_poll_interval_ms * @safety_margin
+
+            return true if remaining_time_ms <= safety_margin_ms
+            return true if remaining_time_ms - @max_processing_cost <= safety_margin_ms
+
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/collapser.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/coordinator.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/processing/coordinators/errors_tracker.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro