karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/consumer_groups/filters/expirer.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Filters
+          # Expirer for removing too old messages.
+          # It never moves offsets in any way and does not impact the processing flow. It always
+          # runs `:skip` action.
+          class Expirer < Base
+            # @param ttl [Integer] maximum age of a message (in ms)
+            def initialize(ttl)
+              super()
+
+              @ttl = ttl
+            end
+
+            # Removes too old messages
+            #
+            # @param messages [Array<Karafka::Messages::Message>]
+            def apply!(messages)
+              @applied = false
+
+              # Time on message is in seconds with ms precision, so we need to convert the ttl that
+              # is in ms to this format
+              border = Time.now.utc - (@ttl / 1_000.to_f)
+
+              messages.delete_if do |message|
+                too_old = message.timestamp < border
+
+                @applied = true if too_old
+
+                too_old
+              end
+            end
+
+            # @return [nil] this filter does not deal with timeouts
+            def timeout
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/filters/inline_insights_delayer.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Filters
+          # Delayer that checks if we have appropriate insights available. If not, pauses for
+          # 5 seconds so the insights can be loaded from the broker.
+          #
+          # In case it would take more than five seconds to load insights, it will just pause again
+          #
+          # This filter ensures, that we always have inline insights that a consumer can use
+          #
+          # It is relevant in most cases only during the process start, when first poll may not
+          # yield statistics yet but will give some data.
+          class InlineInsightsDelayer < Base
+            # Minimum how long should we pause when there are no metrics
+            PAUSE_TIMEOUT = 5_000
+
+            private_constant :PAUSE_TIMEOUT
+
+            # @param topic [Karafka::Routing::Topic]
+            # @param partition [Integer] partition
+            def initialize(topic, partition)
+              super()
+              @topic = topic
+              @partition = partition
+            end
+
+            # Pauses if inline insights would not be available. Does nothing otherwise
+            #
+            # @param messages [Array<Karafka::Messages::Message>]
+            def apply!(messages)
+              @applied = false
+              @cursor = messages.first
+
+              # Nothing to do if there were no messages This can happen when we chain filters
+              return unless @cursor
+
+              insights = Karafka::Processing::ConsumerGroups::InlineInsights::Tracker.find(
+                @topic,
+                @partition
+              )
+
+              # If insights are available, also nothing to do here and we can just process
+              return unless insights.empty?
+
+              messages.clear
+
+              @applied = true
+            end
+
+            # @return [Integer, nil] ms timeout in case of pause or nil if not delaying
+            def timeout
+              (@cursor && applied?) ? PAUSE_TIMEOUT : nil
+            end
+
+            # Pause when we had to back-off or skip if delay is not needed
+            def action
+              applied? ? :pause : :skip
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/filters/throttler.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Namespace containing Pro out of the box filters used by various strategies
+        module Filters
+          # Throttler used to limit number of messages we can process in a given time interval
+          # The tricky thing is, that even if we throttle on 100 messages, if we've reached 100, we
+          # still need to indicate, that we throttle despite not receiving 101. Otherwise we will
+          # not pause the partition and will fetch more data that we should not process.
+          #
+          # This is a special type of a filter that always throttles and makes us wait / seek if
+          # anything is applied out.
+          class Throttler < Base
+            # @param limit [Integer] how many messages we can process in a given time
+            # @param interval [Integer] interval in milliseconds for which we want to process
+            def initialize(limit, interval)
+              super()
+
+              @limit = limit
+              @interval = interval
+              @requests = Hash.new { |h, k| h[k] = 0 }
+            end
+
+            # Limits number of messages to a range that we can process (if needed) and keeps track
+            # of how many messages we've processed in a given time
+            # @param messages [Array<Karafka::Messages::Message>] limits the number of messages to
+            #   number we can accept in the context of throttling constraints
+            def apply!(messages)
+              @applied = false
+              @cursor = nil
+              @time = monotonic_now
+              @requests.delete_if { |timestamp, _| timestamp < (@time - @interval) }
+              values = @requests.values.sum
+              accepted = 0
+
+              messages.delete_if do |message|
+                # +1 because of current
+                @applied = (values + accepted + 1) > @limit
+
+                @cursor = message if @applied && @cursor.nil?
+
+                next true if @applied
+
+                accepted += 1
+
+                false
+              end
+
+              @requests[@time] += accepted
+            end
+
+            # @return [Symbol] action to take upon throttler reaching certain state
+            def action
+              if applied?
+                timeout.zero? ? :seek : :pause
+              else
+                :skip
+              end
+            end
+
+            # @return [Integer] minimum number of milliseconds to wait before getting more messages
+            #   so we are no longer throttled and so we can process at least one message
+            def timeout
+              timeout = @interval - (monotonic_now - @time)
+              [timeout, 0].max
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/filters/virtual_limiter.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Filters
+          # Removes messages that are already marked as consumed in the virtual offset manager
+          # This should operate only when using virtual partitions.
+          #
+          # This cleaner prevents us from duplicated processing of messages that were virtually
+          # marked as consumed even if we could not mark them as consumed in Kafka. This allows us
+          # to limit reprocessing when errors occur drastically when operating with virtual
+          # partitions
+          #
+          # @note It should be registered only when VPs are used
+          class VirtualLimiter < Base
+            # @param manager [Processing::ConsumerGroups::Coordinators::VirtualOffsetManager]
+            # @param collapser [Processing::ConsumerGroups::Collapser]
+            def initialize(manager, collapser)
+              @manager = manager
+              @collapser = collapser
+
+              super()
+            end
+
+            # Remove messages that we already marked as virtually consumed. Does nothing if not in
+            # the collapsed mode.
+            #
+            # @param messages [Array<Karafka::Messages::Message>]
+            def apply!(messages)
+              return unless @collapser.collapsed?
+
+              marked = @manager.marked
+
+              messages.delete_if { |message| marked.include?(message.offset) }
+            end
+
+            # @return [nil] This filter does not deal with pausing, so timeout is always nil
+            def timeout
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{jobs → consumer_groups/jobs}/consume_non_blocking.rb

@@ -32,24 +32,28 @@ module Karafka
   module Pro
     # Pro components related to processing part of Karafka
     module Processing
-      # Pro jobs
-      module Jobs
-        # The main job type in a non-blocking variant.
-        # This variant works "like" the regular consumption but does not block the queue.
-        #
-        # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
-        # if would block.
-        #
-        # @note It needs to be working with a proper consumer that will handle the partition
-        #   management. This layer of the framework knows nothing about Kafka messages consumption.
-        class ConsumeNonBlocking < Karafka::Processing::Jobs::Consume
-          self.action = :consume
+      # Pro consumer-group-specific processing components
+      module ConsumerGroups
+        # Pro jobs
+        module Jobs
+          # The main job type in a non-blocking variant.
+          # This variant works "like" the regular consumption but does not block the queue.
+          #
+          # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
+          # if would block.
+          #
+          # @note It needs to be working with a proper consumer that will handle the partition
+          #   management. This layer of the framework knows nothing about Kafka messages consumption.
+          class ConsumeNonBlocking < Karafka::Processing::ConsumerGroups::Jobs::Consume
+            self.action = :consume
 
-          # Makes this job non-blocking from the start
-          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Consume`
-          def initialize(*args)
-            super
-            @non_blocking = true
+            # Makes this job non-blocking from the start
+            # @param args [Array] any arguments accepted by
+            #   `::Karafka::Processing::ConsumerGroups::Jobs::Consume`
+            def initialize(*args)
+              super
+              @non_blocking = true
+            end
           end
         end
       end

data/lib/karafka/pro/processing/{virtual_partitions/distributors/consistent.rb → consumer_groups/jobs/eofed_non_blocking.rb}

@@ -31,20 +31,22 @@
 module Karafka
   module Pro
     module Processing
-      module VirtualPartitions
-        module Distributors
-          # Consistent distributor that ensures messages with the same partition key
-          # are always processed in the same virtual partition
-          class Consistent < Base
-            # Distributes messages ensuring consistent routing where messages with the same
-            # partition key always go to the same virtual partition
-            # @param messages [Array<Karafka::Messages::Message>]
-            # @return [Hash{Integer => Array<Karafka::Messages::Message>}] hash with group ids as
-            #   keys and message groups as values
-            def call(messages)
-              messages
-                .group_by { |msg| config.reducer.call(config.partitioner.call(msg)) }
-                .to_h
+      # Pro consumer-group-specific processing components
+      module ConsumerGroups
+        # Pro jobs
+        module Jobs
+          # Non-Blocking version of the Eofed job
+          # We use this version for LRJ topics for cases where saturated resources would not allow
+          # to run this job for extended period of time. Under such scenarios, if we would not use
+          # a non-blocking one, we would reach max.poll.interval.ms.
+          class EofedNonBlocking < Karafka::Processing::ConsumerGroups::Jobs::Eofed
+            self.action = :eofed
+
+            # @param args [Array] any arguments accepted by
+            #   `::Karafka::Processing::ConsumerGroups::Jobs::Eofed`
+            def initialize(*args)
+              super
+              @non_blocking = true
             end
           end
         end

data/lib/karafka/pro/processing/consumer_groups/jobs/periodic.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro consumer-group-specific processing components
+      module ConsumerGroups
+        # Pro jobs
+        module Jobs
+          # Job that represents a "ticking" work. Work that we run periodically for the Periodics
+          # enabled topics.
+          class Periodic < Karafka::Processing::Jobs::Base
+            self.action = :tick
+
+            # @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor] pro executor that is suppose to
+            #   run a given job
+            def initialize(executor)
+              @executor = executor
+              super()
+            end
+
+            # Code executed before we schedule this job
+            def before_schedule
+              executor.before_schedule_periodic
+            end
+
+            # Runs the executor periodic action
+            def call
+              executor.periodic
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{jobs → consumer_groups/jobs}/periodic_non_blocking.rb

@@ -31,18 +31,23 @@
 module Karafka
   module Pro
     module Processing
-      module Jobs
-        # Non-Blocking version of the Periodic job
-        # We use this version for LRJ topics for cases where saturated resources would not allow
-        # to run this job for extended period of time. Under such scenarios, if we would not use
-        # a non-blocking one, we would reach max.poll.interval.ms.
-        class PeriodicNonBlocking < Periodic
-          self.action = :tick
+      # Pro consumer-group-specific processing components
+      module ConsumerGroups
+        # Pro jobs
+        module Jobs
+          # Non-Blocking version of the Periodic job
+          # We use this version for LRJ topics for cases where saturated resources would not allow
+          # to run this job for extended period of time. Under such scenarios, if we would not use
+          # a non-blocking one, we would reach max.poll.interval.ms.
+          class PeriodicNonBlocking < Periodic
+            self.action = :tick
 
-          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Periodic`
-          def initialize(*args)
-            super
-            @non_blocking = true
+            # @param args [Array] any arguments accepted by
+            #   `::Karafka::Pro::Processing::ConsumerGroups::Jobs::Periodic`
+            def initialize(*args)
+              super
+              @non_blocking = true
+            end
           end
         end
       end

data/lib/karafka/pro/processing/{jobs → consumer_groups/jobs}/revoked_non_blocking.rb

@@ -32,22 +32,26 @@ module Karafka
   module Pro
     # Pro components related to processing part of Karafka
     module Processing
-      # Pro jobs
-      module Jobs
-        # The revoked job type in a non-blocking variant.
-        # This variant works "like" the regular revoked but does not block the queue.
-        #
-        # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
-        # in scenarios where there are more jobs than threads, without this being async we
-        # would potentially stop polling
-        class RevokedNonBlocking < Karafka::Processing::Jobs::Revoked
-          self.action = :revoked
+      # Pro consumer-group-specific processing components
+      module ConsumerGroups
+        # Pro jobs
+        module Jobs
+          # The revoked job type in a non-blocking variant.
+          # This variant works "like" the regular revoked but does not block the queue.
+          #
+          # It can be useful when having long lasting jobs that would exceed `max.poll.interval`
+          # in scenarios where there are more jobs than threads, without this being async we
+          # would potentially stop polling
+          class RevokedNonBlocking < Karafka::Processing::ConsumerGroups::Jobs::Revoked
+            self.action = :revoked
 
-          # Makes this job non-blocking from the start
-          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Revoked`
-          def initialize(*args)
-            super
-            @non_blocking = true
+            # Makes this job non-blocking from the start
+            # @param args [Array] any arguments accepted by
+            #   `::Karafka::Processing::ConsumerGroups::Jobs::Revoked`
+            def initialize(*args)
+              super
+              @non_blocking = true
+            end
           end
         end
       end