karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/consumer_groups/coordinators/filters_applier.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Coordinators
+          # Applier for all filters we want to have. Whether related to limiting messages based
+          # on the payload or any other things.
+          #
+          # From the outside world perspective, this encapsulates all the filters.
+          # This means that this is the API we expose as a single filter, allowing us to control
+          # the filtering via many filters easily.
+          class FiltersApplier
+            # @return [Array] registered filters array. Useful if we want to inject internal context
+            #   aware filters.
+            attr_reader :filters
+
+            # @param coordinator [Pro::Coordinator] pro coordinator
+            def initialize(coordinator)
+              # Builds filters out of their factories
+              # We build it that way (providing topic and partition) because there may be a case
+              # where someone wants to have a specific logic that is per topic or partition. Like for
+              # example a case where there is a cache bypassing revocations for topic partition.
+              #
+              # We provide full Karafka routing topic here and not the name only, in case the filter
+              # would be customized based on other topic settings (like VPs, etc)
+              #
+              # This setup allows for biggest flexibility also because topic object holds the
+              # reference to the subscription group and consumer group
+              @filters = coordinator.topic.filtering.factories.map do |factory|
+                factory.call(coordinator.topic, coordinator.partition)
+              end
+            end
+
+            # @param messages [Array<Karafka::Messages::Message>] array with messages from the
+            #   partition
+            def apply!(messages)
+              return unless active?
+
+              @filters.each { |filter| filter.apply!(messages) }
+            end
+
+            # @return [Boolean] did we filter out any messages during filtering run
+            def applied?
+              return false unless active?
+
+              !applied.empty?
+            end
+
+            # @return [Symbol] consumer post-filtering action that should be taken
+            def action
+              return :skip unless applied?
+
+              # The highest priority is on a potential backoff from any of the filters because it is
+              # the less risky (delay and continue later)
+              return :pause if applied.any? { |filter| filter.action == :pause }
+
+              # If none of the filters wanted to pause, we can check for any that would want to seek
+              # and if there is any, we can go with this strategy
+              return :seek if applied.any? { |filter| filter.action == :seek }
+
+              :skip
+            end
+
+            # @return [Integer] minimum timeout we need to pause. This is the minimum for all the
+            #   filters to satisfy all of them.
+            def timeout
+              applied.filter_map(&:timeout).min || 0
+            end
+
+            # The first message we do need to get next time we poll. We use the minimum not to jump
+            # accidentally by over any.
+            # @return [Karafka::Messages::Message, nil] cursor message or nil if none
+            # @note Cursor message can also return the offset in the time format
+            def cursor
+              return nil unless active?
+
+              applied.filter_map(&:cursor).min_by(&:offset)
+            end
+
+            # @return [Boolean] did any of the filters requested offset storage during filter
+            #   application
+            def mark_as_consumed?
+              # We can manage filtering offset only when user wanted that and there is a cursor
+              # to use
+              applied.any?(&:mark_as_consumed?) && cursor
+            end
+
+            # @return [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`
+            def marking_method
+              candidates = applied.map(&:marking_method)
+
+              return :mark_as_consumed! if candidates.include?(:mark_as_consumed!)
+
+              :mark_as_consumed
+            end
+
+            # The first (lowest) message we want to mark as consumed in marking. By default it uses
+            #   same position as cursor in case user wants to mark same message as consumed as the
+            #   one on which cursor action is applied.
+            # @return [Karafka::Messages::Message, nil] cursor marking message or nil if none
+            # @note It should not return position in time format, only numerical offset
+            def marking_cursor
+              return nil unless active?
+
+              applied.filter_map(&:marking_cursor).min_by(&:offset)
+            end
+
+            private
+
+            # @return [Boolean] is filtering active
+            def active?
+              !@filters.empty?
+            end
+
+            # @return [Array<Object>] filters that applied any sort of messages limiting
+            def applied
+              @filters.select(&:applied?)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/coordinators/virtual_offset_manager.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Coordinators
+          # Manager that keeps track of our offsets with the virtualization layer that are local
+          # to given partition assignment. It allows for easier offset management for virtual
+          # virtual partition cases as it provides us ability to mark as consumed and move the
+          # real offset behind as expected.
+          #
+          # @note We still use the regular coordinator "real" offset management as we want to have
+          #   them as separated as possible because the real seek offset management is also used for
+          #   pausing, filtering and others and should not be impacted by the virtual one
+          #
+          # @note This manager is **not** thread-safe by itself. It should operate from coordinator
+          #   locked locations.
+          class VirtualOffsetManager
+            attr_reader :groups
+
+            # @param topic [String]
+            # @param partition [Integer]
+            # @param offset_metadata_strategy [Symbol] what metadata should we select. That is,
+            #   should we use the most recent or one picked from the offset that is going to be
+            #   committed
+            #
+            # @note We need topic and partition because we use a seek message (virtual) for real
+            #   offset management. We could keep real message reference but this can be memory
+            #   consuming and not worth it.
+            def initialize(topic, partition, offset_metadata_strategy)
+              @topic = topic
+              @partition = partition
+              @groups = []
+              @marked = {}
+              @offsets_metadata = {}
+              @real_offset = -1
+              @offset_metadata_strategy = offset_metadata_strategy
+              @current_offset_metadata = nil
+            end
+
+            # Clears the manager for a next collective operation
+            def clear
+              @groups.clear
+              @offsets_metadata.clear
+              @current_offset_metadata = nil
+              @marked.clear
+              @real_offset = -1
+            end
+
+            # Registers an offset group coming from one virtual consumer. In order to move the real
+            # underlying offset accordingly, we need to make sure to track the virtual consumers
+            # offsets groups independently and only materialize the end result.
+            #
+            # @param offsets_group [Array<Integer>] offsets from one virtual consumer
+            def register(offsets_group)
+              @groups << offsets_group
+
+              offsets_group.each { |offset| @marked[offset] = false }
+            end
+
+            # Marks given message as marked (virtually consumed).
+            # We mark given message offset and other earlier offsets from the same group as done
+            # and we can refresh our real offset representation based on that as it might have
+            # changed to a newer real offset.
+            # @param message [Karafka::Messages::Message] message coming from VP we want to mark
+            # @param offset_metadata [String, nil] offset metadata. `nil` if none
+            def mark(message, offset_metadata)
+              offset = message.offset
+
+              # Store metadata when we materialize the most stable offset
+              @offsets_metadata[offset] = offset_metadata
+              @current_offset_metadata = offset_metadata
+
+              group = nil
+              position = nil
+
+              @groups.each do |reg_group|
+                pos = reg_group.index(offset)
+
+                if pos
+                  group = reg_group
+                  position = pos
+                  break
+                end
+              end
+
+              # This case can happen when someone uses MoM and wants to mark message from a previous
+              # batch as consumed. We can add it, since the real offset refresh will point to it
+              unless group
+                group = [offset]
+                position = 0
+                @groups << group
+              end
+
+              # Mark all previous messages from the same group also as virtually consumed
+              group[0..position].each do |markable_offset|
+                # Set previous messages metadata offset as the offset of higher one for overwrites
+                # unless a different metadata were set explicitely
+                @offsets_metadata[markable_offset] ||= offset_metadata
+                @marked[markable_offset] = true
+              end
+
+              # Recompute the real offset representation
+              materialize_real_offset
+            end
+
+            # Mark all from all groups including the `message`.
+            # Useful when operating in a collapsed state for marking
+            # @param message [Karafka::Messages::Message]
+            # @param offset_metadata [String, nil]
+            def mark_until(message, offset_metadata)
+              mark(message, offset_metadata)
+
+              @groups.each do |group|
+                group.each do |offset|
+                  next if offset > message.offset
+
+                  @offsets_metadata[offset] = offset_metadata
+                  @marked[offset] = true
+                end
+              end
+
+              materialize_real_offset
+            end
+
+            # @return [Array<Integer>] Offsets of messages already marked as consumed virtually
+            def marked
+              @marked.select { |_, status| status }.map { |offset, _| offset }.sort
+            end
+
+            # Is there a real offset we can mark as consumed
+            # @return [Boolean]
+            def markable?
+              !@real_offset.negative?
+            end
+
+            # @return [Array<Messages::Seek, String>] markable message for real offset marking and
+            #   its associated metadata
+            def markable
+              raise Errors::InvalidRealOffsetUsageError unless markable?
+
+              offset_metadata = case @offset_metadata_strategy
+              when :exact
+                @offsets_metadata.fetch(@real_offset)
+              when :current
+                @current_offset_metadata
+              else
+                raise Errors::UnsupportedCaseError, @offset_metadata_strategy
+              end
+
+              [
+                Messages::Seek.new(
+                  @topic,
+                  @partition,
+                  @real_offset
+                ),
+                offset_metadata
+              ]
+            end
+
+            private
+
+            # Recomputes the biggest possible real offset we can have.
+            # It picks the biggest offset that has uninterrupted stream of virtually marked as
+            # consumed because this will be the collective offset.
+            def materialize_real_offset
+              @marked.keys.sort.each do |offset|
+                break unless @marked[offset]
+
+                @real_offset = offset
+              end
+
+              @real_offset = (@marked.keys.min - 1) if @real_offset.negative?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/{filters/expirer.rb → consumer_groups/executor.rb}

@@ -31,40 +31,26 @@
 module Karafka
   module Pro
     module Processing
-      module Filters
-        # Expirer for removing too old messages.
-        # It never moves offsets in any way and does not impact the processing flow. It always
-        # runs `:skip` action.
-        class Expirer < Base
-          # @param ttl [Integer] maximum age of a message (in ms)
-          def initialize(ttl)
-            super()
-
-            @ttl = ttl
-          end
-
-          # Removes too old messages
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Pro executor that supports periodic jobs
+        class Executor < Karafka::Processing::ConsumerGroups::Executor
+          # Runs the code that should happen before periodic job is scheduled
           #
-          # @param messages [Array<Karafka::Messages::Message>]
-          def apply!(messages)
-            @applied = false
-
-            # Time on message is in seconds with ms precision, so we need to convert the ttl that
-            # is in ms to this format
-            border = Time.now.utc - (@ttl / 1_000.to_f)
-
-            messages.delete_if do |message|
-              too_old = message.timestamp < border
-
-              @applied = true if too_old
-
-              too_old
-            end
+          # @note While jobs are called `Periodic`, from the consumer perspective it is "ticking".
+          #   This name was taken for a reason: we may want to introduce periodic ticking also not
+          #   only during polling but for example on wait and a name "poll" would not align well.
+          #   A name "periodic" is not a verb and our other consumer actions are verbs like:
+          #   consume or revoked. So for the sake of consistency we have ticking here.
+          def before_schedule_periodic
+            consumer.on_before_schedule_tick
           end
 
-          # @return [nil] this filter does not deal with timeouts
-          def timeout
-            nil
+          # Triggers consumer ticking
+          def periodic
+            consumer.on_tick
           end
         end
       end

data/lib/karafka/pro/processing/{jobs/periodic.rb → consumer_groups/expansions_selector.rb}

@@ -31,27 +31,24 @@
 module Karafka
   module Pro
     module Processing
-      module Jobs
-        # Job that represents a "ticking" work. Work that we run periodically for the Periodics
-        # enabled topics.
-        class Periodic < Karafka::Processing::Jobs::Base
-          self.action = :tick
-
-          # @param executor [Karafka::Pro::Processing::Executor] pro executor that is suppose to
-          #   run a given job
-          def initialize(executor)
-            @executor = executor
-            super()
-          end
-
-          # Code executed before we schedule this job
-          def before_schedule
-            executor.before_schedule_periodic
-          end
-
-          # Runs the executor periodic action
-          def call
-            executor.periodic
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Pro selector of appropriate topic setup based features enhancements.
+        class ExpansionsSelector < Karafka::Processing::ConsumerGroups::ExpansionsSelector
+          # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
+          #   expansions
+          # @return [Array<Module>] modules with proper expansions we're suppose to use to enhance
+          #   the consumer
+          def find(topic)
+            # Start with the non-pro expansions
+            expansions = super
+            expansions << Pro::Processing::ConsumerGroups::Piping::Consumer
+            expansions << Pro::Processing::ConsumerGroups::OffsetMetadata::Consumer if topic.offset_metadata?
+            expansions << Pro::Processing::ConsumerGroups::AdaptiveIterator::Consumer if topic.adaptive_iterator?
+            expansions << Pro::Processing::ConsumerGroups::PeriodicJob::Consumer if topic.periodic_job?
+            expansions
           end
         end
       end

data/lib/karafka/pro/processing/consumer_groups/filters/base.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Filters
+          # Base for all the filters. All filters (including custom) need to use this API.
+          #
+          # Due to the fact, that filters can limit data in such a way, that we need to pause or
+          # seek (throttling for example), the api is not just "remove some things from batch" but
+          # also provides ways to control the post-filtering operations that may be needed.
+          class Base
+            # @return [Karafka::Messages::Message, nil] the message that we want to use as a cursor
+            #   one to pause or seek or nil if not applicable.
+            attr_reader :cursor
+
+            include Karafka::Core::Helpers::Time
+
+            # Initializes the filter as not yet applied
+            def initialize
+              @applied = false
+              @cursor = nil
+            end
+
+            # @param messages [Array<Karafka::Messages::Message>] array with messages. Please keep
+            #   in mind, this may already be partial due to execution of previous filters.
+            def apply!(messages)
+              raise NotImplementedError, "Implement in a subclass"
+            end
+
+            # @return [Symbol] filter post-execution action on consumer. Either `:skip`, `:pause` or
+            #   `:seek`.
+            def action
+              :skip
+            end
+
+            # @return [Boolean] did this filter change messages in any way
+            def applied?
+              @applied
+            end
+
+            # @return [Integer, nil] default timeout for pausing (if applicable) or nil if not
+            # @note Please do not return `0` when your filter is not pausing as it may interact
+            #   with other filters that want to pause.
+            def timeout
+              nil
+            end
+
+            # @return [Boolean] should we use the cursor value to mark as consumed. If any of the
+            #   filters returns true, we return lowers applicable cursor value (if any)
+            def mark_as_consumed?
+              false
+            end
+
+            # @return [Symbol] `:mark_as_consumed` or `:mark_as_consumed!`. Applicable only if
+            #   marking is requested
+            def marking_method
+              :mark_as_consumed
+            end
+
+            # @return [Karafka::Messages::Message, nil] cursor message for marking or nil if no
+            #   marking
+            def marking_cursor
+              cursor
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/filters/delayer.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module Filters
+          # A filter that allows us to delay processing by pausing until time is right.
+          class Delayer < Base
+            # @param delay [Integer] ms delay / minimum age of each message we want to process
+            def initialize(delay)
+              super()
+
+              @delay = delay
+            end
+
+            # Removes too young messages
+            #
+            # @param messages [Array<Karafka::Messages::Message>]
+            def apply!(messages)
+              @applied = false
+              @cursor = nil
+
+              # Time on message is in seconds with ms precision, so we need to convert the ttl that
+              # is in ms to this format
+              border = Time.now.utc - (@delay / 1_000.0)
+
+              messages.delete_if do |message|
+                too_young = message.timestamp > border
+
+                if too_young
+                  @applied = true
+
+                  @cursor ||= message
+                end
+
+                @applied
+              end
+            end
+
+            # @return [Integer] timeout delay in ms
+            def timeout
+              return 0 unless @cursor
+
+              timeout = (@delay / 1_000.0) - (::Time.now.utc - @cursor.timestamp)
+
+              (timeout <= 0) ? 0 : timeout * 1_000
+            end
+
+            # @return [Symbol] action to take on post-filtering
+            def action
+              return :skip unless applied?
+
+              (timeout <= 0) ? :seek : :pause
+            end
+          end
+        end
+      end
+    end
+  end
+end