karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/consumer_groups/jobs_builder.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      module ConsumerGroups
+        # Pro jobs builder that supports lrj
+        class JobsBuilder < Karafka::Processing::ConsumerGroups::JobsBuilder
+          # @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor]
+          def idle(executor)
+            Karafka::Processing::ConsumerGroups::Jobs::Idle.new(executor)
+          end
+
+          # @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor]
+          # @param messages [Karafka::Messages::Messages] messages batch to be consumed
+          # @return [Karafka::Processing::ConsumerGroups::Jobs::Consume] blocking job
+          # @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::ConsumeNonBlocking] non blocking
+          #   for lrj
+          def consume(executor, messages)
+            if executor.topic.long_running_job?
+              Jobs::ConsumeNonBlocking.new(executor, messages)
+            else
+              super
+            end
+          end
+
+          # @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor]
+          # @return [Karafka::Processing::ConsumerGroups::Jobs::Eofed] eofed job for non LRJ
+          # @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::EofedNonBlocking] eofed job that
+          #   is non-blocking, so when revocation job is scheduled for LRJ it also will not block
+          def eofed(executor)
+            if executor.topic.long_running_job?
+              Jobs::EofedNonBlocking.new(executor)
+            else
+              super
+            end
+          end
+
+          # @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor]
+          # @return [Karafka::Processing::ConsumerGroups::Jobs::Revoked] revocation job for non LRJ
+          # @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::RevokedNonBlocking] revocation
+          #   job that is non-blocking, so when revocation job is scheduled for LRJ it also will not
+          #   block
+          def revoked(executor)
+            if executor.topic.long_running_job?
+              Jobs::RevokedNonBlocking.new(executor)
+            else
+              super
+            end
+          end
+
+          # @param executor [Karafka::Pro::Processing::ConsumerGroups::Executor]
+          # @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::Periodic] Periodic job
+          # @return [Karafka::Pro::Processing::ConsumerGroups::Jobs::PeriodicNonBlocking] Periodic
+          #   non-blocking job
+          def periodic(executor)
+            if executor.topic.long_running_job?
+              Jobs::PeriodicNonBlocking.new(executor)
+            else
+              Jobs::Periodic.new(executor)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/offset_metadata/consumer.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Offset Metadata support on the processing side
+        module OffsetMetadata
+          # Extra API methods for offset metadata fetching
+          # @note Part of this feature API is embedded directly into the strategies because it alters
+          #   how marking methods (`#mark_as_consumed` and `#mark_as_consumed!`) operate. Because
+          #   of that, they had to be embedded into the strategies.
+          module Consumer
+            # @param cache [Boolean] should we use cached result if present (true by default)
+            # @return [false, Object] false in case we do not own the partition anymore or
+            #   deserialized metadata based on the deserializer
+            # @note Caching is on as the assumption here is, that most of the time user will be
+            #   interested only in the offset metadata that "came" from the time prior to the
+            #   rebalance. That is because the rest of the metadata (current) is created and
+            #   controlled by the user himself, thus there is no need to retrieve it. In case this
+            #   is not true and user wants to always get the Kafka metadata, `cache` value of this
+            #   feature can be set to false.
+            def offset_metadata(cache: true)
+              return false if revoked?
+
+              Fetcher.find(topic, partition, cache: cache)
+            end
+
+            alias_method :committed_offset_metadata, :offset_metadata
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/offset_metadata/fetcher.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Offset Metadata support on the processing side
+        module OffsetMetadata
+          # This fetcher is responsible for fetching and caching committed offsets metadata
+          # information.
+          #
+          # By design we fetch all information for a requested topic assignments. Not all topics from
+          # the same subscription group may need metadata and even if, we can run the few smaller
+          # queries. This approach prevents us from querying all assigned topics data in one go
+          # preventing excessive queries.
+          #
+          # Since the assumption is, that user will not have to reach out for the later metadata
+          # since it is produced in the context of a given consumer assignment, we can cache the
+          # initial result and only allow users for explicit invalidation.
+          class Fetcher
+            include Singleton
+
+            class << self
+              extend Forwardable
+
+              def_delegators :instance, :register, :clear, :find
+            end
+
+            # Initializes the fetcher with empty caches
+            def initialize
+              @mutexes = {}
+              @clients = {}
+              @tpls = {}
+            end
+
+            # Registers a client of a given subscription group, so we can use it for queries later on
+            # @param client [Karafka::Connection::Client]
+            # @note Since we store the client reference and not the underlying rdkafka consumer
+            #   instance, we do not have to deal with the recovery as it is abstracted away
+            def register(client)
+              @clients[client.subscription_group] = client
+              # We use one mutex per SG because independent SGs can query in parallel
+              @mutexes[client.subscription_group] = Mutex.new
+              @tpls[client.subscription_group] = {}
+            end
+
+            # Queries or retrieves from cache the given offset metadata for the selected partition
+            #
+            # @param topic [Karafka::Routing::Topic] routing topic with subscription group reference
+            # @param partition [Integer] partition for which we want to get stored offset metadata
+            # @param cache [Boolean] forces explicit query to Kafka when false and cache refresh.
+            #   By default we use the setting from the topic level but this can be overwritten on
+            #   a per request basis if needed.
+            # @return [Object, false] deserialized metadata (string deserializer by default) or
+            #   false in case we were not able to obtain the details because we have lost the
+            #   assignment
+            def find(topic, partition, cache: true)
+              cache = topic.offset_metadata.cache? && cache
+
+              tpls = fetch(topic, cache)
+
+              return false unless tpls
+
+              t_partitions = tpls.fetch(topic.name, [])
+              t_partition = t_partitions.find { |t_p| t_p.partition == partition }
+
+              # If we do not have given topic partition here, it means it is no longer part of our
+              # assignment and we should return false
+              return false unless t_partition
+
+              topic.offset_metadata.deserializer.call(t_partition.metadata)
+            end
+
+            # Clears cache of a given subscription group. It is triggered on assignment changes.
+            #
+            # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group that
+            #   we want to clear.
+            def clear(subscription_group)
+              @mutexes.fetch(subscription_group).synchronize do
+                @tpls[subscription_group].clear
+              end
+            end
+
+            private
+
+            # Fetches from Kafka all committed offsets for the given topic partitions that are
+            # assigned to this process.
+            #
+            # We fetch all because in majority of the cases, the behavior of the end user code is
+            #   not specific to a given partition both same for all. In such cases we save on
+            #   querying as we get all data for all partitions in one go.
+            #
+            # @param topic [Karafka::Routing::Topic] topic for which we want to fetch tpls data
+            # @param cache [Boolean] should we return cached data if present
+            def fetch(topic, cache)
+              subscription_group = topic.subscription_group
+              t_tpls = @tpls.fetch(subscription_group, false)
+              t_tpl = t_tpls[topic]
+
+              return t_tpl if t_tpl && cache
+
+              assigned_tpls = @clients.fetch(subscription_group).assignment
+              t_tpl = assigned_tpls.to_h.fetch(topic.name, false)
+
+              # May be false in case we lost given assignment but still run LRJ
+              return false unless t_tpl
+              return false if t_tpl.empty?
+
+              @mutexes.fetch(subscription_group).synchronize do
+                rd_tpl = Rdkafka::Consumer::TopicPartitionList.new(topic.name => t_tpl)
+
+                # While in theory we could lost assignment while being here, this will work and will
+                # return us proper tpl, we do not deal with this case on this layer and report anyhow
+                # There will not be any exception and this will operate correctly
+                t_tpls[topic] = @clients.fetch(subscription_group).committed(rd_tpl).to_h
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/offset_metadata/listener.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module OffsetMetadata
+          # Keeps track of rebalances and updates the fetcher
+          # Since we cache the tpls with metadata, we need to invalidate them on events that would
+          # cause changes in the assignments
+          class Listener
+            # When we start listening we need to register this client in the metadata fetcher, so
+            # we have the client related to a given subscription group that we can use in fetcher
+            # since fetcher may be used in filtering API and other places outside of the standard
+            # consumer flow
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_connection_listener_before_fetch_loop(event)
+              Fetcher.register event[:client]
+            end
+
+            # Invalidates internal cache when assignments change so we can get correct metadata
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_rebalance_partitions_assigned(event)
+              Fetcher.clear event[:subscription_group]
+            end
+
+            # Invalidates internal cache when assignments change so we can get correct metadata
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_rebalance_partitions_revoked(event)
+              Fetcher.clear event[:subscription_group]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/base.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        module ParallelSegments
+          # Module for filters injected into the processing pipeline of each of the topics used
+          # within the parallel segmented consumer groups
+          module Filters
+            # Base class for filters for parallel segments that deal with different feature scenarios
+            class Base < Pro::Processing::ConsumerGroups::Filters::Base
+              # @param segment_id [Integer] numeric id of the parallel segment group to use with the
+              #   partitioner and reducer for segment matching comparison
+              # @param partitioner [Proc]
+              # @param reducer [Proc]
+              def initialize(segment_id:, partitioner:, reducer:)
+                super()
+
+                @segment_id = segment_id
+                @partitioner = partitioner
+                @reducer = reducer
+              end
+
+              private
+
+              # @param message [Karafka::Messages::Message] received message
+              # @return [String, Numeric] segment assignment key
+              def partition(message)
+                @partitioner.call(message)
+              rescue => e
+                # This should not happen. If you are seeing this it means your partitioner code
+                # failed and raised an error. We highly recommend mitigating partitioner level errors
+                # on the user side because this type of collapse should be considered a last resort
+                Karafka.monitor.instrument(
+                  "error.occurred",
+                  caller: self,
+                  error: e,
+                  message: message,
+                  type: "parallel_segments.partitioner.error"
+                )
+
+                :failure
+              end
+
+              # @param message_segment_key [String, Numeric] segment key to pass to the reducer
+              # @return [Integer] segment assignment of a given message
+              def reduce(message_segment_key)
+                # Assign to segment 0 always in case of failures in partitioner This is a fail-safe
+                return 0 if message_segment_key == :failure
+
+                @reducer.call(message_segment_key)
+              rescue => e
+                # @see `#partition` method error handling doc
+                Karafka.monitor.instrument(
+                  "error.occurred",
+                  caller: self,
+                  error: e,
+                  message_segment_key: message_segment_key,
+                  type: "parallel_segments.reducer.error"
+                )
+
+                0
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/consumer_groups/parallel_segments/filters/default.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+# Karafka Pro - Source Available Commercial Software
+# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
+#
+# This software is NOT open source. It is source-available commercial software
+# requiring a paid license for use. It is NOT covered by LGPL.
+#
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
+# PROHIBITED:
+# - Use without a valid commercial license
+# - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
+# - Use as training data for AI/ML models or inclusion in datasets
+# - Scraping, crawling, or automated collection for any purpose
+#
+# PERMITTED:
+# - Reading, referencing, and linking for personal or commercial use
+# - Runtime retrieval by AI assistants, coding agents, and RAG systems
+#   for the purpose of providing contextual help to Karafka users
+#
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
+# License: https://karafka.io/docs/Pro-License-Comm/
+# Contact: contact@karafka.io
+
+module Karafka
+  module Pro
+    module Processing
+      # Consumer-group-specific Pro processing components (driven by rebalance callbacks and
+      # partition ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932
+      # lands.
+      module ConsumerGroups
+        # Processing components namespace for parallel segments feature
+        module ParallelSegments
+          module Filters
+            # Filter used for handling parallel segments with automatic offset management. Handles
+            # message distribution and ensures proper offset management when messages are filtered
+            # out during the distribution process.
+            #
+            # When operating in automatic offset management mode, this filter takes care of marking
+            # offsets of messages that were filtered out during the distribution process to maintain
+            # proper offset progression.
+            #
+            # @note This is the default filter that should be used when manual offset management
+            #   is not enabled. For manual offset management scenarios use the Mom filter instead.
+            class Default < Base
+              # Applies the filter to the batch of messages
+              # It removes messages that don't belong to the current parallel segment group
+              # based on the partitioner and reducer logic
+              #
+              # @param messages [Array<Karafka::Messages::Message>] messages batch that we want to
+              #   filter
+              def apply!(messages)
+                @applied = false
+                @all_filtered = false
+                @cursor = messages.first
+
+                # Keep track of how many messages we had initially
+                initial_size = messages.size
+
+                # Filter out messages that don't match our segment group
+                messages.delete_if do |message|
+                  message_segment_key = partition(message)
+
+                  # Use the reducer to get the target group for this message
+                  target_segment = reduce(message_segment_key)
+
+                  # Remove the message if it doesn't belong to our group
+                  remove = target_segment != @segment_id
+
+                  if remove
+                    @cursor = message
+                    @applied = true
+                  end
+
+                  remove
+                end
+
+                # If all messages were filtered out, we want to mark them as consumed
+                @all_filtered = messages.empty? && initial_size.positive?
+              end
+
+              # @return [Boolean] true if any messages were filtered out
+              def applied?
+                @applied
+              end
+
+              # @return [Boolean] true if we should mark as consumed (when all were filtered)
+              def mark_as_consumed?
+                @all_filtered
+              end
+
+              # @return [nil] Since we do not timeout ever in this filter, we should not return
+              #   any value for it.
+              def timeout
+                nil
+              end
+
+              # Only return cursor if we wanted to mark as consumed in case all was filtered.
+              # Otherwise it could interfere with other filters
+              def cursor
+                @all_filtered ? @cursor : nil
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end