karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/processing/offset_metadata/fetcher.rb

@@ -1,149 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Offset Metadata support on the processing side
-      module OffsetMetadata
-        # This fetcher is responsible for fetching and caching committed offsets metadata
-        # information.
-        #
-        # By design we fetch all information for a requested topic assignments. Not all topics from
-        # the same subscription group may need metadata and even if, we can run the few smaller
-        # queries. This approach prevents us from querying all assigned topics data in one go
-        # preventing excessive queries.
-        #
-        # Since the assumption is, that user will not have to reach out for the later metadata
-        # since it is produced in the context of a given consumer assignment, we can cache the
-        # initial result and only allow users for explicit invalidation.
-        class Fetcher
-          include Singleton
-
-          class << self
-            extend Forwardable
-
-            def_delegators :instance, :register, :clear, :find
-          end
-
-          # Initializes the fetcher with empty caches
-          def initialize
-            @mutexes = {}
-            @clients = {}
-            @tpls = {}
-          end
-
-          # Registers a client of a given subscription group, so we can use it for queries later on
-          # @param client [Karafka::Connection::Client]
-          # @note Since we store the client reference and not the underlying rdkafka consumer
-          #   instance, we do not have to deal with the recovery as it is abstracted away
-          def register(client)
-            @clients[client.subscription_group] = client
-            # We use one mutex per SG because independent SGs can query in parallel
-            @mutexes[client.subscription_group] = Mutex.new
-            @tpls[client.subscription_group] = {}
-          end
-
-          # Queries or retrieves from cache the given offset metadata for the selected partition
-          #
-          # @param topic [Karafka::Routing::Topic] routing topic with subscription group reference
-          # @param partition [Integer] partition for which we want to get stored offset metadata
-          # @param cache [Boolean] forces explicit query to Kafka when false and cache refresh.
-          #   By default we use the setting from the topic level but this can be overwritten on
-          #   a per request basis if needed.
-          # @return [Object, false] deserialized metadata (string deserializer by default) or
-          #   false in case we were not able to obtain the details because we have lost the
-          #   assignment
-          def find(topic, partition, cache: true)
-            cache = topic.offset_metadata.cache? && cache
-
-            tpls = fetch(topic, cache)
-
-            return false unless tpls
-
-            t_partitions = tpls.fetch(topic.name, [])
-            t_partition = t_partitions.find { |t_p| t_p.partition == partition }
-
-            # If we do not have given topic partition here, it means it is no longer part of our
-            # assignment and we should return false
-            return false unless t_partition
-
-            topic.offset_metadata.deserializer.call(t_partition.metadata)
-          end
-
-          # Clears cache of a given subscription group. It is triggered on assignment changes.
-          #
-          # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group that
-          #   we want to clear.
-          def clear(subscription_group)
-            @mutexes.fetch(subscription_group).synchronize do
-              @tpls[subscription_group].clear
-            end
-          end
-
-          private
-
-          # Fetches from Kafka all committed offsets for the given topic partitions that are
-          # assigned to this process.
-          #
-          # We fetch all because in majority of the cases, the behavior of the end user code is
-          #   not specific to a given partition both same for all. In such cases we save on
-          #   querying as we get all data for all partitions in one go.
-          #
-          # @param topic [Karafka::Routing::Topic] topic for which we want to fetch tpls data
-          # @param cache [Boolean] should we return cached data if present
-          def fetch(topic, cache)
-            subscription_group = topic.subscription_group
-            t_tpls = @tpls.fetch(subscription_group, false)
-            t_tpl = t_tpls[topic]
-
-            return t_tpl if t_tpl && cache
-
-            assigned_tpls = @clients.fetch(subscription_group).assignment
-            t_tpl = assigned_tpls.to_h.fetch(topic.name, false)
-
-            # May be false in case we lost given assignment but still run LRJ
-            return false unless t_tpl
-            return false if t_tpl.empty?
-
-            @mutexes.fetch(subscription_group).synchronize do
-              rd_tpl = Rdkafka::Consumer::TopicPartitionList.new(topic.name => t_tpl)
-
-              # While in theory we could lost assignment while being here, this will work and will
-              # return us proper tpl, we do not deal with this case on this layer and report anyhow
-              # There will not be any exception and this will operate correctly
-              t_tpls[topic] = @clients.fetch(subscription_group).committed(rd_tpl).to_h
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/offset_metadata/listener.rb

@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module OffsetMetadata
-        # Keeps track of rebalances and updates the fetcher
-        # Since we cache the tpls with metadata, we need to invalidate them on events that would
-        # cause changes in the assignments
-        class Listener
-          # When we start listening we need to register this client in the metadata fetcher, so
-          # we have the client related to a given subscription group that we can use in fetcher
-          # since fetcher may be used in filtering API and other places outside of the standard
-          # consumer flow
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_connection_listener_before_fetch_loop(event)
-            Fetcher.register event[:client]
-          end
-
-          # Invalidates internal cache when assignments change so we can get correct metadata
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_rebalance_partitions_assigned(event)
-            Fetcher.clear event[:subscription_group]
-          end
-
-          # Invalidates internal cache when assignments change so we can get correct metadata
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_rebalance_partitions_revoked(event)
-            Fetcher.clear event[:subscription_group]
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/parallel_segments/filters/base.rb

@@ -1,98 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module ParallelSegments
-        # Module for filters injected into the processing pipeline of each of the topics used
-        # within the parallel segmented consumer groups
-        module Filters
-          # Base class for filters for parallel segments that deal with different feature scenarios
-          class Base < Processing::Filters::Base
-            # @param segment_id [Integer] numeric id of the parallel segment group to use with the
-            #   partitioner and reducer for segment matching comparison
-            # @param partitioner [Proc]
-            # @param reducer [Proc]
-            def initialize(segment_id:, partitioner:, reducer:)
-              super()
-
-              @segment_id = segment_id
-              @partitioner = partitioner
-              @reducer = reducer
-            end
-
-            private
-
-            # @param message [Karafka::Messages::Message] received message
-            # @return [String, Numeric] segment assignment key
-            def partition(message)
-              @partitioner.call(message)
-            rescue => e
-              # This should not happen. If you are seeing this it means your partitioner code
-              # failed and raised an error. We highly recommend mitigating partitioner level errors
-              # on the user side because this type of collapse should be considered a last resort
-              Karafka.monitor.instrument(
-                "error.occurred",
-                caller: self,
-                error: e,
-                message: message,
-                type: "parallel_segments.partitioner.error"
-              )
-
-              :failure
-            end
-
-            # @param message_segment_key [String, Numeric] segment key to pass to the reducer
-            # @return [Integer] segment assignment of a given message
-            def reduce(message_segment_key)
-              # Assign to segment 0 always in case of failures in partitioner
-              # This is a fail-safe
-              return 0 if message_segment_key == :failure
-
-              @reducer.call(message_segment_key)
-            rescue => e
-              # @see `#partition` method error handling doc
-              Karafka.monitor.instrument(
-                "error.occurred",
-                caller: self,
-                error: e,
-                message_segment_key: message_segment_key,
-                type: "parallel_segments.reducer.error"
-              )
-
-              0
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/parallel_segments/filters/default.rb

@@ -1,110 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Processing components namespace for parallel segments feature
-      module ParallelSegments
-        module Filters
-          # Filter used for handling parallel segments with automatic offset management. Handles
-          # message distribution and ensures proper offset management when messages are filtered
-          # out during the distribution process.
-          #
-          # When operating in automatic offset management mode, this filter takes care of marking
-          # offsets of messages that were filtered out during the distribution process to maintain
-          # proper offset progression.
-          #
-          # @note This is the default filter that should be used when manual offset management
-          #   is not enabled. For manual offset management scenarios use the Mom filter instead.
-          class Default < Base
-            # Applies the filter to the batch of messages
-            # It removes messages that don't belong to the current parallel segment group
-            # based on the partitioner and reducer logic
-            #
-            # @param messages [Array<Karafka::Messages::Message>] messages batch that we want to
-            #   filter
-            def apply!(messages)
-              @applied = false
-              @all_filtered = false
-              @cursor = messages.first
-
-              # Keep track of how many messages we had initially
-              initial_size = messages.size
-
-              # Filter out messages that don't match our segment group
-              messages.delete_if do |message|
-                message_segment_key = partition(message)
-
-                # Use the reducer to get the target group for this message
-                target_segment = reduce(message_segment_key)
-
-                # Remove the message if it doesn't belong to our group
-                remove = target_segment != @segment_id
-
-                if remove
-                  @cursor = message
-                  @applied = true
-                end
-
-                remove
-              end
-
-              # If all messages were filtered out, we want to mark them as consumed
-              @all_filtered = messages.empty? && initial_size.positive?
-            end
-
-            # @return [Boolean] true if any messages were filtered out
-            def applied?
-              @applied
-            end
-
-            # @return [Boolean] true if we should mark as consumed (when all were filtered)
-            def mark_as_consumed?
-              @all_filtered
-            end
-
-            # @return [nil] Since we do not timeout ever in this filter, we should not return
-            #   any value for it.
-            def timeout
-              nil
-            end
-
-            # Only return cursor if we wanted to mark as consumed in case all was filtered.
-            # Otherwise it could interfere with other filters
-            def cursor
-              @all_filtered ? @cursor : nil
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/parallel_segments/filters/mom.rb

@@ -1,91 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module ParallelSegments
-        module Filters
-          # Filter used for handling parallel segments when manual offset management (mom) is
-          # enabled. Provides message distribution without any post-filtering offset state
-          # management as it is fully user-based.
-          #
-          # Since with manual offset management we need to ensure that offsets are never marked
-          # even in cases where all data in a batch is filtered out.
-          #
-          # This separation allows for cleaner implementation and easier debugging of each flow.
-          #
-          # @note This filter should be used only when manual offset management is enabled.
-          #   For automatic offset management scenarios use the regular filter instead.
-          class Mom < Base
-            # Applies the filter to the batch of messages
-            # It removes messages that don't belong to the current parallel segment group
-            # based on the partitioner and reducer logic without any offset marking
-            #
-            # @param messages [Array<Karafka::Messages::Message>] messages batch that we want to
-            #   filter
-            def apply!(messages)
-              @applied = false
-
-              # Filter out messages that don't match our segment group
-              messages.delete_if do |message|
-                message_segment_key = partition(message)
-                # Use the reducer to get the target group for this message
-                target_segment = reduce(message_segment_key)
-                # Remove the message if it doesn't belong to our segment
-                remove = target_segment != @segment_id
-
-                @applied = true if remove
-
-                remove
-              end
-            end
-
-            # @return [Boolean] true if any messages were filtered out
-            def applied?
-              @applied
-            end
-
-            # @return [Boolean] false, as mom mode never marks as consumed automatically
-            def mark_as_consumed?
-              false
-            end
-
-            # @return [nil] Since we do not timeout ever in this filter, we should not return
-            #   any value for it.
-            def timeout
-              nil
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/periodic_job/consumer.rb

@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Namespace for periodic jobs related processing APIs
-      module PeriodicJob
-        # Consumer extra methods useful only when periodic jobs are in use
-        module Consumer
-          class << self
-            # Defines an empty `#tick` method if not present
-            #
-            # We define it that way due to our injection strategy flow.
-            #
-            # @param consumer_singleton_class [Karafka::BaseConsumer] consumer singleton class
-            #   that is being enriched with periodic jobs API
-            def included(consumer_singleton_class)
-              # Do not define empty tick method on consumer if it already exists
-              # We only define it when it does not exist to have empty periodic ticking
-              #
-              # We need to check both cases (public and private) since user is not expected to
-              # have this method public
-              return if consumer_singleton_class.method_defined?(:tick)
-              return if consumer_singleton_class.private_instance_methods.include?(:tick)
-
-              # Create empty ticking method
-              consumer_singleton_class.class_eval do
-                def tick
-                end
-              end
-            end
-          end
-
-          # Runs the on-schedule tick periodic operations
-          # This method is an alias but is part of the naming convention used for other flows, this
-          # is why we do not reference the `handle_before_schedule_tick` directly
-          def on_before_schedule_tick
-            handle_before_schedule_tick
-          end
-
-          # Used by the executor to trigger consumer tick
-          # @private
-          def on_tick
-            handle_tick
-          rescue => e
-            Karafka.monitor.instrument(
-              "error.occurred",
-              error: e,
-              caller: self,
-              type: "consumer.tick.error"
-            )
-          end
-        end
-      end
-    end
-  end
-end