karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/pro/routing/features/virtual_partitions/topic.rb

@@ -1,99 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        class VirtualPartitions < Base
-          # Topic extensions to be able to manage virtual partitions feature
-          module Topic
-            # This method calls the parent class initializer and then sets up the
-            # extra instance variable to nil. The explicit initialization
-            # to nil is included as an optimization for Ruby's object shapes system,
-            # which improves memory layout and access performance.
-            def initialize(...)
-              super
-              @virtual_partitions = nil
-            end
-
-            # @param max_partitions [Integer] max number of virtual partitions that can come out of
-            #   the single distribution flow. When set to more than the Karafka threading, will
-            #   create more work than workers. When less, can ensure we have spare resources to
-            #   process other things in parallel.
-            # @param partitioner [nil, #call] nil or callable partitioner
-            # @param offset_metadata_strategy [Symbol] how we should match the metadata for the
-            #   offset. `:exact` will match the offset matching metadata and `:current` will select
-            #   the most recently reported metadata
-            # @param reducer [nil, #call] reducer for VPs key. It allows for using a custom
-            #   reducer to achieve enhanced parallelization when the default reducer is not enough.
-            # @param distribution [Symbol] the strategy to use for virtual partitioning. Can be
-            #   either `:consistent` or `:balanced`. The `:balanced` strategy ensures balanced
-            #   distribution of work across available workers while maintaining message order
-            #   within groups.
-            # @return [VirtualPartitions] method that allows to set the virtual partitions details
-            #   during the routing configuration and then allows to retrieve it
-            def virtual_partitions(
-              max_partitions: Karafka::App.config.concurrency,
-              partitioner: nil,
-              offset_metadata_strategy: :current,
-              reducer: nil,
-              distribution: :consistent
-            )
-              @virtual_partitions ||= Config.new(
-                active: !partitioner.nil?,
-                max_partitions: max_partitions,
-                partitioner: partitioner,
-                offset_metadata_strategy: offset_metadata_strategy,
-                # If no reducer provided, we use this one. It just runs a modulo on the sum of
-                # a stringified version, providing fairly good distribution.
-                reducer: reducer || ->(virtual_key) { virtual_key.to_s.sum % max_partitions },
-                distribution: distribution
-              )
-            end
-
-            # @return [Boolean] are virtual partitions enabled for given topic
-            def virtual_partitions?
-              virtual_partitions.active?
-            end
-
-            # @return [Hash] topic with all its native configuration options plus manual offset
-            #   management namespace settings
-            def to_h
-              super.merge(
-                virtual_partitions: virtual_partitions.to_h
-              ).freeze
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/routing/features/virtual_partitions.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Routing
-      module Features
-        # Virtual Partitions feature config and DSL namespace.
-        #
-        # Virtual Partitions allow you to parallelize the processing of data from a single
-        # partition. This can drastically increase throughput when IO operations are involved.
-        class VirtualPartitions < Base
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/executor.rb

@@ -1,216 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  # Namespace that encapsulates all the logic related to processing data.
-  module Processing
-    # Executors:
-    # - run consumers code (for `#call`) or run given preparation / teardown operations when needed
-    #   from separate threads.
-    # - they re-create consumer instances in case of partitions that were revoked and assigned
-    #   back.
-    #
-    # @note Executors are not removed after partition is revoked. They are not that big and will
-    #   be re-used in case of a re-claim
-    #
-    # @note Since given consumer can run various operations, executor manages that and its
-    #   lifecycle. There are following types of operations with appropriate before/after, etc:
-    #
-    #   - consume - primary operation related to running user consumption code
-    #   - idle - cleanup job that runs on idle runs where no messages would be passed to the end
-    #     user. This is used for complex flows with filters, etc
-    #   - revoked - runs after the partition was revoked
-    #   - shutdown - runs when process is going to shutdown
-    class Executor
-      extend Forwardable
-      include Helpers::ConfigImporter.new(
-        strategy_selector: %i[internal processing strategy_selector],
-        expansions_selector: %i[internal processing expansions_selector]
-      )
-
-      def_delegators :@coordinator, :topic, :partition
-
-      # @return [String] unique id that we use to ensure, that we use for state tracking
-      attr_reader :id
-
-      # @return [String] subscription group id to which a given executor belongs
-      attr_reader :group_id
-
-      # @return [Karafka::Messages::Messages] messages batch
-      attr_reader :messages
-
-      # @return [Karafka::Processing::Coordinator] coordinator for this executor
-      attr_reader :coordinator
-
-      # @param group_id [String] id of the subscription group to which the executor belongs
-      # @param client [Karafka::Connection::Client] kafka client
-      # @param coordinator [Karafka::Processing::Coordinator]
-      def initialize(group_id, client, coordinator)
-        @id = SecureRandom.hex(6)
-        @group_id = group_id
-        @client = client
-        @coordinator = coordinator
-      end
-
-      # Allows us to prepare the consumer in the listener thread prior to the job being send to
-      # be scheduled. It also allows to run some code that is time sensitive and cannot wait in the
-      # queue as it could cause starvation.
-      #
-      # @param messages [Array<Karafka::Messages::Message>]
-      def before_schedule_consume(messages)
-        # Recreate consumer with each batch if persistence is not enabled
-        # We reload the consumers with each batch instead of relying on some external signals
-        # when needed for consistency. That way devs may have it on or off and not in this
-        # middle state, where re-creation of a consumer instance would occur only sometimes
-        @consumer = nil unless topic.consumer_persistence
-
-        # First we build messages batch...
-        consumer.messages = Messages::Builders::Messages.call(
-          messages,
-          topic,
-          partition,
-          # the moment we've received the batch or actually the moment we've enqueued it,
-          # but good enough
-          Time.now
-        )
-
-        consumer.on_before_schedule_consume
-      end
-
-      # Runs setup and warm-up code in the worker prior to running the consumption
-      def before_consume
-        consumer.on_before_consume
-      end
-
-      # Runs the wrap/around execution context appropriate for a given action
-      # @param action [Symbol] action execution wrapped with our block
-      def wrap(action, &)
-        consumer.on_wrap(action, &)
-      end
-
-      # Runs consumer data processing against given batch and handles failures and errors.
-      def consume
-        # We run the consumer client logic...
-        consumer.on_consume
-      end
-
-      # Runs consumer after consumption code
-      def after_consume
-        consumer.on_after_consume
-      end
-
-      # Runs the code needed before idle work is scheduled
-      def before_schedule_idle
-        consumer.on_before_schedule_idle
-      end
-
-      # Runs consumer idle operations
-      # This may include house-keeping or other state management changes that can occur but that
-      # not mean there are any new messages available for the end user to process
-      def idle
-        consumer.on_idle
-      end
-
-      # Runs the code needed before eofed work is scheduled
-      def before_schedule_eofed
-        consumer.on_before_schedule_eofed
-      end
-
-      # Runs consumed eofed operation.
-      # This may run even when there were no messages received prior. This will however not
-      # run when eof is received together with messages as in such case `#consume` will run
-      def eofed
-        consumer.on_eofed
-      end
-
-      # Runs code needed before revoked job is scheduled
-      def before_schedule_revoked
-        consumer.on_before_schedule_revoked if @consumer
-      end
-
-      # Runs the controller `#revoked` method that should be triggered when a given consumer is
-      # no longer needed due to partitions reassignment.
-      #
-      # @note Clearing the consumer will ensure, that if we get the partition back, it will be
-      #   handled with a consumer with a clean state.
-      #
-      # @note We run it only when consumer was present, because presence indicates, that at least
-      #   a single message has been consumed.
-      #
-      # @note We do not reset the consumer but we indicate need for recreation instead, because
-      #   after the revocation, there still may be `#after_consume` running that needs a given
-      #   consumer instance.
-      def revoked
-        consumer.on_revoked if @consumer
-      end
-
-      # Runs code needed before shutdown job is scheduled
-      def before_schedule_shutdown
-        consumer.on_before_schedule_shutdown if @consumer
-      end
-
-      # Runs the controller `#shutdown` method that should be triggered when a given consumer is
-      # no longer needed as we're closing the process.
-      #
-      # @note While we do not need to clear the consumer here, it's a good habit to clean after
-      #   work is done.
-      def shutdown
-        # There is a case, where the consumer no longer exists because it was revoked, in case like
-        # that we do not build a new instance and shutdown should not be triggered.
-        consumer.on_shutdown if @consumer
-      end
-
-      private
-
-      # @return [Object] cached consumer instance
-      def consumer
-        @consumer ||= begin
-          topic = @coordinator.topic
-
-          strategy = strategy_selector.find(topic)
-
-          consumer = topic.consumer_class.new
-          # We use singleton class as the same consumer class may be used to process different
-          # topics with different settings
-          consumer.singleton_class.include(strategy)
-
-          # Specific features may expand consumer API beyond the injected strategy. The difference
-          # here is that strategy impacts the flow of states while extra APIs just provide some
-          # extra methods with informations, etc but do no deviate the flow behavior
-          expansions = expansions_selector.find(topic)
-          expansions.each { |expansion| consumer.singleton_class.include(expansion) }
-
-          consumer.client = @client
-          consumer.coordinator = @coordinator
-          # We assign producer only when not available already. It may already be available if
-          # user redefined the `#producer` method for example. This can be useful for example when
-          # having a multi-cluster setup and using a totally custom producer
-          consumer.producer ||= Karafka::App.producer
-          # Since we have some message-less flows (idle, etc), we initialize consumer with empty
-          # messages set. In production we have persistent consumers, so this is not a performance
-          # overhead as this will happen only once per consumer lifetime
-          consumer.messages = empty_messages
-
-          # Run the post-initialization hook for users that need to run some actions when consumer
-          # is built and ready (all basic state and info).
-          # Users should **not** overwrite the `#initialize` because it won't have all the needed
-          # data assigned yet.
-          consumer.on_initialized
-
-          consumer
-        end
-      end
-
-      # Initializes the messages set in case given operation would happen before any processing
-      # This prevents us from having no messages object at all as the messages object and
-      # its metadata may be used for statistics
-      def empty_messages
-        Messages::Builders::Messages.call(
-          [],
-          topic,
-          partition,
-          Time.now
-        )
-      end
-    end
-  end
-end

data/lib/karafka/processing/executors_buffer.rb

@@ -1,90 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    # Buffer for executors of a given subscription group. It wraps around the concept of building
-    # and caching them, so we can re-use them instead of creating new each time.
-    class ExecutorsBuffer
-      include Helpers::ConfigImporter.new(
-        executor_class: %i[internal processing executor_class]
-      )
-
-      # @param client [Connection::Client]
-      # @param subscription_group [Routing::SubscriptionGroup]
-      # @return [ExecutorsBuffer]
-      def initialize(client, subscription_group)
-        @subscription_group = subscription_group
-        @client = client
-        # We need two layers here to keep track of topics, partitions and processing groups
-        @buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
-      end
-
-      # Finds or creates an executor based on the provided details
-      #
-      # @param topic [String] topic name
-      # @param partition [Integer] partition number
-      # @param parallel_key [String] parallel group key
-      # @param coordinator [Karafka::Processing::Coordinator]
-      # @return [Executor, Pro::Processing::Executor] consumer executor
-      def find_or_create(topic, partition, parallel_key, coordinator)
-        @buffer[topic][partition][parallel_key] ||= executor_class.new(
-          @subscription_group.id,
-          @client,
-          coordinator
-        )
-      end
-
-      # Finds all existing executors for given topic partition or creates one for it
-      # @param topic [String] topic name
-      # @param partition [Integer] partition number
-      # @param coordinator [Karafka::Processing::Coordinator]
-      # @return [Array<Executor, Pro::Processing::Executor>]
-      def find_all_or_create(topic, partition, coordinator)
-        existing = find_all(topic, partition)
-
-        return existing unless existing.empty?
-
-        [find_or_create(topic, partition, 0, coordinator)]
-      end
-
-      # Revokes executors of a given topic partition, so they won't be used anymore for incoming
-      # messages
-      #
-      # @param topic [String] topic name
-      # @param partition [Integer] partition number
-      def revoke(topic, partition)
-        @buffer[topic][partition].clear
-      end
-
-      # Finds all the executors available for a given topic partition
-      #
-      # @param topic [String] topic name
-      # @param partition [Integer] partition number
-      # @return [Array<Executor, Pro::Processing::Executor>] executors in use for this
-      #   topic + partition
-      def find_all(topic, partition)
-        @buffer[topic][partition].values
-      end
-
-      # Iterates over all available executors and yields them together with topic and partition
-      # info
-      # @yieldparam [Routing::Topic] karafka routing topic object
-      # @yieldparam [Integer] partition number
-      # @yieldparam [Executor, Pro::Processing::Executor] given executor
-      def each
-        @buffer.each_value do |partitions|
-          partitions.each_value do |executors|
-            executors.each_value do |executor|
-              yield(executor)
-            end
-          end
-        end
-      end
-
-      # Clears the executors buffer. Useful for critical errors recovery.
-      def clear
-        @buffer.clear
-      end
-    end
-  end
-end

data/lib/karafka/processing/expansions_selector.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    # Selector of appropriate topic setup based features enhancements.
-    #
-    # Those expansions to the consumer API are NOT about the flow of processing. For this we have
-    # strategies. Those are suppose to provide certain extra APIs that user can use to get some
-    # extra non-flow related functionalities.
-    class ExpansionsSelector
-      # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
-      #   expansions
-      # @return [Array<Module>] modules with proper expansions we're suppose to use to enhance the
-      #   consumer
-      def find(topic)
-        expansions = []
-        expansions << Processing::InlineInsights::Consumer if topic.inline_insights?
-        expansions
-      end
-    end
-  end
-end

data/lib/karafka/processing/inline_insights/consumer.rb

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    # Namespace of the Inline Insights feature "non routing" related components
-    #
-    # @note We use both `#insights` because it is the feature name but also `#statistics` to make
-    #   it consistent with the fact that we publish and operate on statistics. User can pick
-    #   whichever name they prefer.
-    module InlineInsights
-      # Module that adds extra methods to the consumer that allow us to fetch the insights
-      module Consumer
-        # @return [Hash] empty hash or hash with given partition insights if already present
-        # @note We cache insights on the consumer, as in some scenarios we may no longer have them
-        #   inside the Tracker, for example under involuntary revocation, incoming statistics may
-        #   no longer have lost partition insights. Since we want to be consistent during single
-        #   batch operations, we want to ensure, that if we have insights they are available
-        #   throughout the whole processing.
-        def insights
-          insights = Tracker.find(topic, partition)
-
-          # If we no longer have new insights but we still have them locally, we can use them
-          return @insights if @insights && insights.empty?
-          # If insights are still the same, we can use them
-          return @insights if @insights.equal?(insights)
-
-          # If we've received new insights that are not empty, we can cache them
-          @insights = insights
-        end
-
-        # @return [Boolean] true if there are insights to work with, otherwise false
-        def insights?
-          !insights.empty?
-        end
-
-        alias_method :statistics, :insights
-        alias_method :statistics?, :insights?
-        alias_method :inline_insights, :insights
-        alias_method :inline_insights?, :insights?
-      end
-    end
-  end
-end

data/lib/karafka/processing/inline_insights/listener.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module InlineInsights
-      # Listener that adds statistics to our inline tracker
-      class Listener
-        # Adds statistics to the tracker
-        # @param event [Karafka::Core::Monitoring::Event] event with statistics
-        def on_statistics_emitted(event)
-          Tracker.add(
-            event[:consumer_group_id],
-            event[:statistics]
-          )
-        end
-      end
-    end
-  end
-end

data/lib/karafka/processing/inline_insights/tracker.rb

@@ -1,129 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Processing
-    module InlineInsights
-      # Object used to track statistics coming from librdkafka in a way that can be accessible by
-      # the consumers
-      #
-      # We use a single tracker because we do not need state management here as our consumer groups
-      # clients identified by statistics name value are unique. On top of that, having a per
-      # process one that is a singleton allows us to use tracker easily also from other places like
-      # filtering API etc.
-      #
-      # @note We include cache of 5 minutes for revoked partitions to compensate for cases where
-      #   when using LRJ a lost partition data would not be present anymore, however we would still
-      #   be in the processing phase. Since those metrics are published with each `poll`, regular
-      #   processing is not a subject of this issue. For LRJ we keep the reference. The only case
-      #   where this could be switched midway is when LRJ is running for an extended period of time
-      #   after the involuntary revocation. Having a time based cache instead of tracking
-      #   simplifies the design as we do not have to deal with state tracking, especially since
-      #   we would have to track also operations running in a revoked state.
-      #
-      # @note This tracker keeps in memory data about all topics and partitions that it encounters
-      #   because in case of routing patterns, we may start getting statistics prior to registering
-      #   given topic via dynamic routing expansions. In such case we would not have insights
-      #   where they were actually available for us to use.
-      #
-      # @note Memory usage is negligible as long as we can evict expired data. Single metrics set
-      #   for a single partition contains around 4KB of data. This means, that in case of an
-      #   assignment of 1000 partitions, we use around 4MB of space for tracking those metrics.
-      class Tracker
-        include Singleton
-        include Karafka::Core::Helpers::Time
-
-        # Empty hash we want to return in any case where we could not locate appropriate topic
-        # partition statistics.
-        EMPTY_HASH = {}.freeze
-
-        # Empty array to save on memory allocations.
-        EMPTY_ARRAY = [].freeze
-
-        # 5 minutes of cache. We cache last result per consumer group topic partition so we are
-        # not affected by involuntary rebalances during LRJ execution.
-        TTL = 5 * 60 * 1_000
-
-        private_constant :EMPTY_HASH, :EMPTY_ARRAY, :TTL
-
-        class << self
-          extend Forwardable
-
-          def_delegators :instance, :find, :add, :exists?, :clear
-        end
-
-        # Initializes the tracker with empty accumulator
-        def initialize
-          @accu = {}
-          @mutex = Mutex.new
-        end
-
-        # Adds each partition statistics into internal accumulator. Single statistics set may
-        # contain data from multiple topics and their partitions because a single client can
-        # operate on multiple topics and partitions.
-        #
-        # We iterate over those topics and partitions and store topics partitions data only.
-        #
-        # @param consumer_group_id [String] id of the consumer group for which statistics were
-        #   emitted.
-        # @param statistics [Hash] librdkafka enriched statistics
-        def add(consumer_group_id, statistics)
-          @mutex.synchronize do
-            statistics.fetch("topics", EMPTY_HASH).each do |topic_name, t_details|
-              t_details.fetch("partitions", EMPTY_HASH).each do |partition_id, p_details|
-                next unless track?(partition_id, p_details)
-
-                key = "#{consumer_group_id}_#{topic_name}_#{partition_id}"
-                @accu[key] = [monotonic_now, p_details]
-              end
-            end
-
-            evict
-          end
-        end
-
-        # Finds statistics about requested consumer group topic partition
-        #
-        # @param topic [Karafka::Routing::Topic]
-        # @param partition [Integer]
-        # @return [Hash] hash with given topic partition statistics or empty hash if not present
-        #
-        # @note We do not enclose it with a mutex mainly because the only thing that could happen
-        #   here that would be a race-condition is a miss that anyhow we need to support due to
-        #   how librdkafka ships metrics and a potential removal of data on heavily revoked LRJ.
-        def find(topic, partition)
-          key = "#{topic.consumer_group.id}_#{topic.name}_#{partition}"
-          @accu.fetch(key, EMPTY_ARRAY).last || EMPTY_HASH
-        end
-
-        # Clears the tracker
-        def clear
-          @mutex.synchronize { @accu.clear }
-        end
-
-        private
-
-        # Evicts expired data from the cache
-        def evict
-          @accu.delete_if { |_, details| monotonic_now - details.first > TTL }
-        end
-
-        # Should we track given partition
-        #
-        # We do not track stopped partitions and the once we do not work with actively
-        # @param partition_id [String] partition id as a string
-        # @param p_details [Hash] partition statistics details
-        # @return [Boolean] true if we should track given partition
-        def track?(partition_id, p_details)
-          return false if partition_id == "-1"
-
-          fetch_state = p_details.fetch("fetch_state")
-
-          return false if fetch_state == "stopped"
-          return false if fetch_state == "none"
-
-          true
-        end
-      end
-    end
-  end
-end