karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/connection/rebalance_manager.rb

@@ -1,116 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Connection
-    # Manager for tracking changes in the partitions assignment after the assignment is done and
-    # for ensuring, that proper buffer related operations that may be impacted by the rebalance
-    # state are applied.
-    #
-    # We need tracking of those to clean up consumers that will no longer process given partitions
-    # as they were taken away.
-    #
-    # @note Since this does not happen really often, we try to stick with same objects for the
-    #   empty states most of the time, so we don't create many objects during the manager life
-    #
-    # @note Internally in the rebalance manager we have a notion of lost partitions. Partitions
-    #   that are lost, are those that got revoked but did not get re-assigned back. We do not
-    #   expose this concept outside and we normalize to have them revoked, as it is irrelevant
-    #   from the rest of the code perspective as only those that are lost are truly revoked.
-    #
-    # @note For cooperative-sticky `#assigned_partitions` holds only the recently assigned
-    #   partitions, not all the partitions that are owned
-    #
-    # @note We have to have the `subscription_group` reference because we have a global pipeline
-    #   for notifications and we need to make sure we track changes only for things that are of
-    #   relevance to our subscription group
-    class RebalanceManager
-      # Empty array for internal usage not to create new objects
-      EMPTY_ARRAY = [].freeze
-
-      attr_reader :assigned_partitions, :revoked_partitions
-
-      private_constant :EMPTY_ARRAY
-
-      # @param subscription_group_id [String] subscription group id
-      # @param buffer [Karafka::Connection::RawMessagesBuffer]
-      # @return [RebalanceManager]
-      def initialize(subscription_group_id, buffer)
-        @assigned_partitions = {}
-        @revoked_partitions = {}
-        @changed = false
-        @active = false
-        @subscription_group_id = subscription_group_id
-        @buffer = buffer
-
-        # Connects itself to the instrumentation pipeline so rebalances can be tracked
-        Karafka.monitor.subscribe(self)
-      end
-
-      # Resets the rebalance manager state
-      # This needs to be done before each polling loop as during the polling, the state may be
-      # changed
-      def clear
-        @assigned_partitions.clear
-        @revoked_partitions.clear
-        @changed = false
-      end
-
-      # @return [Boolean] indicates a state change in the partitions assignment
-      def changed?
-        @changed
-      end
-
-      # @return [Boolean] true if there was at least one rebalance
-      # @note This method is needed to make sure that when using cooperative-sticky, we do not
-      #   close until first rebalance. Otherwise librdkafka may crash.
-      # @see https://github.com/confluentinc/librdkafka/issues/4312
-      def active?
-        @active
-      end
-
-      # Callback that kicks in inside of rdkafka, when new partitions were assigned.
-      #
-      # @private
-      # @param event [Karafka::Core::Monitoring::Event]
-      def on_rebalance_partitions_assigned(event)
-        # Apply changes only for our subscription group
-        return unless event[:subscription_group_id] == @subscription_group_id
-
-        @active = true
-        @assigned_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
-        @changed = true
-      end
-
-      # Callback that kicks in inside of rdkafka, when partitions were revoked.
-      #
-      # @private
-      # @param event [Karafka::Core::Monitoring::Event]
-      def on_rebalance_partitions_revoked(event)
-        # Apply changes only for our subscription group
-        return unless event[:subscription_group_id] == @subscription_group_id
-
-        @active = true
-        @revoked_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
-        @changed = true
-
-        remove_revoked_and_duplicated_messages
-      end
-
-      private
-
-      # We may have a case where in the middle of data polling, we've lost a partition.
-      # In a case like this we should remove all the pre-buffered messages from list partitions as
-      # we are no longer responsible in a given process for processing those messages and they
-      # should have been picked up by a different process.
-      def remove_revoked_and_duplicated_messages
-        @revoked_partitions.each do |topic, partitions|
-          partitions.each do |partition|
-            @buffer.delete(topic, partition)
-          end
-        end
-
-        @buffer.uniq!
-      end
-    end
-  end
-end

data/lib/karafka/instrumentation/callbacks/error.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Instrumentation
-    # Callbacks used to transport things from rdkafka
-    module Callbacks
-      # Callback that kicks in when consumer error occurs and is published in a background thread
-      class Error
-        include Helpers::ConfigImporter.new(
-          monitor: %i[monitor]
-        )
-
-        # @param subscription_group_id [String]
-        # @param consumer_group_id [String]
-        # @param client_name [String] rdkafka client name
-        def initialize(subscription_group_id, consumer_group_id, client_name)
-          @subscription_group_id = subscription_group_id
-          @consumer_group_id = consumer_group_id
-          @client_name = client_name
-        end
-
-        # Runs the instrumentation monitor with error
-        # @param client_name [String] rdkafka client name
-        # @param error [Rdkafka::Error] error that occurred
-        # @note It will only instrument on errors of the client of our consumer
-        def call(client_name, error)
-          # Emit only errors related to our client
-          # Same as with statistics (mor explanation there)
-          return unless @client_name == client_name
-
-          monitor.instrument(
-            "error.occurred",
-            caller: self,
-            subscription_group_id: @subscription_group_id,
-            consumer_group_id: @consumer_group_id,
-            type: "librdkafka.error",
-            error: error
-          )
-        rescue => e
-          monitor.instrument(
-            "error.occurred",
-            caller: self,
-            subscription_group_id: @subscription_group_id,
-            consumer_group_id: @consumer_group_id,
-            type: "callbacks.error.error",
-            error: e
-          )
-        end
-      end
-    end
-  end
-end

data/lib/karafka/instrumentation/callbacks/rebalance.rb

@@ -1,84 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Instrumentation
-    module Callbacks
-      # Callback that connects to the librdkafka rebalance callback and converts those events into
-      # our internal events
-      class Rebalance
-        include Helpers::ConfigImporter.new(
-          monitor: %i[monitor]
-        )
-
-        # @param subscription_group [Karafka::Routes::SubscriptionGroup] subscription group for
-        #   which we want to manage rebalances
-        # @param client_id [String] id of the client managing this rebalance
-        def initialize(subscription_group, client_id)
-          @subscription_group = subscription_group
-          @client_id = client_id
-        end
-
-        # Publishes an event that partitions are going to be revoked.
-        # At this stage we can still commit offsets, etc.
-        #
-        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
-        def on_partitions_revoke(tpl)
-          instrument("partitions_revoke", tpl)
-        end
-
-        # Publishes an event that partitions are going to be assigned
-        #
-        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
-        def on_partitions_assign(tpl)
-          instrument("partitions_assign", tpl)
-        end
-
-        # Publishes an event that partitions were revoked. This is after we've lost them, so no
-        # option to commit offsets.
-        #
-        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
-        def on_partitions_revoked(tpl)
-          instrument("partitions_revoked", tpl)
-        end
-
-        # Publishes an event that partitions were assigned.
-        #
-        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
-        def on_partitions_assigned(tpl)
-          instrument("partitions_assigned", tpl)
-        end
-
-        private
-
-        # Publishes info that a rebalance event of a given type has happened
-        #
-        # @param name [String] name of the event
-        # @param tpl [Rdkafka::Consumer::TopicPartitionList]
-        def instrument(name, tpl)
-          monitor.instrument(
-            "rebalance.#{name}",
-            caller: self,
-            # We keep the id references here for backwards compatibility as some of the monitors
-            # may use the id references
-            subscription_group_id: @subscription_group.id,
-            subscription_group: @subscription_group,
-            consumer_group_id: @subscription_group.consumer_group.id,
-            consumer_group: @subscription_group.consumer_group,
-            client_id: @client_id,
-            tpl: tpl
-          )
-        rescue => e
-          monitor.instrument(
-            "error.occurred",
-            caller: self,
-            subscription_group_id: @subscription_group.id,
-            consumer_group_id: @subscription_group.consumer_group.id,
-            type: "callbacks.rebalance.#{name}.error",
-            client_id: @client_id,
-            error: e
-          )
-        end
-      end
-    end
-  end
-end

data/lib/karafka/instrumentation/callbacks/statistics.rb

@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-module Karafka
-  module Instrumentation
-    module Callbacks
-      # Statistics callback handler
-      # @see `WaterDrop::Instrumentation::Callbacks::Statistics` for details on why we decorate
-      #   those statistics
-      class Statistics
-        include Helpers::ConfigImporter.new(
-          monitor: %i[monitor]
-        )
-
-        # @param subscription_group_id [String]
-        # @param consumer_group_id [String]
-        # @param client_name [String] rdkafka client name
-        def initialize(subscription_group_id, consumer_group_id, client_name)
-          @subscription_group_id = subscription_group_id
-          @consumer_group_id = consumer_group_id
-          @client_name = client_name
-          @statistics_decorator = Karafka::Core::Monitoring::StatisticsDecorator.new
-        end
-
-        # Emits decorated statistics to the monitor
-        # @param statistics [Hash] rdkafka statistics
-        def call(statistics)
-          # Emit only statistics related to our client
-          # rdkafka does not have per-instance statistics hook, thus we need to make sure that we
-          # emit only stats that are related to current producer. Otherwise we would emit all of
-          # all the time.
-          return unless @client_name == statistics["name"]
-
-          monitor.instrument(
-            "statistics.emitted",
-            subscription_group_id: @subscription_group_id,
-            consumer_group_id: @consumer_group_id,
-            statistics: @statistics_decorator.call(statistics)
-          )
-        # We need to catch and handle any potential errors coming from the instrumentation pipeline
-        # as otherwise, in case of statistics which run in the main librdkafka thread, any crash
-        # will hang the whole process.
-        rescue => e
-          monitor.instrument(
-            "error.occurred",
-            caller: self,
-            subscription_group_id: @subscription_group_id,
-            consumer_group_id: @consumer_group_id,
-            type: "callbacks.statistics.error",
-            error: e
-          )
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/adaptive_iterator/consumer.rb

@@ -1,79 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Namespace for adaptive iterator consumer components
-      module AdaptiveIterator
-        # Consumer enhancements needed to wrap the batch iterator for adaptive iterating
-        # It automatically marks as consumed, ensures that we do not reach `max.poll.interval.ms`
-        # and does other stuff to simplify user per-message processing
-        module Consumer
-          # @param args [Array] anything accepted by `Karafka::Messages::Messages#each`
-          def each(*args)
-            adi_config = topic.adaptive_iterator
-
-            tracker = Tracker.new(
-              adi_config.safety_margin,
-              coordinator.last_polled_at,
-              topic.subscription_group.kafka.fetch(:"max.poll.interval.ms")
-            )
-
-            messages.each(*args) do |message|
-              # Always stop if we've lost the assignment
-              return if revoked?
-              # No automatic marking risk when mom is enabled so we can fast stop
-              return if Karafka::App.done? && topic.manual_offset_management?
-
-              # Seek request on done will allow us to stop without marking the offset when user had
-              # the automatic offset marking. This should not be a big network traffic issue for
-              # the end user as we're stopping anyhow but should improve shutdown time
-              if tracker.enough? || Karafka::App.done?
-                # Enough means we no longer have time to process more data without polling as we
-                # risk reaching max poll interval. Instead we seek and we will poll again soon.
-                seek(message.offset, reset_offset: true)
-
-                return
-              end
-
-              tracker.track { yield(message) }
-
-              # Clean if this is what user configured
-              message.clean! if adi_config.clean_after_yielding?
-
-              public_send(adi_config.marking_method, message)
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/adaptive_iterator/tracker.rb

@@ -1,92 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      module AdaptiveIterator
-        # Tracker is responsible for monitoring the processing of messages within the poll
-        # interval limitation.
-        # It ensures that the consumer does not exceed the maximum poll interval by tracking the
-        # processing cost and determining when to halt further processing (if needed).
-        class Tracker
-          include Karafka::Core::Helpers::Time
-
-          # Initializes a new Tracker instance.
-          #
-          # @param safety_margin [Float] The safety margin percentage (0-100) to leave as a buffer.
-          # @param last_polled_at [Float] The timestamp of the last polling in milliseconds.
-          # @param max_poll_interval_ms [Integer] The maximum poll interval time in milliseconds.
-          def initialize(
-            safety_margin,
-            last_polled_at,
-            max_poll_interval_ms
-          )
-            @safety_margin = safety_margin / 100.0 # Convert percentage to decimal
-            @last_polled_at = last_polled_at
-            @max_processing_cost = 0
-            @max_poll_interval_ms = max_poll_interval_ms
-          end
-
-          # Tracks the processing time of a block and updates the maximum processing cost.
-          #
-          # @yield Executes the block, measuring the time taken for processing.
-          def track
-            before = monotonic_now
-
-            yield
-
-            time_taken = monotonic_now - before
-
-            return unless time_taken > @max_processing_cost
-
-            @max_processing_cost = time_taken
-          end
-
-          # Determines if there is enough time left to process more messages without exceeding the
-          # maximum poll interval, considering both the safety margin and adaptive margin.
-          #
-          # @return [Boolean] Returns true if it is time to stop processing. False otherwise.
-          def enough?
-            elapsed_time_ms = monotonic_now - @last_polled_at
-            remaining_time_ms = @max_poll_interval_ms - elapsed_time_ms
-
-            safety_margin_ms = @max_poll_interval_ms * @safety_margin
-
-            return true if remaining_time_ms <= safety_margin_ms
-            return true if remaining_time_ms - @max_processing_cost <= safety_margin_ms
-
-            false
-          end
-        end
-      end
-    end
-  end
-end

data/lib/karafka/pro/processing/collapser.rb

@@ -1,79 +0,0 @@
-# frozen_string_literal: true
-
-# Karafka Pro - Source Available Commercial Software
-# Copyright (c) 2017-present Maciej Mensfeld. All rights reserved.
-#
-# This software is NOT open source. It is source-available commercial software
-# requiring a paid license for use. It is NOT covered by LGPL.
-#
-# The author retains all right, title, and interest in this software,
-# including all copyrights, patents, and other intellectual property rights.
-# No patent rights are granted under this license.
-#
-# PROHIBITED:
-# - Use without a valid commercial license
-# - Redistribution, modification, or derivative works without authorization
-# - Reverse engineering, decompilation, or disassembly of this software
-# - Use as training data for AI/ML models or inclusion in datasets
-# - Scraping, crawling, or automated collection for any purpose
-#
-# PERMITTED:
-# - Reading, referencing, and linking for personal or commercial use
-# - Runtime retrieval by AI assistants, coding agents, and RAG systems
-#   for the purpose of providing contextual help to Karafka users
-#
-# Receipt, viewing, or possession of this software does not convey or
-# imply any license or right beyond those expressly stated above.
-#
-# License: https://karafka.io/docs/Pro-License-Comm/
-# Contact: contact@karafka.io
-
-module Karafka
-  module Pro
-    module Processing
-      # Manages the collapse of virtual partitions
-      # Since any non-virtual partition is actually a virtual partition of size one, we can use
-      # it in a generic manner without having to distinguish between those cases.
-      #
-      # We need to have notion of the offset until we want to collapse because upon pause and retry
-      # rdkafka may purge the buffer. This means, that we may end up with smaller or bigger
-      # (different) dataset and without tracking the end of collapse, there would be a chance for
-      # things to flicker. Tracking allows us to ensure, that collapse is happening until all the
-      # messages from the corrupted batch are processed.
-      class Collapser
-        # When initialized, nothing is collapsed
-        def initialize
-          @collapsed = false
-          @until_offset = -1
-          @mutex = Mutex.new
-        end
-
-        # @return [Boolean] Should we collapse into a single consumer
-        def collapsed?
-          @collapsed
-        end
-
-        # Collapse until given offset. Until given offset is encountered or offset bigger than that
-        # we keep collapsing.
-        # @param offset [Integer] offset until which we keep the collapse
-        def collapse_until!(offset)
-          @mutex.synchronize do
-            # We check it here in case after a pause and re-fetch we would get less messages and
-            # one of them would cause an error. We do not want to overwrite the offset here unless
-            # it is bigger.
-            @until_offset = offset if offset > @until_offset
-          end
-        end
-
-        # Sets the collapse state based on the first collective offset that we are going to process
-        # and makes the decision whether or not we need to still keep the collapse.
-        # @param first_offset [Integer] first offset from a collective batch
-        def refresh!(first_offset)
-          @mutex.synchronize do
-            @collapsed = first_offset < @until_offset
-          end
-        end
-      end
-    end
-  end
-end