karafka 2.5.9 → 2.6.0.beta1

data/lib/karafka/processing/consumer_groups/coordinators_buffer.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Coordinators builder used to build coordinators per topic partition
+      #
+      # It provides direct pauses access for revocation
+      #
+      # @note This buffer operates only from the listener loop, thus we do not have to make it
+      #   thread-safe.
+      class CoordinatorsBuffer
+        include Helpers::ConfigImporter.new(
+          coordinator_class: %i[internal processing consumer_groups coordinator_class]
+        )
+
+        # @param topics [Karafka::Routing::Topics]
+        def initialize(topics)
+          @pauses_manager = Connection::PausesManager.new
+          @coordinators = Hash.new { |h, k| h[k] = {} }
+          @topics = topics
+        end
+
+        # @param topic_name [String] topic name
+        # @param partition [Integer] partition number
+        # @return [Karafka::Processing::ConsumerGroups::Coordinator] found or created coordinator
+        def find_or_create(topic_name, partition)
+          @coordinators[topic_name][partition] ||= begin
+            routing_topic = @topics.find(topic_name)
+
+            coordinator_class.new(
+              routing_topic,
+              partition,
+              @pauses_manager.fetch(routing_topic, partition)
+            )
+          end
+        end
+
+        # Resumes processing of partitions for which pause time has ended.
+        # @yieldparam [String] topic name
+        # @yieldparam [Integer] partition number
+        def resume(&)
+          @pauses_manager.resume(&)
+        end
+
+        # @param topic_name [String] topic name
+        # @param partition [Integer] partition number
+        def revoke(topic_name, partition)
+          return unless @coordinators[topic_name].key?(partition)
+
+          # The fact that we delete here does not change the fact that the executor still holds the
+          # reference to this coordinator. We delete it here, as we will no longer process any
+          # new stuff with it and we may need a new coordinator if we regain this partition, but the
+          # coordinator may still be in use
+          @coordinators[topic_name].delete(partition).revoke
+        end
+
+        # Clears coordinators and re-created the pauses manager
+        # This should be used only for critical errors recovery
+        def reset
+          @pauses_manager = Connection::PausesManager.new
+          @coordinators.clear
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/executor.rb

@@ -0,0 +1,220 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Namespace that encapsulates all the logic related to processing data.
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Executors:
+      # - run consumers code (for `#call`) or run given preparation / teardown operations when needed
+      #   from separate threads.
+      # - they re-create consumer instances in case of partitions that were revoked and assigned
+      #   back.
+      #
+      # @note Executors are not removed after partition is revoked. They are not that big and will
+      #   be re-used in case of a re-claim
+      #
+      # @note Since given consumer can run various operations, executor manages that and its
+      #   lifecycle. There are following types of operations with appropriate before/after, etc:
+      #
+      #   - consume - primary operation related to running user consumption code
+      #   - idle - cleanup job that runs on idle runs where no messages would be passed to the end
+      #     user. This is used for complex flows with filters, etc
+      #   - revoked - runs after the partition was revoked
+      #   - shutdown - runs when process is going to shutdown
+      class Executor
+        extend Forwardable
+        include Helpers::ConfigImporter.new(
+          strategy_selector: %i[internal processing consumer_groups strategy_selector],
+          expansions_selector: %i[internal processing consumer_groups expansions_selector]
+        )
+
+        def_delegators :@coordinator, :topic, :partition
+
+        # @return [String] unique id that we use to ensure, that we use for state tracking
+        attr_reader :id
+
+        # @return [String] subscription group id to which a given executor belongs
+        attr_reader :group_id
+
+        # @return [Karafka::Messages::Messages] messages batch
+        attr_reader :messages
+
+        # @return [Karafka::Processing::ConsumerGroups::Coordinator] coordinator for this executor
+        attr_reader :coordinator
+
+        # @param group_id [String] id of the subscription group to which the executor belongs
+        # @param client [Karafka::Connection::Client] kafka client
+        # @param coordinator [Karafka::Processing::ConsumerGroups::Coordinator]
+        def initialize(group_id, client, coordinator)
+          @id = SecureRandom.hex(6)
+          @group_id = group_id
+          @client = client
+          @coordinator = coordinator
+        end
+
+        # Allows us to prepare the consumer in the listener thread prior to the job being send to
+        # be scheduled. It also allows to run some code that is time sensitive and cannot wait in the
+        # queue as it could cause starvation.
+        #
+        # @param messages [Array<Karafka::Messages::Message>]
+        def before_schedule_consume(messages)
+          # Recreate consumer with each batch if persistence is not enabled
+          # We reload the consumers with each batch instead of relying on some external signals
+          # when needed for consistency. That way devs may have it on or off and not in this
+          # middle state, where re-creation of a consumer instance would occur only sometimes
+          @consumer = nil unless topic.consumer_persistence
+
+          # First we build messages batch...
+          consumer.messages = Messages::Builders::Messages.call(
+            messages,
+            topic,
+            partition,
+            # the moment we've received the batch or actually the moment we've enqueued it,
+            # but good enough
+            Time.now
+          )
+
+          consumer.on_before_schedule_consume
+        end
+
+        # Runs setup and warm-up code in the worker prior to running the consumption
+        def before_consume
+          consumer.on_before_consume
+        end
+
+        # Runs the wrap/around execution context appropriate for a given action
+        # @param action [Symbol] action execution wrapped with our block
+        def wrap(action, &)
+          consumer.on_wrap(action, &)
+        end
+
+        # Runs consumer data processing against given batch and handles failures and errors.
+        def consume
+          # We run the consumer client logic...
+          consumer.on_consume
+        end
+
+        # Runs consumer after consumption code
+        def after_consume
+          consumer.on_after_consume
+        end
+
+        # Runs the code needed before idle work is scheduled
+        def before_schedule_idle
+          consumer.on_before_schedule_idle
+        end
+
+        # Runs consumer idle operations
+        # This may include house-keeping or other state management changes that can occur but that
+        # not mean there are any new messages available for the end user to process
+        def idle
+          consumer.on_idle
+        end
+
+        # Runs the code needed before eofed work is scheduled
+        def before_schedule_eofed
+          consumer.on_before_schedule_eofed
+        end
+
+        # Runs consumed eofed operation.
+        # This may run even when there were no messages received prior. This will however not
+        # run when eof is received together with messages as in such case `#consume` will run
+        def eofed
+          consumer.on_eofed
+        end
+
+        # Runs code needed before revoked job is scheduled
+        def before_schedule_revoked
+          consumer.on_before_schedule_revoked if @consumer
+        end
+
+        # Runs the controller `#revoked` method that should be triggered when a given consumer is
+        # no longer needed due to partitions reassignment.
+        #
+        # @note Clearing the consumer will ensure, that if we get the partition back, it will be
+        #   handled with a consumer with a clean state.
+        #
+        # @note We run it only when consumer was present, because presence indicates, that at least
+        #   a single message has been consumed.
+        #
+        # @note We do not reset the consumer but we indicate need for recreation instead, because
+        #   after the revocation, there still may be `#after_consume` running that needs a given
+        #   consumer instance.
+        def revoked
+          consumer.on_revoked if @consumer
+        end
+
+        # Runs code needed before shutdown job is scheduled
+        def before_schedule_shutdown
+          consumer.on_before_schedule_shutdown if @consumer
+        end
+
+        # Runs the controller `#shutdown` method that should be triggered when a given consumer is
+        # no longer needed as we're closing the process.
+        #
+        # @note While we do not need to clear the consumer here, it's a good habit to clean after
+        #   work is done.
+        def shutdown
+          # There is a case, where the consumer no longer exists because it was revoked, in case like
+          # that we do not build a new instance and shutdown should not be triggered.
+          consumer.on_shutdown if @consumer
+        end
+
+        private
+
+        # @return [Object] cached consumer instance
+        def consumer
+          @consumer ||= begin
+            topic = @coordinator.topic
+
+            strategy = strategy_selector.find(topic)
+
+            consumer = topic.consumer_class.new
+            # We use singleton class as the same consumer class may be used to process different
+            # topics with different settings
+            consumer.singleton_class.include(strategy)
+
+            # Specific features may expand consumer API beyond the injected strategy. The difference
+            # here is that strategy impacts the flow of states while extra APIs just provide some
+            # extra methods with informations, etc but do no deviate the flow behavior
+            expansions = expansions_selector.find(topic)
+            expansions.each { |expansion| consumer.singleton_class.include(expansion) }
+
+            consumer.client = @client
+            consumer.coordinator = @coordinator
+            # We assign producer only when not available already. It may already be available if
+            # user redefined the `#producer` method for example. This can be useful for example when
+            # having a multi-cluster setup and using a totally custom producer
+            consumer.producer ||= Karafka::App.producer
+            # Since we have some message-less flows (idle, etc), we initialize consumer with empty
+            # messages set. In production we have persistent consumers, so this is not a performance
+            # overhead as this will happen only once per consumer lifetime
+            consumer.messages = empty_messages
+
+            # Run the post-initialization hook for users that need to run some actions when consumer
+            # is built and ready (all basic state and info).
+            # Users should **not** overwrite the `#initialize` because it won't have all the needed
+            # data assigned yet.
+            consumer.on_initialized
+
+            consumer
+          end
+        end
+
+        # Initializes the messages set in case given operation would happen before any processing
+        # This prevents us from having no messages object at all as the messages object and
+        # its metadata may be used for statistics
+        def empty_messages
+          Messages::Builders::Messages.call(
+            [],
+            topic,
+            partition,
+            Time.now
+          )
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/executors_buffer.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Buffer for executors of a given subscription group. It wraps around the concept of building
+      # and caching them, so we can re-use them instead of creating new each time.
+      class ExecutorsBuffer
+        include Helpers::ConfigImporter.new(
+          executor_class: %i[internal processing consumer_groups executor_class]
+        )
+
+        # @param client [Connection::Client]
+        # @param subscription_group [Routing::SubscriptionGroup]
+        # @return [ExecutorsBuffer]
+        def initialize(client, subscription_group)
+          @subscription_group = subscription_group
+          @client = client
+          # We need two layers here to keep track of topics, partitions and processing groups
+          @buffer = Hash.new { |h, k| h[k] = Hash.new { |h2, k2| h2[k2] = {} } }
+        end
+
+        # Finds or creates an executor based on the provided details
+        #
+        # @param topic [String] topic name
+        # @param partition [Integer] partition number
+        # @param parallel_key [String] parallel group key
+        # @param coordinator [Karafka::Processing::ConsumerGroups::Coordinator]
+        # @return [Executor, Pro::Processing::ConsumerGroups::Executor] consumer executor
+        def find_or_create(topic, partition, parallel_key, coordinator)
+          @buffer[topic][partition][parallel_key] ||= executor_class.new(
+            @subscription_group.id,
+            @client,
+            coordinator
+          )
+        end
+
+        # Finds all existing executors for given topic partition or creates one for it
+        # @param topic [String] topic name
+        # @param partition [Integer] partition number
+        # @param coordinator [Karafka::Processing::ConsumerGroups::Coordinator]
+        # @return [Array<Executor, Pro::Processing::ConsumerGroups::Executor>]
+        def find_all_or_create(topic, partition, coordinator)
+          existing = find_all(topic, partition)
+
+          return existing unless existing.empty?
+
+          [find_or_create(topic, partition, 0, coordinator)]
+        end
+
+        # Revokes executors of a given topic partition, so they won't be used anymore for incoming
+        # messages
+        #
+        # @param topic [String] topic name
+        # @param partition [Integer] partition number
+        def revoke(topic, partition)
+          @buffer[topic][partition].clear
+        end
+
+        # Finds all the executors available for a given topic partition
+        #
+        # @param topic [String] topic name
+        # @param partition [Integer] partition number
+        # @return [Array<Executor, Pro::Processing::ConsumerGroups::Executor>] executors in use for this
+        #   topic + partition
+        def find_all(topic, partition)
+          @buffer[topic][partition].values
+        end
+
+        # Iterates over all available executors and yields them together with topic and partition
+        # info
+        # @yieldparam [Routing::Topic] karafka routing topic object
+        # @yieldparam [Integer] partition number
+        # @yieldparam [Executor, Pro::Processing::ConsumerGroups::Executor] given executor
+        def each
+          @buffer.each_value do |partitions|
+            partitions.each_value do |executors|
+              executors.each_value do |executor|
+                yield(executor)
+              end
+            end
+          end
+        end
+
+        # Clears the executors buffer. Useful for critical errors recovery.
+        def clear
+          @buffer.clear
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/expansions_selector.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Selector of appropriate topic setup based features enhancements.
+      #
+      # Those expansions to the consumer API are NOT about the flow of processing. For this we have
+      # strategies. Those are suppose to provide certain extra APIs that user can use to get some
+      # extra non-flow related functionalities.
+      class ExpansionsSelector
+        # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
+        #   expansions
+        # @return [Array<Module>] modules with proper expansions we're suppose to use to enhance the
+        #   consumer
+        def find(topic)
+          expansions = []
+          expansions << Processing::ConsumerGroups::InlineInsights::Consumer if topic.inline_insights?
+          expansions
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/inline_insights/consumer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Namespace of the Inline Insights feature "non routing" related components
+      #
+      # @note We use both `#insights` because it is the feature name but also `#statistics` to make
+      #   it consistent with the fact that we publish and operate on statistics. User can pick
+      #   whichever name they prefer.
+      module InlineInsights
+        # Module that adds extra methods to the consumer that allow us to fetch the insights
+        module Consumer
+          # @return [Hash] empty hash or hash with given partition insights if already present
+          # @note We cache insights on the consumer, as in some scenarios we may no longer have them
+          #   inside the Tracker, for example under involuntary revocation, incoming statistics may
+          #   no longer have lost partition insights. Since we want to be consistent during single
+          #   batch operations, we want to ensure, that if we have insights they are available
+          #   throughout the whole processing.
+          def insights
+            insights = Tracker.find(topic, partition)
+
+            # If we no longer have new insights but we still have them locally, we can use them
+            return @insights if @insights && insights.empty?
+            # If insights are still the same, we can use them
+            return @insights if @insights.equal?(insights)
+
+            # If we've received new insights that are not empty, we can cache them
+            @insights = insights
+          end
+
+          # @return [Boolean] true if there are insights to work with, otherwise false
+          def insights?
+            !insights.empty?
+          end
+
+          alias_method :statistics, :insights
+          alias_method :statistics?, :insights?
+          alias_method :inline_insights, :insights
+          alias_method :inline_insights?, :insights?
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/inline_insights/listener.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      module InlineInsights
+        # Listener that adds statistics to our inline tracker
+        class Listener
+          # Adds statistics to the tracker
+          # @param event [Karafka::Core::Monitoring::Event] event with statistics
+          def on_statistics_emitted(event)
+            Tracker.add(
+              event[:group_id],
+              event[:statistics]
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/inline_insights/tracker.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      module InlineInsights
+        # Object used to track statistics coming from librdkafka in a way that can be accessible by
+        # the consumers
+        #
+        # We use a single tracker because we do not need state management here as our consumer groups
+        # clients identified by statistics name value are unique. On top of that, having a per
+        # process one that is a singleton allows us to use tracker easily also from other places like
+        # filtering API etc.
+        #
+        # @note We include cache of 5 minutes for revoked partitions to compensate for cases where
+        #   when using LRJ a lost partition data would not be present anymore, however we would still
+        #   be in the processing phase. Since those metrics are published with each `poll`, regular
+        #   processing is not a subject of this issue. For LRJ we keep the reference. The only case
+        #   where this could be switched midway is when LRJ is running for an extended period of time
+        #   after the involuntary revocation. Having a time based cache instead of tracking
+        #   simplifies the design as we do not have to deal with state tracking, especially since
+        #   we would have to track also operations running in a revoked state.
+        #
+        # @note This tracker keeps in memory data about all topics and partitions that it encounters
+        #   because in case of routing patterns, we may start getting statistics prior to registering
+        #   given topic via dynamic routing expansions. In such case we would not have insights
+        #   where they were actually available for us to use.
+        #
+        # @note Memory usage is negligible as long as we can evict expired data. Single metrics set
+        #   for a single partition contains around 4KB of data. This means, that in case of an
+        #   assignment of 1000 partitions, we use around 4MB of space for tracking those metrics.
+        class Tracker
+          include Singleton
+          include Karafka::Core::Helpers::Time
+
+          # Empty hash we want to return in any case where we could not locate appropriate topic
+          # partition statistics.
+          EMPTY_HASH = {}.freeze
+
+          # Empty array to save on memory allocations.
+          EMPTY_ARRAY = [].freeze
+
+          # 5 minutes of cache. We cache last result per consumer group topic partition so we are
+          # not affected by involuntary rebalances during LRJ execution.
+          TTL = 5 * 60 * 1_000
+
+          private_constant :EMPTY_HASH, :EMPTY_ARRAY, :TTL
+
+          class << self
+            extend Forwardable
+
+            def_delegators :instance, :find, :add, :exists?, :clear
+          end
+
+          # Initializes the tracker with empty accumulator
+          def initialize
+            @accu = {}
+            @mutex = Mutex.new
+          end
+
+          # Adds each partition statistics into internal accumulator. Single statistics set may
+          # contain data from multiple topics and their partitions because a single client can
+          # operate on multiple topics and partitions.
+          #
+          # We iterate over those topics and partitions and store topics partitions data only.
+          #
+          # @param group_id [String] id of the group for which statistics were emitted.
+          # @param statistics [Hash] librdkafka enriched statistics
+          def add(group_id, statistics)
+            @mutex.synchronize do
+              statistics.fetch("topics", EMPTY_HASH).each do |topic_name, t_details|
+                t_details.fetch("partitions", EMPTY_HASH).each do |partition_id, p_details|
+                  next unless track?(partition_id, p_details)
+
+                  key = "#{group_id}_#{topic_name}_#{partition_id}"
+                  @accu[key] = [monotonic_now, p_details]
+                end
+              end
+
+              evict
+            end
+          end
+
+          # Finds statistics about requested consumer group topic partition
+          #
+          # @param topic [Karafka::Routing::Topic]
+          # @param partition [Integer]
+          # @return [Hash] hash with given topic partition statistics or empty hash if not present
+          #
+          # @note We do not enclose it with a mutex mainly because the only thing that could happen
+          #   here that would be a race-condition is a miss that anyhow we need to support due to
+          #   how librdkafka ships metrics and a potential removal of data on heavily revoked LRJ.
+          def find(topic, partition)
+            key = "#{topic.group.id}_#{topic.name}_#{partition}"
+            @accu.fetch(key, EMPTY_ARRAY).last || EMPTY_HASH
+          end
+
+          # Clears the tracker
+          def clear
+            @mutex.synchronize { @accu.clear }
+          end
+
+          private
+
+          # Evicts expired data from the cache
+          def evict
+            @accu.delete_if { |_, details| monotonic_now - details.first > TTL }
+          end
+
+          # Should we track given partition
+          #
+          # We do not track stopped partitions and the once we do not work with actively
+          # @param partition_id [String] partition id as a string
+          # @param p_details [Hash] partition statistics details
+          # @return [Boolean] true if we should track given partition
+          def track?(partition_id, p_details)
+            return false if partition_id == "-1"
+
+            fetch_state = p_details.fetch("fetch_state")
+
+            return false if fetch_state == "stopped"
+            return false if fetch_state == "none"
+
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/consumer_groups/jobs/consume.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    # Consumer-group-specific processing components (driven by rebalance callbacks and partition
+    # ticks). Parallel `ShareGroups` will live next to this namespace once KIP-932 lands.
+    module ConsumerGroups
+      # Consumer-group-specific job types
+      module Jobs
+        # The main job type. It runs the executor that triggers given topic partition messages
+        # processing in an underlying consumer instance.
+        class Consume < ::Karafka::Processing::Jobs::Base
+          # @return [Array<Rdkafka::Consumer::Message>] array with messages
+          attr_reader :messages
+
+          self.action = :consume
+
+          # @param executor [Karafka::Processing::ConsumerGroups::Executor] executor that is suppose to run a given
+          #   job
+          # @param messages [Karafka::Messages::Messages] karafka messages batch
+          # @return [Consume]
+          def initialize(executor, messages)
+            @executor = executor
+            @messages = messages
+            super()
+          end
+
+          # Runs all the preparation code on the executor that needs to happen before the job is
+          # scheduled.
+          def before_schedule
+            executor.before_schedule_consume(@messages)
+          end
+
+          # Runs the before consumption preparations on the executor
+          def before_call
+            executor.before_consume
+          end
+
+          # Runs the given executor
+          def call
+            executor.consume
+          end
+
+          # Runs any error handling and other post-consumption stuff on the executor
+          def after_call
+            executor.after_consume
+          end
+        end
+      end
+    end
+  end
+end