karafka 2.2.13 → 2.3.0.alpha1

data/lib/karafka/pro/connection/manager.rb

@@ -0,0 +1,300 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Connection
+      # Manager that can handle working with multiplexed connections.
+      #
+      # This manager takes into consideration the number of partitions assigned to the topics and
+      # does its best to balance. Additional connections may not always be utilized because
+      # alongside of them, other processes may "hijack" the assignment. In such cases those extra
+      # empty connections will be turned off after a while.
+      #
+      # @note Manager operations relate to consumer groups and not subscription groups. Since
+      #   cluster operations can cause consumer group wide effects, we always apply only one
+      #   change on a consumer group.
+      #
+      # @note Since we collect statistical data from listeners and this happens in a background
+      #   thread, we need to make sure we lock not to have race conditions with expired data
+      #   eviction.
+      class Manager < Karafka::Connection::Manager
+        include Core::Helpers::Time
+
+        # How long should we keep stale stats before evicting them completely
+        EVICTION_DELAY = 5 * 60 * 1_000
+
+        private_constant :EVICTION_DELAY
+
+        # How long should we wait after a rebalance before doing anything on a consumer group
+        #
+        # @param scale_delay [Integer] How long should we wait before making any changes. Any
+        #   change related to this consumer group will postpone the scaling operations. This is
+        #   done that way to prevent too many friction in the cluster. It is 1 minute by default
+        def initialize(scale_delay = 60 * 1_000)
+          super()
+          @scale_delay = scale_delay
+          @mutex = Mutex.new
+          @changes = Hash.new do |h, k|
+            h[k] = {
+              state: '',
+              join_state: '',
+              state_age: 0,
+              state_age_sync: monotonic_now,
+              changed_at: monotonic_now
+            }
+          end
+        end
+
+        # Registers listeners and starts the scaling procedures
+        #
+        # When using dynamic multiplexing, it will start the absolute minimum of connections for
+        # subscription group available.
+        #
+        # @param listeners [Connection::ListenersBatch]
+        def register(listeners)
+          @listeners = listeners
+
+          in_sg_families do |first_subscription_group, sg_listeners|
+            multiplexing = first_subscription_group.multiplexing
+
+            if multiplexing.active? && multiplexing.dynamic?
+              # Start as many boot listeners as user wants. If not configured, starts half of max.
+              sg_listeners.first(multiplexing.boot).each(&:start!)
+            else
+              sg_listeners.each(&:start!)
+            end
+          end
+        end
+
+        # Collects data from the statistics about given subscription group. This is used to ensure
+        # that we do not rescale short after rebalances, deployments, etc.
+        # @param subscription_group_id [String] id of the subscription group for which statistics
+        #   were emitted
+        # @param statistics [Hash] emitted statistics
+        #
+        # @note Please note that while we collect here per subscription group, we use those metrics
+        #   collectively on a whole consumer group. This reduces the friction.
+        def notice(subscription_group_id, statistics)
+          @mutex.synchronize do
+            times = []
+            # stateage is in microseconds
+            # We monitor broker changes to make sure we do not introduce extra friction
+            times << statistics['brokers'].values.map { |stats| stats['stateage'] }.min / 1_000
+            times << statistics['cgrp']['rebalance_age']
+            times << statistics['cgrp']['stateage']
+
+            # Keep the previous change age for changes that were triggered by us
+            previous_changed_at = @changes[subscription_group_id][:changed_at]
+
+            @changes[subscription_group_id] = {
+              state_age: times.min,
+              changed_at: previous_changed_at,
+              join_state: statistics['cgrp']['join_state'],
+              state: statistics['cgrp']['state'],
+              state_age_sync: monotonic_now
+            }
+          end
+        end
+
+        # Shuts down all the listeners when it is time (including moving to quiet) or rescales
+        # when it is needed
+        def control
+          Karafka::App.done? ? shutdown : rescale
+        end
+
+        private
+
+        # Handles the shutdown and quiet flows
+        def shutdown
+          active_listeners = @listeners.active
+
+          # When we are done processing immediately quiet all the listeners so they do not pick up
+          # new work to do
+          once(:quiet!) { active_listeners.each(&:quiet!) }
+
+          # If we are in the process of moving to quiet state, we need to check it.
+          if Karafka::App.quieting? && active_listeners.all?(&:quiet?)
+            once(:quieted!) { Karafka::App.quieted! }
+          end
+
+          return if Karafka::App.quiet?
+
+          # Since separate subscription groups are subscribed to different topics, there is no risk
+          # in shutting them down independently even if they operate in the same subscription group
+          in_sg_families do |first_subscription_group, sg_listeners|
+            active_sg_listeners = sg_listeners.select(&:active?)
+
+            # Do nothing until all listeners from the same consumer group are quiet. Otherwise we
+            # could have problems with in-flight rebalances during shutdown
+            next unless active_sg_listeners.all?(&:quiet?)
+
+            # Do not stop the same family twice
+            once(:stop!, first_subscription_group.name) { active_sg_listeners.each(&:stop!) }
+          end
+
+          return unless @listeners.active.all?(&:stopped?)
+
+          # All listeners including pending need to be moved at the end to stopped state for
+          # the whole server to stop
+          once(:stop!) { @listeners.each(&:stopped!) }
+        end
+
+        # Handles two scenarios:
+        #   - Selects subscriptions that could benefit from having more parallel connections
+        #     to kafka and then upscales them
+        #   - Selects subscriptions that are idle (have nothing subscribed to them) and then shuts
+        #     them down
+        #
+        # We always run scaling down and up because it may be applicable to different CGs
+        def rescale
+          evict
+
+          scale_down
+          scale_up
+        end
+
+        # Checks for connections without any assignments and scales them down.
+        # Does that only for dynamically multiplexed subscription groups
+        def scale_down
+          sgs_in_use = Karafka::App.assignments.keys.map(&:subscription_group).uniq
+
+          # Select connections for scaling down
+          in_sg_families do |first_subscription_group, sg_listeners|
+            next unless stable?(sg_listeners)
+
+            multiplexing = first_subscription_group.multiplexing
+
+            next unless multiplexing.active?
+            next unless multiplexing.dynamic?
+
+            # If we cannot downscale, do not
+            next if sg_listeners.count(&:active?) <= multiplexing.min
+
+            sg_listeners.each do |sg_listener|
+              # Do not stop connections with subscriptions
+              next if sgs_in_use.include?(sg_listener.subscription_group)
+              # Skip listeners that are already in standby
+              next unless sg_listener.active?
+
+              touch(sg_listener.subscription_group.id)
+
+              # Shut down not used connection
+              sg_listener.stop!
+
+              break
+            end
+          end
+        end
+
+        # Checks if we have space to scale and if there are any assignments with multiple topics
+        # partitions assigned in sgs that can be scaled. If that is the case, we scale up.
+        def scale_up
+          multi_part_sgs_families = Karafka::App
+                                    .assignments
+                                    .select { |_, partitions| partitions.size > 1 }
+                                    .keys
+                                    .map(&:subscription_group)
+                                    .map(&:name)
+                                    .uniq
+
+          # Select connections for scaling up
+          in_sg_families do |first_subscription_group, sg_listeners|
+            next unless stable?(sg_listeners)
+
+            multiplexing = first_subscription_group.multiplexing
+
+            next unless multiplexing.active?
+            next unless multiplexing.dynamic?
+            # If we cannot downscale, do not
+            next if sg_listeners.count(&:active?) >= multiplexing.max
+
+            sg_listeners.each do |sg_listener|
+              next unless multi_part_sgs_families.include?(sg_listener.subscription_group.name)
+              # Skip already active connections
+              next unless sg_listener.pending? || sg_listener.stopped?
+
+              touch(sg_listener.subscription_group.id)
+              sg_listener.start!
+
+              break
+            end
+          end
+        end
+
+        # Removes states that are no longer being reported for stopped/pending listeners
+        def evict
+          @mutex.synchronize do
+            @changes.delete_if do |_, details|
+              monotonic_now - details[:state_age_sync] >= EVICTION_DELAY
+            end
+          end
+        end
+
+        # Indicates, that something has changed on a subscription group. We consider every single
+        # change we make as a change to the setup as well.
+        # @param subscription_group_id [String]
+        def touch(subscription_group_id)
+          @mutex.synchronize do
+            @changes[subscription_group_id][:changed_at] = 0
+            @changes[subscription_group_id][:state_age_sync] = monotonic_now
+          end
+        end
+
+        # @param sg_listeners [Array<Listener>] listeners from one multiplexed sg
+        # @return [Boolean] is given subscription group listeners set stable. It is considered
+        #   stable when it had no changes happening on it recently and all relevant states in it
+        #   are also stable. This is a strong indicator that no rebalances or other operations are
+        #   happening at a given moment.
+        def stable?(sg_listeners)
+          # If none of listeners has changes reported it means we did not yet start collecting
+          # metrics about any of them and at least one must be present. We do not consider it
+          # stable in such case as we still are waiting for metrics.
+          return false if sg_listeners.none? do |sg_listener|
+            @changes.key?(sg_listener.subscription_group.id)
+          end
+
+          sg_listeners.all? do |sg_listener|
+            # Not all SGs may be started initially or may be stopped, we ignore them here as they
+            # are irrelevant from the point of view of establishing stability
+            next true unless @changes.key?(sg_listener.subscription_group.id)
+
+            state = @changes[sg_listener.subscription_group.id]
+
+            state[:state_age] >= @scale_delay &&
+              (monotonic_now - state[:changed_at]) >= @scale_delay &&
+              state[:state] == 'up' &&
+              state[:join_state] == 'steady'
+          end
+        end
+
+        # Yields listeners in groups based on their subscription groups
+        # @yieldparam [Karafka::Routing::SubscriptionGroup] first subscription group out of the
+        #   family
+        # @yieldparam [Array<Listener>] listeners of a single subscription group
+        def in_sg_families
+          grouped = @listeners.group_by { |listener| listener.subscription_group.name }
+
+          grouped.each_value do |listeners|
+            listener = listeners.first
+
+            yield(
+              listener.subscription_group,
+              listeners
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/connection/multiplexing/listener.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Pro connections related components
+    module Connection
+      # Namespace for Multiplexing management related components
+      module Multiplexing
+        # Listener used to connect listeners manager to the lifecycle events that are significant
+        # to its operations
+        class Listener
+          def initialize
+            @manager = App.config.internal.connection.manager
+          end
+
+          # Triggers connection manage subscription groups details noticing
+          #
+          # @param event [Karafka::Core::Monitoring::Event] event with statistics
+          def on_statistics_emitted(event)
+            @manager.notice(
+              event[:subscription_group_id],
+              event[:statistics]
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/instrumentation/performance_tracker.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Pro components instrumentation related code
+    module Instrumentation
+      # Tracker used to keep track of performance metrics
+      # It provides insights that can be used to optimize processing flow
+      # @note Even if we have some race-conditions here it is relevant due to the quantity of data.
+      #   This is why we do not mutex it.
+      class PerformanceTracker
+        include Singleton
+
+        # How many samples do we collect per topic partition
+        SAMPLES_COUNT = 200
+
+        private_constant :SAMPLES_COUNT
+
+        # Builds up nested concurrent hash for data tracking
+        def initialize
+          @processing_times = Hash.new do |topics_hash, topic|
+            topics_hash[topic] = Hash.new do |partitions_hash, partition|
+              partitions_hash[partition] = []
+            end
+          end
+        end
+
+        # @param topic [String]
+        # @param partition [Integer]
+        # @return [Float] p95 processing time of a single message from a single topic partition
+        def processing_time_p95(topic, partition)
+          values = @processing_times[topic][partition]
+
+          return 0 if values.empty?
+          return values.first if values.size == 1
+
+          percentile(0.95, values)
+        end
+
+        # @private
+        # @param event [Karafka::Core::Monitoring::Event] event details
+        # Tracks time taken to process a single message of a given topic partition
+        def on_consumer_consumed(event)
+          consumer = event[:caller]
+          messages = consumer.messages
+          topic = messages.metadata.topic
+          partition = messages.metadata.partition
+
+          samples = @processing_times[topic][partition]
+          samples << event[:time] / messages.count
+
+          return unless samples.size > SAMPLES_COUNT
+
+          samples.shift
+        end
+
+        private
+
+        # Computers the requested percentile out of provided values
+        # @param percentile [Float]
+        # @param values [Array<String>] all the values based on which we should
+        # @return [Float] computed percentile
+        def percentile(percentile, values)
+          values_sorted = values.sort
+
+          floor = (percentile * (values_sorted.length - 1) + 1).floor - 1
+          mod = (percentile * (values_sorted.length - 1) + 1).modulo(1)
+
+          values_sorted[floor] + (mod * (values_sorted[floor + 1] - values_sorted[floor]))
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -25,7 +25,7 @@ module Karafka
         # @param consumer [::Rdkafka::Consumer] consumer instance needed to talk with Kafka
         # @param expanded_topics [Hash] hash with expanded and normalized topics data
         def initialize(consumer, expanded_topics)
-          @consumer = Connection::Proxy.new(consumer)
+          @consumer = ::Karafka::Connection::Proxy.new(consumer)
           @expanded_topics = expanded_topics
           @mapped_topics = Hash.new { |h, k| h[k] = {} }
         end

data/lib/karafka/pro/iterator.rb

@@ -22,11 +22,6 @@ module Karafka
     #
     # It does **not** create a consumer group and does not have any offset management.
     class Iterator
-      # Local partition reference for librdkafka
-      Partition = Struct.new(:partition, :offset)
-
-      private_constant :Partition
-
       # A simple API allowing to iterate over topic/partition data, without having to subscribe
       # and deal with rebalances. This API allows for multi-partition streaming and is optimized
       # for data lookups. It allows for explicit stopping iteration over any partition during
@@ -127,7 +122,7 @@ module Karafka
 
         @current_consumer.pause(
           Rdkafka::Consumer::TopicPartitionList.new(
-            name => [Partition.new(partition, 0)]
+            name => [Rdkafka::Consumer::Partition.new(partition, 0)]
           )
         )
       end

data/lib/karafka/pro/loader.rb

@@ -53,6 +53,7 @@ module Karafka
           features.each { |feature| feature.pre_setup(config) }
 
           reconfigure(config)
+          expand
 
           load_topic_features
         end
@@ -82,18 +83,31 @@ module Karafka
 
           icfg.cli.contract = Contracts::ServerCliOptions.new
 
+          # Use manager that supports multiplexing
+          icfg.connection.manager = Connection::Manager.new
+
           icfg.processing.coordinator_class = Processing::Coordinator
           icfg.processing.partitioner_class = Processing::Partitioner
-          icfg.processing.scheduler_class = Processing::Scheduler
+          icfg.processing.scheduler_class = Processing::Schedulers::Default
           icfg.processing.jobs_queue_class = Processing::JobsQueue
+          icfg.processing.executor_class = Processing::Executor
           icfg.processing.jobs_builder = Processing::JobsBuilder.new
           icfg.processing.strategy_selector = Processing::StrategySelector.new
+          icfg.processing.expansions_selector = Processing::ExpansionsSelector.new
 
           icfg.active_job.consumer_class = ActiveJob::Consumer
           icfg.active_job.dispatcher = ActiveJob::Dispatcher.new
           icfg.active_job.job_options_contract = ActiveJob::JobOptionsContract.new
 
-          config.monitor.subscribe(PerformanceTracker.instance)
+          config.monitor.subscribe(Instrumentation::PerformanceTracker.instance)
+        end
+
+        # Adds extra modules to certain classes
+        # This expands their functionalities with things that are needed when operating in Pro
+        # It is used only when given class is part of the end user API and cannot be swapped by
+        # a pluggable component
+        def expand
+          Karafka::BaseConsumer.include Pro::BaseConsumer
         end
 
         # Loads the Pro features of Karafka

data/lib/karafka/pro/processing/coordinator.rb

@@ -42,7 +42,8 @@ module Karafka
 
           @virtual_offset_manager = VirtualOffsetManager.new(
             topic.name,
-            partition
+            partition,
+            topic.virtual_partitions.offset_metadata_strategy
           )
 
           # We register our own "internal" filter to support filtering of messages that were marked

data/lib/karafka/pro/processing/executor.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro executor that supports periodic jobs
+      class Executor < Karafka::Processing::Executor
+        # Runs the code that should happen before periodic job is scheduled
+        #
+        # @note While jobs are called `Periodic`, from the consumer perspective it is "ticking".
+        #   This name was taken for a reason: we may want to introduce periodic ticking also not
+        #   only during polling but for example on wait and a name "poll" would not align well.
+        #   A name "periodic" is not a verb and our other consumer actions are verbs like:
+        #   consume or revoked. So for the sake of consistency we have ticking here.
+        def before_schedule_periodic
+          consumer.on_before_schedule_tick
+        end
+
+        # Triggers consumer ticking
+        def periodic
+          consumer.on_tick
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/expansions_selector.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      # Pro selector of appropriate topic setup based features enhancements.
+      class ExpansionsSelector < Karafka::Processing::ExpansionsSelector
+        # @param topic [Karafka::Routing::Topic] topic with settings based on which we find
+        #   expansions
+        # @return [Array<Module>] modules with proper expansions we're suppose to use to enhance
+        #   the consumer
+        def find(topic)
+          # Start with the non-pro expansions
+          expansions = super
+          expansions << Pro::Processing::OffsetMetadata::Consumer if topic.offset_metadata?
+          expansions
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs/periodic.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Jobs
+        # Job that represents a "ticking" work. Work that we run periodically for the Periodics
+        # enabled topics.
+        class Periodic < ::Karafka::Processing::Jobs::Base
+          # @param executor [Karafka::Pro::Processing::Executor] pro executor that is suppose to
+          #   run a given job
+          def initialize(executor)
+            @executor = executor
+            super()
+          end
+
+          # Code executed before we schedule this job
+          def before_schedule
+            executor.before_schedule_periodic
+          end
+
+          # Runs the executor periodic action
+          def call
+            executor.periodic
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs/periodic_non_blocking.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Jobs
+        # Non-Blocking version of the Periodic job
+        # We use this version for LRJ topics for cases where saturated resources would not allow
+        # to run this job for extended period of time. Under such scenarios, if we would not use
+        # a non-blocking one, we would reach max.poll.interval.ms.
+        class PeriodicNonBlocking < Periodic
+          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Periodic`
+          def initialize(*args)
+            super
+            @non_blocking = true
+          end
+        end
+      end
+    end
+  end
+end