karafka 2.2.14 → 2.3.0.alpha2

data/lib/karafka/instrumentation/logger_listener.rb

@@ -247,6 +247,9 @@ module Karafka
         when 'consumer.shutdown.error'
           error "Consumer on shutdown failed due to an error: #{error}"
           error details
+        when 'consumer.tick.error'
+          error "Consumer tick failed due to an error: #{error}"
+          error details
         when 'worker.process.error'
           fatal "Worker processing failed due to an error: #{error}"
           fatal details

data/lib/karafka/instrumentation/notifications.rb

@@ -36,22 +36,25 @@ module Karafka
         connection.listener.before_fetch_loop
         connection.listener.fetch_loop
         connection.listener.fetch_loop.received
-
-        rebalance.partitions_assign
-        rebalance.partitions_assigned
-        rebalance.partitions_revoke
-        rebalance.partitions_revoked
+        connection.listener.after_fetch_loop
 
         consumer.before_schedule_consume
         consumer.consume
         consumer.consumed
         consumer.consuming.pause
         consumer.consuming.retry
+
         consumer.before_schedule_idle
         consumer.idle
+
         consumer.before_schedule_revoked
         consumer.revoke
         consumer.revoked
+
+        consumer.before_schedule_tick
+        consumer.tick
+        consumer.ticked
+
         consumer.before_schedule_shutdown
         consumer.shutting_down
         consumer.shutdown
@@ -63,6 +66,11 @@ module Karafka
 
         process.notice_signal
 
+        rebalance.partitions_assign
+        rebalance.partitions_assigned
+        rebalance.partitions_revoke
+        rebalance.partitions_revoked
+
         statistics.emitted
 
         worker.process

data/lib/karafka/instrumentation/vendors/appsignal/metrics_listener.rb

@@ -42,6 +42,16 @@ module Karafka
 
           configure
 
+          # Types of errors originating from user code in the consumer flow
+          USER_CONSUMER_ERROR_TYPES = %w[
+            consumer.consume.error
+            consumer.revoked.error
+            consumer.shutdown.error
+            consumer.tick.error
+          ].freeze
+
+          private_constant :USER_CONSUMER_ERROR_TYPES
+
           # Before each consumption process, lets start a transaction associated with it
           # We also set some basic metadata about the given consumption that can be useful for
           # debugging
@@ -94,34 +104,27 @@ module Karafka
             client.register_probe(:karafka, -> { minute_probe })
           end
 
-          # Keeps track of revocation user code execution
-          #
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_consumer_revoke(event)
-            consumer = event.payload[:caller]
-            start_transaction(consumer, 'revoked')
-          end
-
-          # Finishes the revocation transaction
-          #
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_revoked(_event)
-            stop_transaction
-          end
-
-          # Keeps track of revocation user code execution
-          #
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_consumer_shutting_down(event)
-            consumer = event.payload[:caller]
-            start_transaction(consumer, 'shutdown')
-          end
+          [
+            %i[revoke revoked revoked],
+            %i[shutting_down shutdown shutdown],
+            %i[tick ticked tick]
+          ].each do |before, after, name|
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              # Keeps track of user code execution
+              #
+              # @param event [Karafka::Core::Monitoring::Event]
+              def on_consumer_#{before}(event)
+                consumer = event.payload[:caller]
+                start_transaction(consumer, '#{name}')
+              end
 
-          # Finishes the shutdown transaction
-          #
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_shutdown(_event)
-            stop_transaction
+              # Finishes the transaction
+              #
+              # @param _event [Karafka::Core::Monitoring::Event]
+              def on_consumer_#{after}(_event)
+                stop_transaction
+              end
+            RUBY
           end
 
           # Counts DLQ dispatches
@@ -141,7 +144,7 @@ module Karafka
           # @param event [Karafka::Core::Monitoring::Event] error event details
           def on_error_occurred(event)
             # If this is a user consumption related error, we bump the counters for metrics
-            if event[:type] == 'consumer.consume.error'
+            if USER_CONSUMER_ERROR_TYPES.include?(event[:type])
               consumer = event.payload[:caller]
 
               with_multiple_resolutions(consumer) do |tags|

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -55,7 +55,24 @@ module Karafka
             consumer = job.executor.topic.consumer
             topic = job.executor.topic.name
 
-            current_span.resource = "#{consumer}#consume"
+            action = case job_type
+                     when 'Periodic'
+                       'tick'
+                     when 'PeriodicNonBlocking'
+                       'tick'
+                     when 'Shutdown'
+                       'shutdown'
+                     when 'Revoked'
+                       'revoked'
+                     when 'RevokedNonBlocking'
+                       'revoked'
+                     when 'Idle'
+                       'idle'
+                     else
+                       'consume'
+                     end
+
+            current_span.resource = "#{consumer}##{action}"
             info "[#{job.id}] #{job_type} job for #{consumer} on #{topic} started"
 
             pop_tags
@@ -102,6 +119,8 @@ module Karafka
               error "Consumer after consume failed due to an error: #{error}"
             when 'consumer.shutdown.error'
               error "Consumer on shutdown failed due to an error: #{error}"
+            when 'consumer.tick.error'
+              error "Consumer tick failed due to an error: #{error}"
             when 'worker.process.error'
               fatal "Worker processing failed due to an error: #{error}"
             when 'connection.listener.fetch_loop.error'

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -128,18 +128,21 @@ module Karafka
             histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
           end
 
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_consumer_revoked(event)
-            tags = default_tags + consumer_tags(event.payload[:caller])
-
-            count('consumer.revoked', 1, tags: tags)
-          end
-
-          # @param event [Karafka::Core::Monitoring::Event]
-          def on_consumer_shutdown(event)
-            tags = default_tags + consumer_tags(event.payload[:caller])
-
-            count('consumer.shutdown', 1, tags: tags)
+          {
+            revoked: :revoked,
+            shutdown: :shutdown,
+            ticked: :tick
+          }.each do |after, name|
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              # Keeps track of user code execution
+              #
+              # @param event [Karafka::Core::Monitoring::Event]
+              def on_consumer_#{after}(event)
+                tags = default_tags + consumer_tags(event.payload[:caller])
+
+                count('consumer.#{name}', 1, tags: tags)
+              end
+            RUBY
           end
 
           # Worker related metrics

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -15,6 +15,14 @@ module Karafka
         # data would be processed, but process itself would still be active. This listener allows
         # for defining of a ttl that gets bumped on each poll loop and before and after processing
         # of a given messages batch.
+        #
+        # @note This listener will bind itself only when Karafka will actually attempt to start
+        #   and moves from initializing to running. Before that, the TCP server will NOT be active.
+        #   This is done on purpose to mitigate a case where users would subscribe this listener
+        #   in `karafka.rb` without checking the recommendations of conditional assignment.
+        #
+        # @note In case of usage within an embedding with Puma, you need to select different port
+        #   then the one used by Puma itself.
         class LivenessListener
           include ::Karafka::Core::Helpers::Time
 
@@ -40,12 +48,18 @@ module Karafka
             consuming_ttl: 5 * 60 * 1_000,
             polling_ttl: 5 * 60 * 1_000
           )
-            @server = TCPServer.new(*[hostname, port].compact)
+            @hostname = hostname
+            @port = port
             @polling_ttl = polling_ttl
             @consuming_ttl = consuming_ttl
             @mutex = Mutex.new
             @pollings = {}
             @consumptions = {}
+          end
+
+          # @param _event [Karafka::Core::Monitoring::Event]
+          def on_app_running(_event)
+            @server = TCPServer.new(*[@hostname, @port].compact)
 
             Thread.new do
               loop do
@@ -54,42 +68,37 @@ module Karafka
             end
           end
 
-          # Tick on each fetch
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_connection_listener_fetch_loop(_event)
-            mark_polling_tick
-          end
-
-          # Tick on starting work
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_consume(_event)
-            mark_consumption_tick
-          end
-
-          # Tick on finished work
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_consumed(_event)
-            clear_consumption_tick
-          end
-
+          # Stop the http server when we stop the process
           # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_revoke(_event)
-            mark_consumption_tick
+          def on_app_stopped(_event)
+            @server.close
           end
 
+          # Tick on each fetch
           # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_revoked(_event)
-            clear_consumption_tick
+          def on_connection_listener_fetch_loop(_event)
+            mark_polling_tick
           end
 
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_shutting_down(_event)
-            mark_consumption_tick
-          end
+          {
+            consume: :consumed,
+            revoke: :revoked,
+            shutting_down: :shutdown,
+            tick: :ticked
+          }.each do |before, after|
+            class_eval <<~RUBY, __FILE__, __LINE__ + 1
+              # Tick on starting work
+              # @param _event [Karafka::Core::Monitoring::Event]
+              def on_consumer_#{before}(_event)
+                mark_consumption_tick
+              end
 
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_consumer_shutdown(_event)
-            clear_consumption_tick
+              # Tick on finished work
+              # @param _event [Karafka::Core::Monitoring::Event]
+              def on_consumer_#{after}(_event)
+                clear_consumption_tick
+              end
+            RUBY
           end
 
           # @param _event [Karafka::Core::Monitoring::Event]
@@ -98,12 +107,6 @@ module Karafka
             clear_polling_tick
           end
 
-          # Stop the http server when we stop the process
-          # @param _event [Karafka::Core::Monitoring::Event]
-          def on_app_stopped(_event)
-            @server.close
-          end
-
           private
 
           # Wraps the logic with a mutex

data/lib/karafka/pro/base_consumer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Extra methods always used in the base consumer in the pro mode
+    #
+    # We do not define those methods as part of the strategies flows, because they are injected
+    # (strategies) on singletons and often used only in one of the strategy variants
+    #
+    # Methods here are suppose to be always available or are expected to be redefined
+    module BaseConsumer
+      # Runs the on-schedule tick periodic operations
+      # This method is an alias but is part of the naming convention used for other flows, this
+      # is why we do not reference the `handle_before_schedule_tick` directly
+      def on_before_schedule_tick
+        handle_before_schedule_tick
+      end
+
+      # Used by the executor to trigger consumer tick
+      # @private
+      def on_tick
+        handle_tick
+      rescue StandardError => e
+        Karafka.monitor.instrument(
+          'error.occurred',
+          error: e,
+          caller: self,
+          type: 'consumer.tick.error'
+        )
+      end
+
+      # By default we do nothing when ticking
+      def tick; end
+    end
+  end
+end

data/lib/karafka/pro/connection/manager.rb

@@ -0,0 +1,269 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Connection
+      # Manager that can handle working with multiplexed connections.
+      #
+      # This manager takes into consideration the number of partitions assigned to the topics and
+      # does its best to balance. Additional connections may not always be utilized because
+      # alongside of them, other processes may "hijack" the assignment. In such cases those extra
+      # empty connections will be turned off after a while.
+      #
+      # @note Manager operations relate to consumer groups and not subscription groups. Since
+      #   cluster operations can cause consumer group wide effects, we always apply only one
+      #   change on a consumer group.
+      class Manager < Karafka::Connection::Manager
+        include Core::Helpers::Time
+
+        # How long should we wait after a rebalance before doing anything on a consumer group
+        #
+        # @param scale_delay [Integer] How long should we wait before making any changes. Any
+        #   change related to this consumer group will postpone the scaling operations. This is
+        #   done that way to prevent too many friction in the cluster. It is 1 minute by default
+        def initialize(scale_delay = 60 * 1_000)
+          super()
+          @scale_delay = scale_delay
+          @mutex = Mutex.new
+          @changes = Hash.new do |h, k|
+            h[k] = {
+              state: '',
+              join_state: '',
+              state_age: 0,
+              changed_at: monotonic_now
+            }
+          end
+        end
+
+        # Registers listeners and starts the scaling procedures
+        #
+        # When using dynamic multiplexing, it will start the absolute minimum of connections for
+        # subscription group available.
+        #
+        # @param listeners [Connection::ListenersBatch]
+        def register(listeners)
+          @listeners = listeners
+
+          # Preload all the keys into the hash so we never add keys to changes but just change them
+          listeners.each { |listener| @changes[listener.subscription_group.id] }
+
+          in_sg_families do |first_subscription_group, sg_listeners|
+            multiplexing = first_subscription_group.multiplexing
+
+            if multiplexing.active? && multiplexing.dynamic?
+              # Start as many boot listeners as user wants. If not configured, starts half of max.
+              sg_listeners.first(multiplexing.boot).each(&:start!)
+            else
+              sg_listeners.each(&:start!)
+            end
+          end
+        end
+
+        # Collects data from the statistics about given subscription group. This is used to ensure
+        # that we do not rescale short after rebalances, deployments, etc.
+        # @param subscription_group_id [String] id of the subscription group for which statistics
+        #   were emitted
+        # @param statistics [Hash] emitted statistics
+        #
+        # @note Please note that while we collect here per subscription group, we use those metrics
+        #   collectively on a whole consumer group. This reduces the friction.
+        def notice(subscription_group_id, statistics)
+          times = []
+          # stateage is in microseconds
+          # We monitor broker changes to make sure we do not introduce extra friction
+          times << statistics['brokers'].values.map { |stats| stats['stateage'] }.min / 1_000
+          times << statistics['cgrp']['rebalance_age']
+          times << statistics['cgrp']['stateage']
+
+          # Keep the previous change age for changes that were triggered by us
+          previous_changed_at = @changes[subscription_group_id][:changed_at]
+
+          @changes[subscription_group_id].merge!(
+            state_age: times.min,
+            changed_at: previous_changed_at,
+            join_state: statistics['cgrp']['join_state'],
+            state: statistics['cgrp']['state']
+          )
+        end
+
+        # Shuts down all the listeners when it is time (including moving to quiet) or rescales
+        # when it is needed
+        def control
+          Karafka::App.done? ? shutdown : rescale
+        end
+
+        private
+
+        # Handles the shutdown and quiet flows
+        def shutdown
+          active_listeners = @listeners.active
+
+          # When we are done processing immediately quiet all the listeners so they do not pick up
+          # new work to do
+          once(:quiet!) { active_listeners.each(&:quiet!) }
+
+          # If we are in the process of moving to quiet state, we need to check it.
+          if Karafka::App.quieting? && active_listeners.all?(&:quiet?)
+            once(:quieted!) { Karafka::App.quieted! }
+          end
+
+          return if Karafka::App.quiet?
+
+          # Since separate subscription groups are subscribed to different topics, there is no risk
+          # in shutting them down independently even if they operate in the same subscription group
+          in_sg_families do |first_subscription_group, sg_listeners|
+            active_sg_listeners = sg_listeners.select(&:active?)
+
+            # Do nothing until all listeners from the same consumer group are quiet. Otherwise we
+            # could have problems with in-flight rebalances during shutdown
+            next unless active_sg_listeners.all?(&:quiet?)
+
+            # Do not stop the same family twice
+            once(:stop!, first_subscription_group.name) { active_sg_listeners.each(&:stop!) }
+          end
+
+          return unless @listeners.active.all?(&:stopped?)
+
+          # All listeners including pending need to be moved at the end to stopped state for
+          # the whole server to stop
+          once(:stop!) { @listeners.each(&:stopped!) }
+        end
+
+        # Handles two scenarios:
+        #   - Selects subscriptions that could benefit from having more parallel connections
+        #     to kafka and then upscales them
+        #   - Selects subscriptions that are idle (have nothing subscribed to them) and then shuts
+        #     them down
+        #
+        # We always run scaling down and up because it may be applicable to different CGs
+        def rescale
+          scale_down
+          scale_up
+        end
+
+        # Checks for connections without any assignments and scales them down.
+        # Does that only for dynamically multiplexed subscription groups
+        def scale_down
+          sgs_in_use = Karafka::App.assignments.keys.map(&:subscription_group).uniq
+
+          # Select connections for scaling down
+          in_sg_families do |first_subscription_group, sg_listeners|
+            next unless stable?(sg_listeners)
+
+            multiplexing = first_subscription_group.multiplexing
+
+            next unless multiplexing.active?
+            next unless multiplexing.dynamic?
+
+            # If we cannot downscale, do not
+            next if sg_listeners.count(&:active?) <= multiplexing.min
+
+            sg_listeners.each do |sg_listener|
+              # Do not stop connections with subscriptions
+              next if sgs_in_use.include?(sg_listener.subscription_group)
+              # Skip listeners that are already in standby
+              next unless sg_listener.active?
+
+              touch(sg_listener.subscription_group.id)
+
+              # Shut down not used connection
+              sg_listener.stop!
+
+              break
+            end
+          end
+        end
+
+        # Checks if we have space to scale and if there are any assignments with multiple topics
+        # partitions assigned in sgs that can be scaled. If that is the case, we scale up.
+        def scale_up
+          multi_part_sgs_families = Karafka::App
+                                    .assignments
+                                    .select { |_, partitions| partitions.size > 1 }
+                                    .keys
+                                    .map(&:subscription_group)
+                                    .map(&:name)
+                                    .uniq
+
+          # Select connections for scaling up
+          in_sg_families do |first_subscription_group, sg_listeners|
+            next unless stable?(sg_listeners)
+
+            multiplexing = first_subscription_group.multiplexing
+
+            next unless multiplexing.active?
+            next unless multiplexing.dynamic?
+            # If we cannot downscale, do not
+            next if sg_listeners.count(&:active?) >= multiplexing.max
+
+            sg_listeners.each do |sg_listener|
+              next unless multi_part_sgs_families.include?(sg_listener.subscription_group.name)
+              # Skip already active connections
+              next unless sg_listener.pending? || sg_listener.stopped?
+
+              touch(sg_listener.subscription_group.id)
+              sg_listener.start!
+
+              break
+            end
+          end
+        end
+
+        # Indicates, that something has changed on a subscription group. We consider every single
+        # change we make as a change to the setup as well.
+        # @param subscription_group_id [String]
+        def touch(subscription_group_id)
+          @changes[subscription_group_id][:changed_at] = 0
+        end
+
+        # @param sg_listeners [Array<Listener>] listeners from one multiplexed sg
+        # @return [Boolean] is given subscription group listeners set stable. It is considered
+        #   stable when it had no changes happening on it recently and all relevant states in it
+        #   are also stable. This is a strong indicator that no rebalances or other operations are
+        #   happening at a given moment.
+        def stable?(sg_listeners)
+          sg_listeners.all? do |sg_listener|
+            # If a listener is not active, we do not take it into consideration when looking at
+            # the stability data
+            next true unless sg_listener.active?
+
+            state = @changes[sg_listener.subscription_group.id]
+
+            state[:state_age] >= @scale_delay &&
+              (monotonic_now - state[:changed_at]) >= @scale_delay &&
+              state[:state] == 'up' &&
+              state[:join_state] == 'steady'
+          end
+        end
+
+        # Yields listeners in groups based on their subscription groups
+        # @yieldparam [Karafka::Routing::SubscriptionGroup] first subscription group out of the
+        #   family
+        # @yieldparam [Array<Listener>] listeners of a single subscription group
+        def in_sg_families
+          grouped = @listeners.group_by { |listener| listener.subscription_group.name }
+
+          grouped.each_value do |listeners|
+            listener = listeners.first
+
+            yield(
+              listener.subscription_group,
+              listeners
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/connection/multiplexing/listener.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Namespace for Pro connections related components
+    module Connection
+      # Namespace for Multiplexing management related components
+      module Multiplexing
+        # Listener used to connect listeners manager to the lifecycle events that are significant
+        # to its operations
+        class Listener
+          def initialize
+            @manager = App.config.internal.connection.manager
+          end
+
+          # Triggers connection manage subscription groups details noticing
+          #
+          # @param event [Karafka::Core::Monitoring::Event] event with statistics
+          def on_statistics_emitted(event)
+            @manager.notice(
+              event[:subscription_group_id],
+              event[:statistics]
+            )
+          end
+        end
+      end
+    end
+  end
+end