karafka 2.4.18 → 2.5.0.beta1

data/lib/karafka/instrumentation/logger_listener.rb

@@ -24,14 +24,29 @@ module Karafka
         @log_polling = log_polling
       end
 
+      #
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_connection_listener_before_fetch_loop(event)
+        listener_id = event[:caller].id
+        subscription_group = event[:subscription_group]
+        consumer_group_id = subscription_group.consumer_group.id
+        topics = subscription_group.topics.select(&:active?).map(&:name).join(', ')
+        group_details = "#{consumer_group_id}/#{subscription_group.id}"
+
+        info(
+          "[#{listener_id}] Group #{group_details} subscribing to topics: #{topics}"
+        )
+      end
+
       # Logs each messages fetching attempt
       #
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_connection_listener_fetch_loop(event)
         return unless log_polling?
 
-        listener = event[:caller]
-        debug "[#{listener.id}] Polling messages..."
+        listener_id = event[:caller].id
+        debug "[#{listener_id}] Polling messages..."
       end
 
       # Logs about messages that we've received from Kafka
@@ -40,11 +55,11 @@ module Karafka
       def on_connection_listener_fetch_loop_received(event)
         return unless log_polling?
 
-        listener = event[:caller]
+        listener_id = event[:caller].id
         time = event[:time].round(2)
         messages_count = event[:messages_buffer].size
 
-        message = "[#{listener.id}] Polled #{messages_count} messages in #{time}ms"
+        message = "[#{listener_id}] Polled #{messages_count} messages in #{time}ms"
 
         # We don't want the "polled 0" in dev as it would spam the log
         # Instead we publish only info when there was anything we could poll and fail over to the
@@ -147,7 +162,8 @@ module Karafka
       #
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_process_notice_signal(event)
-        info "Received #{event[:signal]} system signal"
+        server_id = Karafka::Server.id
+        info "[#{server_id}] Received #{event[:signal]} system signal"
 
         # We print backtrace only for ttin
         return unless event[:signal] == :SIGTTIN
@@ -168,38 +184,76 @@ module Karafka
 
       # Logs info that we're running Karafka app.
       #
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_running(_event)
-        info "Running in #{RUBY_DESCRIPTION}"
-        info "Running Karafka #{Karafka::VERSION} server"
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_running(event)
+        server_id = event[:server_id]
+
+        info "[#{server_id}] Running in #{RUBY_DESCRIPTION}"
+        info "[#{server_id}] Running Karafka #{Karafka::VERSION} server"
 
         return if Karafka.pro?
 
-        info 'See LICENSE and the LGPL-3.0 for licensing details'
+        info "[#{server_id}] See LICENSE and the LGPL-3.0 for licensing details"
       end
 
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_quieting(_event)
-        info 'Switching to quiet mode. New messages will not be processed'
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_quieting(event)
+        info "[#{event[:server_id]}] Switching to quiet mode. New messages will not be processed"
       end
 
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_quiet(_event)
-        info 'Reached quiet mode. No messages will be processed anymore'
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_quiet(event)
+        info "[#{event[:server_id]}] Reached quiet mode. No messages will be processed anymore"
       end
 
       # Logs info that we're going to stop the Karafka server.
       #
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_stopping(_event)
-        info 'Stopping Karafka server'
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_stopping(event)
+        info "[#{event[:server_id]}] Stopping Karafka server"
       end
 
       # Logs info that we stopped the Karafka server.
       #
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_stopped(_event)
-        info 'Stopped Karafka server'
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_stopped(event)
+        info "[#{event[:server_id]}] Stopped Karafka server"
+      end
+
+      # Logs info about partitions we have lost
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details with revoked partitions
+      def on_rebalance_partitions_revoked(event)
+        revoked_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
+        group_id = event[:consumer_group_id]
+        client_id = event[:client_id]
+        group_prefix = "[#{client_id}] Group #{group_id} rebalance"
+
+        if revoked_partitions.empty?
+          info "#{group_prefix}: No partitions revoked"
+        else
+          revoked_partitions.each do |topic, partitions|
+            info "#{group_prefix}: Partition(s) #{partitions.join(', ')} of #{topic} revoked"
+          end
+        end
+      end
+
+      # Logs info about partitions that we've gained
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details with assigned partitions
+      def on_rebalance_partitions_assigned(event)
+        assigned_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
+        group_id = event[:consumer_group_id]
+        client_id = event[:client_id]
+        group_prefix = "[#{client_id}] Group #{group_id} rebalance"
+
+        if assigned_partitions.empty?
+          info "#{group_prefix}: No partitions assigned"
+        else
+          assigned_partitions.each do |topic, partitions|
+            info "#{group_prefix}: Partition(s) #{partitions.join(', ')} of #{topic} assigned"
+          end
+        end
       end
 
       # Logs info when we have dispatched a message the the DLQ
@@ -371,9 +425,18 @@ module Karafka
         when 'connection.client.unsubscribe.error'
           error "Client unsubscribe error occurred: #{error}"
           error details
+        when 'parallel_segments.reducer.error'
+          error "Parallel segments reducer error occurred: #{error}"
+          error details
+        when 'parallel_segments.partitioner.error'
+          error "Parallel segments partitioner error occurred: #{error}"
+          error details
+        when 'virtual_partitions.partitioner.error'
+          error "Virtual partitions partitioner error occurred: #{error}"
+          error details
         # This handles any custom errors coming from places like Web-UI, etc
         else
-          error "#{type} error occurred: #{error}"
+          error "#{type} error occurred: #{error.class} - #{error}"
           error details
         end
       end

data/lib/karafka/instrumentation/proctitle_listener.rb

@@ -4,6 +4,10 @@ module Karafka
   module Instrumentation
     # Listener that sets a proc title with a nice descriptive value
     class ProctitleListener
+      include Helpers::ConfigImporter.new(
+        client_id: %i[client_id]
+      )
+
       Status::STATES.each_key do |state|
         class_eval <<~RUBY, __FILE__, __LINE__ + 1
           # Updates proc title to an appropriate state
@@ -19,7 +23,7 @@ module Karafka
       # @param status [String] any status we want to set
       def setproctitle(status)
         ::Process.setproctitle(
-          "karafka #{Karafka::App.config.client_id} (#{status})"
+          "karafka #{client_id} (#{status})"
         )
       end
     end

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -131,11 +131,11 @@ module Karafka
             tags = consumer_tags(consumer)
             tags.concat(default_tags)
 
-            count('consumer.messages', messages.count, tags: tags)
+            count('consumer.messages', messages.size, tags: tags)
             count('consumer.batches', 1, tags: tags)
             gauge('consumer.offset', metadata.last_offset, tags: tags)
             histogram('consumer.consumed.time_taken', event[:time], tags: tags)
-            histogram('consumer.batch_size', messages.count, tags: tags)
+            histogram('consumer.batch_size', messages.size, tags: tags)
             histogram('consumer.processing_lag', metadata.processing_lag, tags: tags)
             histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
           end

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -18,7 +18,7 @@ module Karafka
           #   picked up for processing.
           def call(messages, topic, partition, scheduled_at)
             Karafka::Messages::BatchMetadata.new(
-              size: messages.count,
+              size: messages.size,
               first_offset: messages.first&.offset || -1001,
               last_offset: messages.last&.offset || -1001,
               deserializers: topic.deserializers,

data/lib/karafka/pro/cleaner.rb

@@ -28,6 +28,14 @@ module Karafka
         def post_setup(_config)
           true
         end
+
+        # This feature does not need any changes post-fork
+        #
+        # @param _config [Karafka::Core::Configurable::Node]
+        # @param _pre_fork_producer [WaterDrop::Producer]
+        def post_fork(_config, _pre_fork_producer)
+          true
+        end
       end
     end
   end

data/lib/karafka/pro/cli/parallel_segments/base.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Cli
+      class ParallelSegments < Karafka::Cli::Base
+        # Base class for all the parallel segments related operations
+        class Base
+          include Helpers::Colorize
+
+          # @param options [Hash] cli flags options
+          def initialize(options)
+            @options = options
+          end
+
+          private
+
+          # @return [Hash]
+          attr_reader :options
+
+          # Returns consumer groups for parallel segments with which we should be working
+          #
+          # @return [Hash<String, Array<Karafka::Routing::ConsumerGroup>>] hash with all parallel
+          #   consumer groups as values and names of segments origin consumer group as the key.
+          def applicable_groups
+            requested_groups = options[:groups].dup || []
+
+            workable_groups = ::Karafka::App
+                              .routes
+                              .select(&:parallel_segments?)
+                              .group_by(&:segment_origin)
+
+            # Use all if none provided
+            return workable_groups if requested_groups.empty?
+
+            applicable_groups = {}
+
+            requested_groups.each do |requested_group|
+              workable_group = workable_groups[requested_group]
+
+              if workable_group
+                requested_groups.delete(requested_group)
+                applicable_groups[requested_group] = workable_group
+              else
+                raise(
+                  ::Karafka::Errors::ConsumerGroupNotFoundError,
+                  "Consumer group #{requested_group} was not found"
+                )
+              end
+            end
+
+            applicable_groups
+          end
+
+          # Collects the offsets for the segment origin consumer group and the parallel segments
+          # consumers groups. We use segment origin cg offsets as a baseline for the distribution
+          # and use existing (if any) parallel segments cgs offsets for validations.
+          #
+          # @param segment_origin [String] name of the origin consumer group
+          # @param segments [Array<Karafka::Routing::ConsumerGroup>]
+          # @return [Hash] fetched offsets for all the cg topics for all the consumer groups
+          def collect_offsets(segment_origin, segments)
+            topics_names = segments.first.topics.map(&:name)
+            consumer_groups = [segment_origin, segments.map(&:name)].flatten
+
+            consumer_groups_with_topics = consumer_groups
+                                          .map { |name| [name, topics_names] }
+                                          .to_h
+
+            lags_with_offsets = Karafka::Admin.read_lags_with_offsets(
+              consumer_groups_with_topics
+            )
+
+            lags_with_offsets.each do |_cg_name, topics|
+              topics.each do |_topic_name, partitions|
+                partitions.transform_values! { |details| details[:offset] }
+              end
+            end
+
+            lags_with_offsets
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cli/parallel_segments/collapse.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Cli
+      class ParallelSegments < Karafka::Cli::Base
+        # Takes the committed offset of each parallel segment for each topic and records
+        # them back onto the segment origin consumer group. Without `--force` it will raise an
+        # error on conflicts. With `--force` it will take the lowest possible offset for each
+        # topic partition as the baseline.
+        #
+        # @note Running this can cause you some double processing if the parallel segments final
+        #   offsets are not aligned.
+        #
+        # @note This will **not** remove the parallel segments consumer groups. Please use the
+        #   Admin API if you want them to be removed.
+        class Collapse < Base
+          # Runs the collapse operation
+          def call
+            puts 'Starting parallel segments collapse...'
+
+            segments_count = applicable_groups.size
+
+            if segments_count.zero?
+              puts "#{red('No')} consumer groups with parallel segments configuration found"
+
+              return
+            end
+
+            puts(
+              "Found #{green(segments_count)} consumer groups with parallel segments configuration"
+            )
+
+            collapses = []
+
+            applicable_groups.each do |segment_origin, segments|
+              puts
+              puts "Collecting group #{yellow(segment_origin)} details..."
+              offsets = collect_offsets(segment_origin, segments)
+
+              unless options.key?(:force)
+                puts
+                puts "Validating offsets positions for #{yellow(segment_origin)} consumer group..."
+                validate!(offsets, segment_origin)
+              end
+
+              puts
+              puts "Computing collapsed offsets for #{yellow(segment_origin)} consumer group..."
+              collapses << collapse(offsets, segments)
+            end
+
+            collapses.each do |collapse|
+              apply(collapse)
+            end
+
+            puts
+            puts "Collapse completed #{green('successfully')}!"
+          end
+
+          private
+
+          # Computes the lowest possible offset available for each topic partition and sets it
+          # on the segment origin consumer group.
+          #
+          # @param offsets [Hash]
+          # @param segments [Array<Karafka::Routing::ConsumerGroup>]
+          # @note This code does **not** apply the offsets, just computes their positions
+          def collapse(offsets, segments)
+            collapse = Hash.new { |h, k| h[k] = {} }
+            segments_names = segments.map(&:name)
+
+            offsets.each do |cg_name, topics|
+              next unless segments_names.include?(cg_name)
+
+              topics.each do |topic_name, partitions|
+                partitions.each do |partition_id, offset|
+                  current_lowest_offset = collapse[topic_name][partition_id]
+
+                  next if current_lowest_offset && current_lowest_offset < offset
+
+                  collapse[topic_name][partition_id] = offset
+                end
+              end
+            end
+
+            {
+              collapse: collapse,
+              segment_origin: segments.first.segment_origin
+            }
+          end
+
+          # In order to collapse the offsets of parallel segments back to one, we need to know
+          # to what offsets to collapse. The issue (that we solve picking lowest when forced)
+          # arises when there are more offsets that are not even in parallel segments for one
+          # topic partition. We should let user know about this if this happens so he does not
+          # end up with double-processing.
+          #
+          # @param offsets [Hash]
+          # @param segment_origin [String]
+          def validate!(offsets, segment_origin)
+            collapse = Hash.new { |h, k| h[k] = {} }
+
+            offsets.each do |cg_name, topics|
+              next if cg_name == segment_origin
+
+              topics.each do |topic_name, partitions|
+                partitions.each do |partition_id, offset|
+                  collapse[topic_name][partition_id] ||= Set.new
+                  collapse[topic_name][partition_id] << offset
+                end
+              end
+            end
+
+            inconclusive = false
+
+            collapse.each do |topic_name, partitions|
+              partitions.each do |partition_id, parallel_offsets|
+                next if parallel_offsets.size <= 1
+
+                inconclusive = true
+
+                puts(
+                  "  Inconclusive offsets for #{red(topic_name)}##{red(partition_id)}:" \
+                  " #{parallel_offsets.to_a.join(', ')}"
+                )
+              end
+            end
+
+            return unless inconclusive
+
+            raise(
+              ::Karafka::Errors::CommandValidationError,
+              "Parallel segments for #{red(segment_origin)} have #{red('inconclusive')} offsets"
+            )
+          end
+
+          # Applies the collapsed lowest offsets onto the segment origin consumer group
+          #
+          # @param collapse [Hash]
+          def apply(collapse)
+            segment_origin = collapse[:segment_origin]
+            alignments = collapse[:collapse]
+
+            puts
+            puts "Adjusting offsets of segment origin consumer group: #{green(segment_origin)}"
+
+            alignments.each do |topic_name, partitions|
+              puts "  Topic #{green(topic_name)}:"
+
+              partitions.each do |partition_id, offset|
+                puts "    Partition #{green(partition_id)}: starting offset #{green(offset)}"
+              end
+            end
+
+            Karafka::Admin.seek_consumer_group(segment_origin, alignments)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cli/parallel_segments/distribute.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Cli
+      class ParallelSegments < Karafka::Cli::Base
+        # Command that makes it easier for users to migrate from regular consumer groups to
+        # the parallel segments consumers groups by automatically distributing offsets based on
+        # the used "normal" consumer group.
+        #
+        # Takes the segments origin consumer group offsets for a given set of topics and
+        # distributes those offsets onto the parallel segments consumer groups, so they can pick
+        # up where the origin group left.
+        #
+        # To make sure users do not accidentally "re-distribute" their offsets from the original
+        # consumer group after the parallel consumer groups had offsets assigned and started to
+        # work, we check if the parallel groups have any offsets, if so unless forced we halt.
+        #
+        # @note This command does not remove the original consumer group from Kafka. We keep it
+        #   just as a backup. User can remove it himself.
+        #
+        # @note Kafka has no atomic operations this is why we first collect all the data and run
+        #   needed validations before applying offsets.
+        class Distribute < Base
+          # Runs the distribution process
+          def call
+            puts 'Starting parallel segments distribution...'
+
+            segments_count = applicable_groups.size
+
+            if segments_count.zero?
+              puts "#{red('No')} consumer groups with parallel segments configuration found"
+
+              return
+            end
+
+            puts(
+              "Found #{green(segments_count)} consumer groups with parallel segments configuration"
+            )
+
+            distributions = []
+
+            applicable_groups.each do |segment_origin, segments|
+              puts
+              puts "Collecting group #{yellow(segment_origin)} details..."
+              offsets = collect_offsets(segment_origin, segments)
+
+              unless options.key?(:force)
+                puts "Validating group #{yellow(segment_origin)} parallel segments..."
+                validate!(offsets, segments)
+              end
+
+              puts "Distributing group #{yellow(segment_origin)} offsets..."
+              distributions += distribute(offsets, segments)
+            end
+
+            distributions.each do |distribution|
+              apply(distribution)
+            end
+
+            puts
+            puts "Distribution completed #{green('successfully')}!"
+          end
+
+          private
+
+          # Validates the current state of topics offsets assignments.
+          # We want to make sure, that users do not run distribution twice, especially for a
+          # parallel segments consumers group set that was already actively consumed. This is why
+          # we check if there was any offsets already present in the parallel segments consumer
+          # groups and if so, we raise an error. This can be disabled with `--force`.
+          #
+          # It prevents users from overwriting the already set segments distribution.
+          # Adding new topics to the same parallel segments consumer group does not require us to
+          # run this at all and on top of that users can always use `--consumer_groups` flag to
+          # limit the cgs that we will be operating here
+          #
+          # @param offsets [Hash]
+          # @param segments [Array<Karafka::Routing::ConsumerGroup>]
+          def validate!(offsets, segments)
+            segments_names = segments.map(&:name)
+
+            offsets.each do |cg_name, topics|
+              next unless segments_names.include?(cg_name)
+
+              topics.each do |topic_name, partitions|
+                partitions.each do |partition_id, offset|
+                  next unless offset.to_i.positive?
+
+                  raise(
+                    ::Karafka::Errors::CommandValidationError,
+                    "Parallel segment #{red(cg_name)} already has offset #{red(offset)}" \
+                    " set for #{red("#{topic_name}##{partition_id}")}"
+                  )
+                end
+              end
+            end
+          end
+
+          # Computes the offsets distribution for all the segments consumer groups so when user
+          # migrates from one CG to parallel segments, those segments know where to start consuming
+          # the data.
+          #
+          # @param offsets [Hash]
+          # @param segments [Array<Karafka::Routing::ConsumerGroup>]
+          # @note This code does **not** apply the offsets, just computes their positions
+          def distribute(offsets, segments)
+            distributions = []
+            segments_names = segments.map(&:name)
+
+            offsets.each do |cg_name, topics|
+              next if segments_names.include?(cg_name)
+
+              distribution = {}
+
+              topics.each do |topic_name, partitions|
+                partitions.each do |partition_id, offset|
+                  distribution[topic_name] ||= {}
+                  distribution[topic_name][partition_id] = offset
+                end
+              end
+
+              next if distribution.empty?
+
+              segments_names.each do |segment_name|
+                distributions << {
+                  segment_name: segment_name,
+                  distribution: distribution
+                }
+              end
+            end
+
+            distributions
+          end
+
+          # Takes the details of the distribution of offsets for a given segment and adjust the
+          # starting offsets for all the consumer group topics based on the distribution.
+          #
+          # @param distribution [Hash]
+          def apply(distribution)
+            segment_name = distribution[:segment_name]
+            alignments = distribution[:distribution]
+
+            puts
+            puts "Adjusting offsets of parallel segments consumer group: #{green(segment_name)}"
+
+            alignments.each do |topic_name, partitions|
+              puts "  Topic #{green(topic_name)}:"
+
+              partitions.each do |partition_id, offset|
+                puts "    Partition #{green(partition_id)}: starting offset #{green(offset)}"
+              end
+            end
+
+            Karafka::Admin.seek_consumer_group(segment_name, alignments)
+          end
+        end
+      end
+    end
+  end
+end