karafka 2.4.18 → 2.5.0

data/lib/karafka/instrumentation/vendors/kubernetes/base_listener.rb

@@ -8,11 +8,12 @@ module Karafka
       # Namespace for instrumentation related with Kubernetes
       module Kubernetes
         # Base Kubernetes Listener providing basic HTTP server capabilities to respond with health
+        # statuses
         class BaseListener
           include ::Karafka::Core::Helpers::Time
 
           # All good with Karafka
-          OK_CODE = '204 No Content'
+          OK_CODE = '200 OK'
 
           # Some timeouts, fail
           FAIL_CODE = '500 Internal Server Error'
@@ -38,11 +39,15 @@ module Karafka
 
           # Responds to a HTTP request with the process liveness status
           def respond
+            body = JSON.generate(status_body)
+
             client = @server.accept
             client.gets
             client.print "HTTP/1.1 #{healthy? ? OK_CODE : FAIL_CODE}\r\n"
-            client.print "Content-Type: text/plain\r\n"
+            client.print "Content-Type: application/json\r\n"
+            client.print "Content-Length: #{body.bytesize}\r\n"
             client.print "\r\n"
+            client.print body
             client.close
 
             true
@@ -50,6 +55,16 @@ module Karafka
             !@server.closed?
           end
 
+          # @return [Hash] hash that will be the response body
+          def status_body
+            {
+              status: healthy? ? 'healthy' : 'unhealthy',
+              timestamp: Time.now.to_i,
+              port: @port,
+              process_id: ::Process.pid
+            }
+          end
+
           # Starts background thread with micro-http monitoring
           def start
             @server = TCPServer.new(*[@hostname, @port].compact)

data/lib/karafka/instrumentation/vendors/kubernetes/liveness_listener.rb

@@ -53,7 +53,7 @@ module Karafka
             consuming_ttl: 5 * 60 * 1_000,
             polling_ttl: 5 * 60 * 1_000
           )
-            # If this is set to true, it indicates unrecoverable error like fencing
+            # If this is set to a symbol, it indicates unrecoverable error like fencing
             # While fencing can be partial (for one of the SGs), we still should consider this
             # as an undesired state for the whole process because it halts processing in a
             # non-recoverable manner forever
@@ -116,7 +116,7 @@ module Karafka
             # We mark as unrecoverable only on certain errors that will not be fixed by retrying
             return unless UNRECOVERABLE_RDKAFKA_ERRORS.include?(error.code)
 
-            @unrecoverable = true
+            @unrecoverable = error.code
           end
 
           # Deregister the polling tracker for given listener
@@ -142,17 +142,29 @@ module Karafka
           # Did we exceed any of the ttls
           # @return [String] 204 string if ok, 500 otherwise
           def healthy?
-            time = monotonic_now
-
             return false if @unrecoverable
-            return false if @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
-            return false if @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+            return false if polling_ttl_exceeded?
+            return false if consuming_ttl_exceeded?
 
             true
           end
 
           private
 
+          # @return [Boolean] true if the consumer exceeded the polling ttl
+          def polling_ttl_exceeded?
+            time = monotonic_now
+
+            @pollings.values.any? { |tick| (time - tick) > @polling_ttl }
+          end
+
+          # @return [Boolean] true if the consumer exceeded the consuming ttl
+          def consuming_ttl_exceeded?
+            time = monotonic_now
+
+            @consumptions.values.any? { |tick| (time - tick) > @consuming_ttl }
+          end
+
           # Wraps the logic with a mutex
           # @param block [Proc] code we want to run in mutex
           def synchronize(&block)
@@ -191,6 +203,17 @@ module Karafka
               @consumptions.delete(thread_id)
             end
           end
+
+          # @return [Hash] response body status
+          def status_body
+            super.merge!(
+              errors: {
+                polling_ttl_exceeded: polling_ttl_exceeded?,
+                consumption_ttl_exceeded: consuming_ttl_exceeded?,
+                unrecoverable: @unrecoverable
+              }
+            )
+          end
         end
       end
     end

data/lib/karafka/instrumentation/vendors/kubernetes/swarm_liveness_listener.rb

@@ -47,6 +47,15 @@ module Karafka
           def healthy?
             (monotonic_now - @controlling) < @controlling_ttl
           end
+
+          # @return [Hash] response body status
+          def status_body
+            super.merge!(
+              errors: {
+                controlling_ttl_exceeded: !healthy?
+              }
+            )
+          end
         end
       end
     end

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -18,7 +18,7 @@ module Karafka
           #   picked up for processing.
           def call(messages, topic, partition, scheduled_at)
             Karafka::Messages::BatchMetadata.new(
-              size: messages.count,
+              size: messages.size,
               first_offset: messages.first&.offset || -1001,
               last_offset: messages.last&.offset || -1001,
               deserializers: topic.deserializers,

data/lib/karafka/pro/cleaner.rb

@@ -28,6 +28,14 @@ module Karafka
         def post_setup(_config)
           true
         end
+
+        # This feature does not need any changes post-fork
+        #
+        # @param _config [Karafka::Core::Configurable::Node]
+        # @param _pre_fork_producer [WaterDrop::Producer]
+        def post_fork(_config, _pre_fork_producer)
+          true
+        end
       end
     end
   end

data/lib/karafka/pro/cli/parallel_segments/base.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Cli
+      class ParallelSegments < Karafka::Cli::Base
+        # Base class for all the parallel segments related operations
+        class Base
+          include Helpers::Colorize
+
+          # @param options [Hash] cli flags options
+          def initialize(options)
+            @options = options
+          end
+
+          private
+
+          # @return [Hash]
+          attr_reader :options
+
+          # Returns consumer groups for parallel segments with which we should be working
+          #
+          # @return [Hash<String, Array<Karafka::Routing::ConsumerGroup>>] hash with all parallel
+          #   consumer groups as values and names of segments origin consumer group as the key.
+          def applicable_groups
+            requested_groups = options[:groups].dup || []
+
+            workable_groups = ::Karafka::App
+                              .routes
+                              .select(&:parallel_segments?)
+                              .group_by(&:segment_origin)
+
+            # Use all if none provided
+            return workable_groups if requested_groups.empty?
+
+            applicable_groups = {}
+
+            requested_groups.each do |requested_group|
+              workable_group = workable_groups[requested_group]
+
+              if workable_group
+                requested_groups.delete(requested_group)
+                applicable_groups[requested_group] = workable_group
+              else
+                raise(
+                  ::Karafka::Errors::ConsumerGroupNotFoundError,
+                  "Consumer group #{requested_group} was not found"
+                )
+              end
+            end
+
+            applicable_groups
+          end
+
+          # Collects the offsets for the segment origin consumer group and the parallel segments
+          # consumers groups. We use segment origin cg offsets as a baseline for the distribution
+          # and use existing (if any) parallel segments cgs offsets for validations.
+          #
+          # @param segment_origin [String] name of the origin consumer group
+          # @param segments [Array<Karafka::Routing::ConsumerGroup>]
+          # @return [Hash] fetched offsets for all the cg topics for all the consumer groups
+          def collect_offsets(segment_origin, segments)
+            topics_names = segments.first.topics.map(&:name)
+            consumer_groups = [segment_origin, segments.map(&:name)].flatten
+
+            consumer_groups_with_topics = consumer_groups
+                                          .map { |name| [name, topics_names] }
+                                          .to_h
+
+            lags_with_offsets = Karafka::Admin.read_lags_with_offsets(
+              consumer_groups_with_topics
+            )
+
+            lags_with_offsets.each do |_cg_name, topics|
+              topics.each do |_topic_name, partitions|
+                partitions.transform_values! { |details| details[:offset] }
+              end
+            end
+
+            lags_with_offsets
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cli/parallel_segments/collapse.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Cli
+      class ParallelSegments < Karafka::Cli::Base
+        # Takes the committed offset of each parallel segment for each topic and records
+        # them back onto the segment origin consumer group. Without `--force` it will raise an
+        # error on conflicts. With `--force` it will take the lowest possible offset for each
+        # topic partition as the baseline.
+        #
+        # @note Running this can cause you some double processing if the parallel segments final
+        #   offsets are not aligned.
+        #
+        # @note This will **not** remove the parallel segments consumer groups. Please use the
+        #   Admin API if you want them to be removed.
+        class Collapse < Base
+          # Runs the collapse operation
+          def call
+            puts 'Starting parallel segments collapse...'
+
+            segments_count = applicable_groups.size
+
+            if segments_count.zero?
+              puts "#{red('No')} consumer groups with parallel segments configuration found"
+
+              return
+            end
+
+            puts(
+              "Found #{green(segments_count)} consumer groups with parallel segments configuration"
+            )
+
+            collapses = []
+
+            applicable_groups.each do |segment_origin, segments|
+              puts
+              puts "Collecting group #{yellow(segment_origin)} details..."
+              offsets = collect_offsets(segment_origin, segments)
+
+              unless options.key?(:force)
+                puts
+                puts "Validating offsets positions for #{yellow(segment_origin)} consumer group..."
+                validate!(offsets, segment_origin)
+              end
+
+              puts
+              puts "Computing collapsed offsets for #{yellow(segment_origin)} consumer group..."
+              collapses << collapse(offsets, segments)
+            end
+
+            collapses.each do |collapse|
+              apply(collapse)
+            end
+
+            puts
+            puts "Collapse completed #{green('successfully')}!"
+          end
+
+          private
+
+          # Computes the lowest possible offset available for each topic partition and sets it
+          # on the segment origin consumer group.
+          #
+          # @param offsets [Hash]
+          # @param segments [Array<Karafka::Routing::ConsumerGroup>]
+          # @note This code does **not** apply the offsets, just computes their positions
+          def collapse(offsets, segments)
+            collapse = Hash.new { |h, k| h[k] = {} }
+            segments_names = segments.map(&:name)
+
+            offsets.each do |cg_name, topics|
+              next unless segments_names.include?(cg_name)
+
+              topics.each do |topic_name, partitions|
+                partitions.each do |partition_id, offset|
+                  current_lowest_offset = collapse[topic_name][partition_id]
+
+                  next if current_lowest_offset && current_lowest_offset < offset
+
+                  collapse[topic_name][partition_id] = offset
+                end
+              end
+            end
+
+            {
+              collapse: collapse,
+              segment_origin: segments.first.segment_origin
+            }
+          end
+
+          # In order to collapse the offsets of parallel segments back to one, we need to know
+          # to what offsets to collapse. The issue (that we solve picking lowest when forced)
+          # arises when there are more offsets that are not even in parallel segments for one
+          # topic partition. We should let user know about this if this happens so he does not
+          # end up with double-processing.
+          #
+          # @param offsets [Hash]
+          # @param segment_origin [String]
+          def validate!(offsets, segment_origin)
+            collapse = Hash.new { |h, k| h[k] = {} }
+
+            offsets.each do |cg_name, topics|
+              next if cg_name == segment_origin
+
+              topics.each do |topic_name, partitions|
+                partitions.each do |partition_id, offset|
+                  collapse[topic_name][partition_id] ||= Set.new
+                  collapse[topic_name][partition_id] << offset
+                end
+              end
+            end
+
+            inconclusive = false
+
+            collapse.each do |topic_name, partitions|
+              partitions.each do |partition_id, parallel_offsets|
+                next if parallel_offsets.size <= 1
+
+                inconclusive = true
+
+                puts(
+                  "  Inconclusive offsets for #{red(topic_name)}##{red(partition_id)}:" \
+                  " #{parallel_offsets.to_a.join(', ')}"
+                )
+              end
+            end
+
+            return unless inconclusive
+
+            raise(
+              ::Karafka::Errors::CommandValidationError,
+              "Parallel segments for #{red(segment_origin)} have #{red('inconclusive')} offsets"
+            )
+          end
+
+          # Applies the collapsed lowest offsets onto the segment origin consumer group
+          #
+          # @param collapse [Hash]
+          def apply(collapse)
+            segment_origin = collapse[:segment_origin]
+            alignments = collapse[:collapse]
+
+            puts
+            puts "Adjusting offsets of segment origin consumer group: #{green(segment_origin)}"
+
+            alignments.each do |topic_name, partitions|
+              puts "  Topic #{green(topic_name)}:"
+
+              partitions.each do |partition_id, offset|
+                puts "    Partition #{green(partition_id)}: starting offset #{green(offset)}"
+              end
+            end
+
+            Karafka::Admin.seek_consumer_group(segment_origin, alignments)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cli/parallel_segments/distribute.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Cli
+      class ParallelSegments < Karafka::Cli::Base
+        # Command that makes it easier for users to migrate from regular consumer groups to
+        # the parallel segments consumers groups by automatically distributing offsets based on
+        # the used "normal" consumer group.
+        #
+        # Takes the segments origin consumer group offsets for a given set of topics and
+        # distributes those offsets onto the parallel segments consumer groups, so they can pick
+        # up where the origin group left.
+        #
+        # To make sure users do not accidentally "re-distribute" their offsets from the original
+        # consumer group after the parallel consumer groups had offsets assigned and started to
+        # work, we check if the parallel groups have any offsets, if so unless forced we halt.
+        #
+        # @note This command does not remove the original consumer group from Kafka. We keep it
+        #   just as a backup. User can remove it himself.
+        #
+        # @note Kafka has no atomic operations this is why we first collect all the data and run
+        #   needed validations before applying offsets.
+        class Distribute < Base
+          # Runs the distribution process
+          def call
+            puts 'Starting parallel segments distribution...'
+
+            segments_count = applicable_groups.size
+
+            if segments_count.zero?
+              puts "#{red('No')} consumer groups with parallel segments configuration found"
+
+              return
+            end
+
+            puts(
+              "Found #{green(segments_count)} consumer groups with parallel segments configuration"
+            )
+
+            distributions = []
+
+            applicable_groups.each do |segment_origin, segments|
+              puts
+              puts "Collecting group #{yellow(segment_origin)} details..."
+              offsets = collect_offsets(segment_origin, segments)
+
+              unless options.key?(:force)
+                puts "Validating group #{yellow(segment_origin)} parallel segments..."
+                validate!(offsets, segments)
+              end
+
+              puts "Distributing group #{yellow(segment_origin)} offsets..."
+              distributions += distribute(offsets, segments)
+            end
+
+            distributions.each do |distribution|
+              apply(distribution)
+            end
+
+            puts
+            puts "Distribution completed #{green('successfully')}!"
+          end
+
+          private
+
+          # Validates the current state of topics offsets assignments.
+          # We want to make sure, that users do not run distribution twice, especially for a
+          # parallel segments consumers group set that was already actively consumed. This is why
+          # we check if there was any offsets already present in the parallel segments consumer
+          # groups and if so, we raise an error. This can be disabled with `--force`.
+          #
+          # It prevents users from overwriting the already set segments distribution.
+          # Adding new topics to the same parallel segments consumer group does not require us to
+          # run this at all and on top of that users can always use `--consumer_groups` flag to
+          # limit the cgs that we will be operating here
+          #
+          # @param offsets [Hash]
+          # @param segments [Array<Karafka::Routing::ConsumerGroup>]
+          def validate!(offsets, segments)
+            segments_names = segments.map(&:name)
+
+            offsets.each do |cg_name, topics|
+              next unless segments_names.include?(cg_name)
+
+              topics.each do |topic_name, partitions|
+                partitions.each do |partition_id, offset|
+                  next unless offset.to_i.positive?
+
+                  raise(
+                    ::Karafka::Errors::CommandValidationError,
+                    "Parallel segment #{red(cg_name)} already has offset #{red(offset)}" \
+                    " set for #{red("#{topic_name}##{partition_id}")}"
+                  )
+                end
+              end
+            end
+          end
+
+          # Computes the offsets distribution for all the segments consumer groups so when user
+          # migrates from one CG to parallel segments, those segments know where to start consuming
+          # the data.
+          #
+          # @param offsets [Hash]
+          # @param segments [Array<Karafka::Routing::ConsumerGroup>]
+          # @note This code does **not** apply the offsets, just computes their positions
+          def distribute(offsets, segments)
+            distributions = []
+            segments_names = segments.map(&:name)
+
+            offsets.each do |cg_name, topics|
+              next if segments_names.include?(cg_name)
+
+              distribution = {}
+
+              topics.each do |topic_name, partitions|
+                partitions.each do |partition_id, offset|
+                  distribution[topic_name] ||= {}
+                  distribution[topic_name][partition_id] = offset
+                end
+              end
+
+              next if distribution.empty?
+
+              segments_names.each do |segment_name|
+                distributions << {
+                  segment_name: segment_name,
+                  distribution: distribution
+                }
+              end
+            end
+
+            distributions
+          end
+
+          # Takes the details of the distribution of offsets for a given segment and adjust the
+          # starting offsets for all the consumer group topics based on the distribution.
+          #
+          # @param distribution [Hash]
+          def apply(distribution)
+            segment_name = distribution[:segment_name]
+            alignments = distribution[:distribution]
+
+            puts
+            puts "Adjusting offsets of parallel segments consumer group: #{green(segment_name)}"
+
+            alignments.each do |topic_name, partitions|
+              puts "  Topic #{green(topic_name)}:"
+
+              partitions.each do |partition_id, offset|
+                puts "    Partition #{green(partition_id)}: starting offset #{green(offset)}"
+              end
+            end
+
+            Karafka::Admin.seek_consumer_group(segment_name, alignments)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cli/parallel_segments.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    # Pro related CLI commands
+    module Cli
+      # CLI entry-point for parallel segments management commands
+      class ParallelSegments < Karafka::Cli::Base
+        include Helpers::Colorize
+        include Helpers::ConfigImporter.new(
+          kafka_config: %i[kafka]
+        )
+
+        desc 'Allows for parallel segments management'
+
+        option(
+          :groups,
+          'Names of consumer groups on which we want to run the command. All if not provided',
+          Array,
+          %w[
+            --groups
+            --consumer_groups
+          ]
+        )
+
+        # Some operations may not be allowed to run again after data is set in certain ways.
+        # For example if a distribution command is invoked when the parallel group segment
+        # consumer groups already have offsets set, we will fail unless user wants to force it.
+        # This prevents users from accidentally running the command in such ways that would cause
+        # their existing distributed offsets to be reset.
+        option(
+          :force,
+          'Should an operation on the parallel segments consumer group be forced',
+          TrueClass,
+          %w[
+            --force
+          ]
+        )
+
+        # @param action [String] action we want to take
+        def call(action = 'distribute')
+          case action
+          when 'distribute'
+            Distribute.new(options).call
+          when 'collapse'
+            Collapse.new(options).call
+          when 'reset'
+            Collapse.new(options).call
+            Distribute.new(options).call
+          else
+            raise ::ArgumentError, "Invalid topics action: #{action}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/connection/manager.rb

@@ -19,14 +19,9 @@ module Karafka
       class Manager < Karafka::Connection::Manager
         include Core::Helpers::Time
 
-        # How long should we wait after a rebalance before doing anything on a consumer group
-        #
-        # @param scale_delay [Integer] How long should we wait before making any changes. Any
-        #   change related to this consumer group will postpone the scaling operations. This is
-        #   done that way to prevent too many friction in the cluster. It is 1 minute by default
-        def initialize(scale_delay = 60 * 1_000)
+        # Creates new manager instance
+        def initialize
           super()
-          @scale_delay = scale_delay
           @mutex = Mutex.new
           @changes = Hash.new do |h, k|
             h[k] = {
@@ -201,7 +196,7 @@ module Karafka
 
             next unless multiplexing.active?
             next unless multiplexing.dynamic?
-            # If we cannot downscale, do not
+            # If we cannot upscale, do not
             next if sg_listeners.count(&:active?) >= multiplexing.max
 
             sg_listeners.each do |sg_listener|
@@ -234,6 +229,8 @@ module Karafka
         #   are also stable. This is a strong indicator that no rebalances or other operations are
         #   happening at a given moment.
         def stable?(sg_listeners)
+          @scale_delay ||= sg_listeners.first.subscription_group.multiplexing.scale_delay
+
           sg_listeners.all? do |sg_listener|
             # If a listener is not active, we do not take it into consideration when looking at
             # the stability data