karafka 2.4.18 → 2.5.0.beta2

data/lib/karafka/pro/cli/parallel_segments/collapse.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Cli
+      class ParallelSegments < Karafka::Cli::Base
+        # Takes the committed offset of each parallel segment for each topic and records
+        # them back onto the segment origin consumer group. Without `--force` it will raise an
+        # error on conflicts. With `--force` it will take the lowest possible offset for each
+        # topic partition as the baseline.
+        #
+        # @note Running this can cause you some double processing if the parallel segments final
+        #   offsets are not aligned.
+        #
+        # @note This will **not** remove the parallel segments consumer groups. Please use the
+        #   Admin API if you want them to be removed.
+        class Collapse < Base
+          # Runs the collapse operation
+          def call
+            puts 'Starting parallel segments collapse...'
+
+            segments_count = applicable_groups.size
+
+            if segments_count.zero?
+              puts "#{red('No')} consumer groups with parallel segments configuration found"
+
+              return
+            end
+
+            puts(
+              "Found #{green(segments_count)} consumer groups with parallel segments configuration"
+            )
+
+            collapses = []
+
+            applicable_groups.each do |segment_origin, segments|
+              puts
+              puts "Collecting group #{yellow(segment_origin)} details..."
+              offsets = collect_offsets(segment_origin, segments)
+
+              unless options.key?(:force)
+                puts
+                puts "Validating offsets positions for #{yellow(segment_origin)} consumer group..."
+                validate!(offsets, segment_origin)
+              end
+
+              puts
+              puts "Computing collapsed offsets for #{yellow(segment_origin)} consumer group..."
+              collapses << collapse(offsets, segments)
+            end
+
+            collapses.each do |collapse|
+              apply(collapse)
+            end
+
+            puts
+            puts "Collapse completed #{green('successfully')}!"
+          end
+
+          private
+
+          # Computes the lowest possible offset available for each topic partition and sets it
+          # on the segment origin consumer group.
+          #
+          # @param offsets [Hash]
+          # @param segments [Array<Karafka::Routing::ConsumerGroup>]
+          # @note This code does **not** apply the offsets, just computes their positions
+          def collapse(offsets, segments)
+            collapse = Hash.new { |h, k| h[k] = {} }
+            segments_names = segments.map(&:name)
+
+            offsets.each do |cg_name, topics|
+              next unless segments_names.include?(cg_name)
+
+              topics.each do |topic_name, partitions|
+                partitions.each do |partition_id, offset|
+                  current_lowest_offset = collapse[topic_name][partition_id]
+
+                  next if current_lowest_offset && current_lowest_offset < offset
+
+                  collapse[topic_name][partition_id] = offset
+                end
+              end
+            end
+
+            {
+              collapse: collapse,
+              segment_origin: segments.first.segment_origin
+            }
+          end
+
+          # In order to collapse the offsets of parallel segments back to one, we need to know
+          # to what offsets to collapse. The issue (that we solve picking lowest when forced)
+          # arises when there are more offsets that are not even in parallel segments for one
+          # topic partition. We should let user know about this if this happens so he does not
+          # end up with double-processing.
+          #
+          # @param offsets [Hash]
+          # @param segment_origin [String]
+          def validate!(offsets, segment_origin)
+            collapse = Hash.new { |h, k| h[k] = {} }
+
+            offsets.each do |cg_name, topics|
+              next if cg_name == segment_origin
+
+              topics.each do |topic_name, partitions|
+                partitions.each do |partition_id, offset|
+                  collapse[topic_name][partition_id] ||= Set.new
+                  collapse[topic_name][partition_id] << offset
+                end
+              end
+            end
+
+            inconclusive = false
+
+            collapse.each do |topic_name, partitions|
+              partitions.each do |partition_id, parallel_offsets|
+                next if parallel_offsets.size <= 1
+
+                inconclusive = true
+
+                puts(
+                  "  Inconclusive offsets for #{red(topic_name)}##{red(partition_id)}:" \
+                  " #{parallel_offsets.to_a.join(', ')}"
+                )
+              end
+            end
+
+            return unless inconclusive
+
+            raise(
+              ::Karafka::Errors::CommandValidationError,
+              "Parallel segments for #{red(segment_origin)} have #{red('inconclusive')} offsets"
+            )
+          end
+
+          # Applies the collapsed lowest offsets onto the segment origin consumer group
+          #
+          # @param collapse [Hash]
+          def apply(collapse)
+            segment_origin = collapse[:segment_origin]
+            alignments = collapse[:collapse]
+
+            puts
+            puts "Adjusting offsets of segment origin consumer group: #{green(segment_origin)}"
+
+            alignments.each do |topic_name, partitions|
+              puts "  Topic #{green(topic_name)}:"
+
+              partitions.each do |partition_id, offset|
+                puts "    Partition #{green(partition_id)}: starting offset #{green(offset)}"
+              end
+            end
+
+            Karafka::Admin.seek_consumer_group(segment_origin, alignments)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cli/parallel_segments/distribute.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Cli
+      class ParallelSegments < Karafka::Cli::Base
+        # Command that makes it easier for users to migrate from regular consumer groups to
+        # the parallel segments consumers groups by automatically distributing offsets based on
+        # the used "normal" consumer group.
+        #
+        # Takes the segments origin consumer group offsets for a given set of topics and
+        # distributes those offsets onto the parallel segments consumer groups, so they can pick
+        # up where the origin group left.
+        #
+        # To make sure users do not accidentally "re-distribute" their offsets from the original
+        # consumer group after the parallel consumer groups had offsets assigned and started to
+        # work, we check if the parallel groups have any offsets, if so unless forced we halt.
+        #
+        # @note This command does not remove the original consumer group from Kafka. We keep it
+        #   just as a backup. User can remove it himself.
+        #
+        # @note Kafka has no atomic operations this is why we first collect all the data and run
+        #   needed validations before applying offsets.
+        class Distribute < Base
+          # Runs the distribution process
+          def call
+            puts 'Starting parallel segments distribution...'
+
+            segments_count = applicable_groups.size
+
+            if segments_count.zero?
+              puts "#{red('No')} consumer groups with parallel segments configuration found"
+
+              return
+            end
+
+            puts(
+              "Found #{green(segments_count)} consumer groups with parallel segments configuration"
+            )
+
+            distributions = []
+
+            applicable_groups.each do |segment_origin, segments|
+              puts
+              puts "Collecting group #{yellow(segment_origin)} details..."
+              offsets = collect_offsets(segment_origin, segments)
+
+              unless options.key?(:force)
+                puts "Validating group #{yellow(segment_origin)} parallel segments..."
+                validate!(offsets, segments)
+              end
+
+              puts "Distributing group #{yellow(segment_origin)} offsets..."
+              distributions += distribute(offsets, segments)
+            end
+
+            distributions.each do |distribution|
+              apply(distribution)
+            end
+
+            puts
+            puts "Distribution completed #{green('successfully')}!"
+          end
+
+          private
+
+          # Validates the current state of topics offsets assignments.
+          # We want to make sure, that users do not run distribution twice, especially for a
+          # parallel segments consumers group set that was already actively consumed. This is why
+          # we check if there was any offsets already present in the parallel segments consumer
+          # groups and if so, we raise an error. This can be disabled with `--force`.
+          #
+          # It prevents users from overwriting the already set segments distribution.
+          # Adding new topics to the same parallel segments consumer group does not require us to
+          # run this at all and on top of that users can always use `--consumer_groups` flag to
+          # limit the cgs that we will be operating here
+          #
+          # @param offsets [Hash]
+          # @param segments [Array<Karafka::Routing::ConsumerGroup>]
+          def validate!(offsets, segments)
+            segments_names = segments.map(&:name)
+
+            offsets.each do |cg_name, topics|
+              next unless segments_names.include?(cg_name)
+
+              topics.each do |topic_name, partitions|
+                partitions.each do |partition_id, offset|
+                  next unless offset.to_i.positive?
+
+                  raise(
+                    ::Karafka::Errors::CommandValidationError,
+                    "Parallel segment #{red(cg_name)} already has offset #{red(offset)}" \
+                    " set for #{red("#{topic_name}##{partition_id}")}"
+                  )
+                end
+              end
+            end
+          end
+
+          # Computes the offsets distribution for all the segments consumer groups so when user
+          # migrates from one CG to parallel segments, those segments know where to start consuming
+          # the data.
+          #
+          # @param offsets [Hash]
+          # @param segments [Array<Karafka::Routing::ConsumerGroup>]
+          # @note This code does **not** apply the offsets, just computes their positions
+          def distribute(offsets, segments)
+            distributions = []
+            segments_names = segments.map(&:name)
+
+            offsets.each do |cg_name, topics|
+              next if segments_names.include?(cg_name)
+
+              distribution = {}
+
+              topics.each do |topic_name, partitions|
+                partitions.each do |partition_id, offset|
+                  distribution[topic_name] ||= {}
+                  distribution[topic_name][partition_id] = offset
+                end
+              end
+
+              next if distribution.empty?
+
+              segments_names.each do |segment_name|
+                distributions << {
+                  segment_name: segment_name,
+                  distribution: distribution
+                }
+              end
+            end
+
+            distributions
+          end
+
+          # Takes the details of the distribution of offsets for a given segment and adjust the
+          # starting offsets for all the consumer group topics based on the distribution.
+          #
+          # @param distribution [Hash]
+          def apply(distribution)
+            segment_name = distribution[:segment_name]
+            alignments = distribution[:distribution]
+
+            puts
+            puts "Adjusting offsets of parallel segments consumer group: #{green(segment_name)}"
+
+            alignments.each do |topic_name, partitions|
+              puts "  Topic #{green(topic_name)}:"
+
+              partitions.each do |partition_id, offset|
+                puts "    Partition #{green(partition_id)}: starting offset #{green(offset)}"
+              end
+            end
+
+            Karafka::Admin.seek_consumer_group(segment_name, alignments)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cli/parallel_segments.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    # Pro related CLI commands
+    module Cli
+      # CLI entry-point for parallel segments management commands
+      class ParallelSegments < Karafka::Cli::Base
+        include Helpers::Colorize
+        include Helpers::ConfigImporter.new(
+          kafka_config: %i[kafka]
+        )
+
+        desc 'Allows for parallel segments management'
+
+        option(
+          :groups,
+          'Names of consumer groups on which we want to run the command. All if not provided',
+          Array,
+          %w[
+            --groups
+            --consumer_groups
+          ]
+        )
+
+        # Some operations may not be allowed to run again after data is set in certain ways.
+        # For example if a distribution command is invoked when the parallel group segment
+        # consumer groups already have offsets set, we will fail unless user wants to force it.
+        # This prevents users from accidentally running the command in such ways that would cause
+        # their existing distributed offsets to be reset.
+        option(
+          :force,
+          'Should an operation on the parallel segments consumer group be forced',
+          TrueClass,
+          %w[
+            --force
+          ]
+        )
+
+        # @param action [String] action we want to take
+        def call(action = 'distribute')
+          case action
+          when 'distribute'
+            Distribute.new(options).call
+          when 'collapse'
+            Collapse.new(options).call
+          when 'reset'
+            Collapse.new(options).call
+            Distribute.new(options).call
+          else
+            raise ::ArgumentError, "Invalid topics action: #{action}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/connection/manager.rb

@@ -19,14 +19,9 @@ module Karafka
       class Manager < Karafka::Connection::Manager
         include Core::Helpers::Time
 
-        # How long should we wait after a rebalance before doing anything on a consumer group
-        #
-        # @param scale_delay [Integer] How long should we wait before making any changes. Any
-        #   change related to this consumer group will postpone the scaling operations. This is
-        #   done that way to prevent too many friction in the cluster. It is 1 minute by default
-        def initialize(scale_delay = 60 * 1_000)
+        # Creates new manager instance
+        def initialize
           super()
-          @scale_delay = scale_delay
           @mutex = Mutex.new
           @changes = Hash.new do |h, k|
             h[k] = {
@@ -201,7 +196,7 @@ module Karafka
 
             next unless multiplexing.active?
             next unless multiplexing.dynamic?
-            # If we cannot downscale, do not
+            # If we cannot upscale, do not
             next if sg_listeners.count(&:active?) >= multiplexing.max
 
             sg_listeners.each do |sg_listener|
@@ -234,6 +229,8 @@ module Karafka
         #   are also stable. This is a strong indicator that no rebalances or other operations are
         #   happening at a given moment.
         def stable?(sg_listeners)
+          @scale_delay ||= sg_listeners.first.subscription_group.multiplexing.scale_delay
+
           sg_listeners.all? do |sg_listener|
             # If a listener is not active, we do not take it into consideration when looking at
             # the stability data

data/lib/karafka/pro/encryption.rb

@@ -33,6 +33,14 @@ module Karafka
           # Encryption for WaterDrop
           config.producer.middleware.append(Messages::Middleware.new)
         end
+
+        # This feature does not need any changes post-fork
+        #
+        # @param _config [Karafka::Core::Configurable::Node]
+        # @param _pre_fork_producer [WaterDrop::Producer]
+        def post_fork(_config, _pre_fork_producer)
+          true
+        end
       end
     end
   end

data/lib/karafka/pro/instrumentation/performance_tracker.rb

@@ -50,7 +50,7 @@ module Karafka
           partition = messages.metadata.partition
 
           samples = @processing_times[topic][partition]
-          samples << event[:time] / messages.count
+          samples << event[:time] / messages.size
 
           return unless samples.size > SAMPLES_COUNT
 

data/lib/karafka/pro/iterator/expander.rb

@@ -21,8 +21,10 @@ module Karafka
       # - { 'topic1' => 100 } - means we run all partitions from the offset 100
       # - { 'topic1' => Time.now - 60 } - we run all partitions from the message from 60s ago
       # - { 'topic1' => { 1 => Time.now - 60 } } - partition1 from message 60s ago
-      # - { 'topic1' => { 1 => true } } - will pick first offset not consumed on this CG for p 1
-      # - { 'topic1' => true } - will pick first offset not consumed on this CG for all p
+      # - { 'topic1' => { 1 => true } } - will pick first offset on this CG for partition 1
+      # - { 'topic1' => true } - will pick first offset for all partitions
+      # - { 'topic1' => :earliest } - will pick earliest offset for all partitions
+      # - { 'topic1' => :latest } - will pick latest (high-watermark) for all partitions
       class Expander
         # Expands topics to which we want to subscribe with partitions information in case this
         # info is not provided.
@@ -80,7 +82,7 @@ module Karafka
             .find { |topic| topic.fetch(:topic_name) == name }
             .tap { |topic| topic || raise(Errors::TopicNotFoundError, name) }
             .fetch(:partitions)
-            .count
+            .size
         end
       end
     end

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -14,6 +14,11 @@ module Karafka
       # This builder resolves that and builds a tpl to which we can safely subscribe the way
       # we want it.
       class TplBuilder
+        # Supported named offset positions that we can reference via their name
+        SUPPORTED_NAMED_POSITIONS = %w[earliest latest].freeze
+
+        private_constant :SUPPORTED_NAMED_POSITIONS
+
         # @param consumer [::Rdkafka::Consumer] consumer instance needed to talk with Kafka
         # @param expanded_topics [Hash] hash with expanded and normalized topics data
         def initialize(consumer, expanded_topics)
@@ -28,6 +33,7 @@ module Karafka
           resolve_partitions_with_exact_offsets
           resolve_partitions_with_negative_offsets
           resolve_partitions_with_time_offsets
+          resolve_partitions_with_named_offsets
           resolve_partitions_with_cg_expectations
 
           # Final tpl with all the data
@@ -143,6 +149,23 @@ module Karafka
           end
         end
 
+        # If we get named offsets, we can just remap them to librdkafka special offset positions
+        def resolve_partitions_with_named_offsets
+          @expanded_topics.each do |name, partitions|
+            next unless partitions.is_a?(Hash)
+
+            partitions.each do |partition, offset|
+              # Skip offsets that do not match our named expectations
+              named_offset = offset.to_s
+
+              next unless SUPPORTED_NAMED_POSITIONS.include?(named_offset)
+
+              @mapped_topics[name][partition] = -1 if named_offset == 'latest'
+              @mapped_topics[name][partition] = -2 if named_offset == 'earliest'
+            end
+          end
+        end
+
         # Fetches last used offsets for those partitions for which we want to consume from last
         # moment where given consumer group has finished
         # This is indicated by given partition value being set to `true`.

data/lib/karafka/pro/loader.rb

@@ -60,6 +60,15 @@ module Karafka
           Processing::SubscriptionGroupsCoordinator.instance
         end
 
+        # Runs operations needed after fork in swarm for features that need it
+        #
+        # @param config [Karafka::Core::Configurable::Node]
+        # @param pre_fork_producer [WaterDrop::Producer] pre fork producer instance that may be
+        #   needed to be replaced with newly changed one post-fork.
+        def post_fork(config, pre_fork_producer)
+          features.each { |feature| feature.post_fork(config, pre_fork_producer) }
+        end
+
         private
 
         # @return [Array<Module>] extra non-routing related pro features and routing components
@@ -84,6 +93,7 @@ module Karafka
           icfg.connection.manager = Connection::Manager.new
 
           icfg.processing.coordinator_class = Processing::Coordinator
+          icfg.processing.errors_tracker_class = Processing::Coordinators::ErrorsTracker
           icfg.processing.partitioner_class = Processing::Partitioner
           icfg.processing.scheduler_class = Processing::Schedulers::Default
           icfg.processing.jobs_queue_class = Processing::JobsQueue

data/lib/karafka/pro/processing/coordinator.rb

@@ -10,6 +10,9 @@ module Karafka
       # within the same partition
       class Coordinator < ::Karafka::Processing::Coordinator
         extend Forwardable
+        include Helpers::ConfigImporter.new(
+          errors_tracker_class: %i[internal processing errors_tracker_class]
+        )
 
         def_delegators :@collapser, :collapsed?, :collapse_until!
 
@@ -20,7 +23,7 @@ module Karafka
           super
 
           @executed = []
-          @errors_tracker = Coordinators::ErrorsTracker.new
+          @errors_tracker = errors_tracker_class.new(topic, partition)
           @flow_mutex = Mutex.new
           # Lock for user code synchronization
           # We do not want to mix coordinator lock with the user lock not to create cases where

data/lib/karafka/pro/processing/coordinators/errors_tracker.rb

@@ -13,25 +13,47 @@ module Karafka
         class ErrorsTracker
           include Enumerable
 
+          # @return [Karafka::Routing::Topic] topic of this error tracker
+          attr_reader :topic
+
+          # @return [Integer] partition of this error tracker
+          attr_reader :partition
+
+          # @return [Hash]
+          attr_reader :counts
+
           # Max errors we keep in memory.
           # We do not want to keep more because for DLQ-less this would cause memory-leaks.
+          # We do however count per class for granular error counting
           STORAGE_LIMIT = 100
 
           private_constant :STORAGE_LIMIT
 
-          def initialize
+          # @param topic [Karafka::Routing::Topic]
+          # @param partition [Integer]
+          # @param limit [Integer] max number of errors we want to keep for reference when
+          #   implementing custom error handling.
+          # @note `limit` does not apply to the counts. They will work beyond the number of errors
+          #   occurring
+          def initialize(topic, partition, limit: STORAGE_LIMIT)
             @errors = []
+            @counts = Hash.new { |hash, key| hash[key] = 0 }
+            @topic = topic
+            @partition = partition
+            @limit = limit
           end
 
           # Clears all the errors
           def clear
             @errors.clear
+            @counts.clear
           end
 
           # @param error [StandardError] adds the error to the tracker
           def <<(error)
-            @errors.shift if @errors.size >= STORAGE_LIMIT
+            @errors.shift if @errors.size >= @limit
             @errors << error
+            @counts[error.class] += 1
           end
 
           # @return [Boolean] is the error tracker empty
@@ -41,7 +63,9 @@ module Karafka
 
           # @return [Integer] number of elements
           def size
-            count
+            # We use counts reference of all errors and not the `@errors` array because it allows
+            # us to go beyond the whole errors storage limit
+            @counts.values.sum
           end
 
           # @return [StandardError, nil] last error that occurred or nil if no errors

data/lib/karafka/pro/processing/coordinators/filters_applier.rb

@@ -98,6 +98,17 @@ module Karafka
             :mark_as_consumed
           end
 
+          # The first (lowest) message we want to mark as consumed in marking. By default it uses
+          #   same position as cursor in case user wants to mark same message as consumed as the
+          #   one on which cursor action is applied.
+          # @return [Karafka::Messages::Message, nil] cursor marking message or nil if none
+          # @note It should not return position in time format, only numerical offset
+          def marking_cursor
+            return nil unless active?
+
+            applied.map(&:marking_cursor).compact.min_by(&:offset)
+          end
+
           private
 
           # @return [Boolean] is filtering active

data/lib/karafka/pro/processing/filters/base.rb

@@ -42,9 +42,11 @@ module Karafka
             @applied
           end
 
-          # @return [Integer] default timeout for pausing (if applicable)
+          # @return [Integer, nil] default timeout for pausing (if applicable) or nil if not
+          # @note Please do not return `0` when your filter is not pausing as it may interact
+          #   with other filters that want to pause.
           def timeout
-            0
+            nil
           end
 
           # @return [Boolean] should we use the cursor value to mark as consumed. If any of the
@@ -58,6 +60,12 @@ module Karafka
           def marking_method
             :mark_as_consumed
           end
+
+          # @return [Karafka::Messages::Message, nil] cursor message for marking or nil if no
+          #   marking
+          def marking_cursor
+            cursor
+          end
         end
       end
     end

data/lib/karafka/pro/processing/filters/expirer.rb

@@ -36,6 +36,11 @@ module Karafka
               too_old
             end
           end
+
+          # @return [nil] this filter does not deal with timeouts
+          def timeout
+            nil
+          end
         end
       end
     end