karafka 2.4.18 → 2.5.0.beta2

data/lib/karafka/admin/acl.rb

@@ -11,6 +11,10 @@ module Karafka
     # This API works based on ability to create a `Karafka:Admin::Acl` object that can be then used
     # using `#create`, `#delete` and `#describe` class API.
     class Acl
+      extend Helpers::ConfigImporter.new(
+        max_wait_time: %i[admin max_wait_time]
+      )
+
       # Types of resources for which we can assign permissions.
       #
       # Resource refers to any entity within the Kafka ecosystem for which access control can be
@@ -162,7 +166,7 @@ module Karafka
         # Makes sure that admin is closed afterwards.
         def with_admin_wait
           Admin.with_admin do |admin|
-            yield(admin).wait(max_wait_timeout: Karafka::App.config.admin.max_wait_time)
+            yield(admin).wait(max_wait_timeout: max_wait_time)
           end
         end
 

data/lib/karafka/admin/configs.rb

@@ -10,6 +10,10 @@ module Karafka
     #
     # Altering is done in the incremental way.
     module Configs
+      extend Helpers::ConfigImporter.new(
+        max_wait_time: %i[admin max_wait_time]
+      )
+
       class << self
         # Fetches given resources configurations from Kafka
         #
@@ -94,7 +98,7 @@ module Karafka
         # Makes sure that admin is closed afterwards.
         def with_admin_wait
           Admin.with_admin do |admin|
-            yield(admin).wait(max_wait_timeout: Karafka::App.config.admin.max_wait_time)
+            yield(admin).wait(max_wait_timeout: max_wait_time)
           end
         end
       end

data/lib/karafka/admin.rb

@@ -10,11 +10,25 @@ module Karafka
   #   Cluster on which operations are performed can be changed via `admin.kafka` config, however
   #   there is no multi-cluster runtime support.
   module Admin
-    # More or less number of seconds of 1 hundred years
-    # Used for time referencing that does not have to be accurate but needs to be big
-    HUNDRED_YEARS = 100 * 365.25 * 24 * 60 * 60
-
-    private_constant :HUNDRED_YEARS
+    extend Helpers::ConfigImporter.new(
+      max_wait_time: %i[admin max_wait_time],
+      poll_timeout: %i[admin poll_timeout],
+      max_attempts: %i[admin max_attempts],
+      group_id: %i[admin group_id],
+      app_kafka: %i[kafka],
+      admin_kafka: %i[admin kafka]
+    )
+
+    # 2010-01-01 00:00:00 - way before Kafka was released so no messages should exist prior to
+    # this date
+    # We do not use the explicit -2 librdkafka value here because we resolve this offset without
+    # consuming data
+    LONG_TIME_AGO = Time.at(1_262_300_400)
+
+    # one day in seconds for future time reference
+    DAY_IN_SECONDS = 60 * 60 * 24
+
+    private_constant :LONG_TIME_AGO, :DAY_IN_SECONDS
 
     class << self
       # Allows us to read messages from the topic
@@ -55,7 +69,7 @@ module Karafka
           possible_range = requested_range.select { |offset| available_range.include?(offset) }
 
           start_offset = possible_range.first
-          count = possible_range.count
+          count = possible_range.size
 
           tpl.add_topic_and_partitions_with_offsets(name, partition => start_offset)
           consumer.assign(tpl)
@@ -108,7 +122,7 @@ module Karafka
           handler = admin.create_topic(name, partitions, replication_factor, topic_config)
 
           with_re_wait(
-            -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
+            -> { handler.wait(max_wait_timeout: max_wait_time) },
             -> { topics_names.include?(name) }
           )
         end
@@ -122,7 +136,7 @@ module Karafka
           handler = admin.delete_topic(name)
 
           with_re_wait(
-            -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
+            -> { handler.wait(max_wait_timeout: max_wait_time) },
             -> { !topics_names.include?(name) }
           )
         end
@@ -137,7 +151,7 @@ module Karafka
           handler = admin.create_partitions(name, partitions)
 
           with_re_wait(
-            -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
+            -> { handler.wait(max_wait_timeout: max_wait_time) },
             -> { topic_info(name).fetch(:partition_count) >= partitions }
           )
         end
@@ -203,14 +217,14 @@ module Karafka
             # Earliest is not always 0. When compacting/deleting it can be much later, that's why
             # we fetch the oldest possible offset
             when 'earliest'
-              Time.now - HUNDRED_YEARS
+              LONG_TIME_AGO
             # Latest will always be the high-watermark offset and we can get it just by getting
             # a future position
             when 'latest'
-              Time.now + HUNDRED_YEARS
-            # Same as `'latest'`
+              Time.now + DAY_IN_SECONDS
+            # Same as `'earliest'`
             when false
-              Time.now - HUNDRED_YEARS
+              LONG_TIME_AGO
             # Regular offset case
             else
               position
@@ -274,27 +288,24 @@ module Karafka
         end
       end
 
-      # Takes consumer group and its topics and migrates all the offsets to a new named group
+      # Takes consumer group and its topics and copies all the offsets to a new named group
       #
       # @param previous_name [String] old consumer group name
       # @param new_name [String] new consumer group name
       # @param topics [Array<String>] topics for which we want to migrate offsets during rename
-      # @param delete_previous [Boolean] should we delete previous consumer group after rename.
-      #   Defaults to true.
+      # @return [Boolean] true if anything was migrated, otherwise false
       #
       # @note This method should **not** be executed on a running consumer group as it creates a
       #   "fake" consumer and uses it to move offsets.
       #
-      # @note After migration unless `delete_previous` is set to `false`, old group will be
-      #   removed.
-      #
       # @note If new consumer group exists, old offsets will be added to it.
-      def rename_consumer_group(previous_name, new_name, topics, delete_previous: true)
+      def copy_consumer_group(previous_name, new_name, topics)
         remap = Hash.new { |h, k| h[k] = {} }
 
         old_lags = read_lags_with_offsets({ previous_name => topics })
 
-        return if old_lags.empty?
+        return false if old_lags.empty?
+        return false if old_lags.values.all? { |topic_data| topic_data.values.all?(&:empty?) }
 
         read_lags_with_offsets({ previous_name => topics })
           .fetch(previous_name)
@@ -311,9 +322,35 @@ module Karafka
 
         seek_consumer_group(new_name, remap)
 
-        return unless delete_previous
+        true
+      end
+
+      # Takes consumer group and its topics and migrates all the offsets to a new named group
+      #
+      # @param previous_name [String] old consumer group name
+      # @param new_name [String] new consumer group name
+      # @param topics [Array<String>] topics for which we want to migrate offsets during rename
+      # @param delete_previous [Boolean] should we delete previous consumer group after rename.
+      #   Defaults to true.
+      # @return [Boolean] true if rename (and optionally removal) was ok or false if there was
+      #   nothing really to rename
+      #
+      # @note This method should **not** be executed on a running consumer group as it creates a
+      #   "fake" consumer and uses it to move offsets.
+      #
+      # @note After migration unless `delete_previous` is set to `false`, old group will be
+      #   removed.
+      #
+      # @note If new consumer group exists, old offsets will be added to it.
+      def rename_consumer_group(previous_name, new_name, topics, delete_previous: true)
+        copy_result = copy_consumer_group(previous_name, new_name, topics)
+
+        return false unless copy_result
+        return copy_result unless delete_previous
 
         delete_consumer_group(previous_name)
+
+        true
       end
 
       # Removes given consumer group (if exists)
@@ -325,7 +362,7 @@ module Karafka
       def delete_consumer_group(consumer_group_id)
         with_admin do |admin|
           handler = admin.delete_group(consumer_group_id)
-          handler.wait(max_wait_timeout: app_config.admin.max_wait_time)
+          handler.wait(max_wait_timeout: max_wait_time)
         end
       end
 
@@ -509,7 +546,11 @@ module Karafka
       def with_admin
         bind_id = SecureRandom.uuid
 
-        admin = config(:producer, {}).admin(native_kafka_auto_start: false)
+        admin = config(:producer, {}).admin(
+          native_kafka_auto_start: false,
+          native_kafka_poll_timeout_ms: poll_timeout
+        )
+
         bind_oauth(bind_id, admin)
 
         admin.start
@@ -572,7 +613,7 @@ module Karafka
       rescue Rdkafka::AbstractHandle::WaitTimeoutError, Errors::ResultNotVisibleError
         return if breaker.call
 
-        retry if attempt <= app_config.admin.max_attempts
+        retry if attempt <= max_attempts
 
         raise
       end
@@ -581,11 +622,10 @@ module Karafka
       # @param settings [Hash] extra settings for config (if needed)
       # @return [::Rdkafka::Config] rdkafka config
       def config(type, settings)
-        app_config
-          .kafka
+        app_kafka
           .then(&:dup)
-          .merge(app_config.admin.kafka)
-          .tap { |config| config[:'group.id'] = app_config.admin.group_id }
+          .merge(admin_kafka)
+          .tap { |config| config[:'group.id'] = group_id }
           # We merge after setting the group id so it can be altered if needed
           # In general in admin we only should alter it when we need to impersonate a given
           # consumer group or do something similar
@@ -619,11 +659,6 @@ module Karafka
           offset
         end
       end
-
-      # @return [Karafka::Core::Configurable::Node] root node config
-      def app_config
-        ::Karafka::App.config
-      end
     end
   end
 end

data/lib/karafka/base_consumer.rb

@@ -21,7 +21,7 @@ module Karafka
 
     # @return [String] id of the current consumer
     attr_reader :id
-    # @return [Karafka::Routing::Topic] topic to which a given consumer is subscribed
+    # @return [Karafka::Messages::Messages] current messages batch
     attr_accessor :messages
     # @return [Karafka::Connection::Client] kafka connection client
     attr_accessor :client
@@ -304,7 +304,12 @@ module Karafka
 
       offset = nil if offset == :consecutive
 
-      client.pause(topic.name, partition, offset)
+      client.pause(
+        topic.name,
+        partition,
+        offset,
+        coordinator.pause_tracker.current_timeout
+      )
 
       # Indicate, that user took a manual action of pausing
       coordinator.manual_pause if manual_pause
@@ -333,17 +338,21 @@ module Karafka
 
     # Seeks in the context of current topic and partition
     #
-    # @param offset [Integer, Time] offset where we want to seek or time of the offset where we
-    #   want to seek.
+    # @param offset [Integer, Time, Symbol, String] one of:
+    #   - offset where we want to seek
+    #   - time of the offset where we want to seek
+    #   - :earliest (or as a string) to move to earliest message
+    #   - :latest (or as a string) to move to latest (high-watermark)
+    #
     # @param manual_seek [Boolean] Flag to differentiate between user seek and system/strategy
     #   based seek. User seek operations should take precedence over system actions, hence we need
     #   to know who invoked it.
-    # @param reset_offset [Boolean] should we reset offset when seeking backwards. It is false by
-    #   default to prevent marking in the offset that was earlier than the highest marked offset
-    #   for given consumer group. It can be set to true if we want to reprocess data once again and
+    # @param reset_offset [Boolean] should we reset offset when seeking backwards. It is false
+    #   it prevents marking in the offset that was earlier than the highest marked offset
+    #   for given consumer group. It is set to true by default to reprocess data once again and
     #   want to make sure that the marking starts from where we moved to.
     # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
-    def seek(offset, manual_seek = true, reset_offset: false)
+    def seek(offset, manual_seek = true, reset_offset: true)
       coordinator.manual_seek if manual_seek
       self.seek_offset = nil if reset_offset
 

data/lib/karafka/cli/base.rb

@@ -112,7 +112,7 @@ module Karafka
                 *[names, option[2], option[1]].flatten
               ) { |value| options[option[0]] = value }
             end
-          end.parse(ARGV)
+          end.parse(ARGV.dup)
 
           options
         end
@@ -130,8 +130,14 @@ module Karafka
         #   given Cli command
         # @example for Karafka::Cli::Install
         #   name #=> 'install'
+        # @example for Karafka::Cli::TestMe
+        #   name => 'test_me'
         def name
-          to_s.split('::').last.downcase
+          to_s
+            .split('::')
+            .last
+            .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+            .downcase
         end
 
         # @return [Array<String>] names and aliases for command matching

data/lib/karafka/cli/topics/align.rb

@@ -30,10 +30,13 @@ module Karafka
             return false
           end
 
-          names = resources_to_migrate.map(&:name).join(', ')
-          puts "Updating configuration of the following topics: #{names}"
-          Karafka::Admin::Configs.alter(resources_to_migrate)
-          puts "#{green('Updated')} all requested topics configuration."
+          resources_to_migrate.each do |resource|
+            supervised("Updating topic: #{resource.name} configuration") do
+              Karafka::Admin::Configs.alter(resource)
+            end
+
+            puts "#{green('Updated')} topic #{resource.name} configuration."
+          end
 
           true
         end

data/lib/karafka/cli/topics/base.rb

@@ -12,6 +12,23 @@ module Karafka
 
         private
 
+        # Used to run Karafka Admin commands that talk with Kafka and that can fail due to broker
+        # errors and other issues. We catch errors and provide nicer printed output prior to
+        # re-raising the mapped error for proper exit code status handling
+        #
+        # @param operation_message [String] message that we use to print that it is going to run
+        #   and if case if failed with a failure indication.
+        def supervised(operation_message)
+          puts "#{operation_message}..."
+
+          yield
+        rescue Rdkafka::RdkafkaError => e
+          puts "#{operation_message} #{red('failed')}:"
+          puts e
+
+          raise Errors::CommandValidationError, cause: e
+        end
+
         # @return [Array<Karafka::Routing::Topic>] all available topics that can be managed
         # @note If topic is defined in multiple consumer groups, first config will be used. This
         #   means, that this CLI will not work for simultaneous management of multiple clusters

data/lib/karafka/cli/topics/create.rb

@@ -15,13 +15,15 @@ module Karafka
             if existing_topics_names.include?(name)
               puts "#{yellow('Skipping')} because topic #{name} already exists."
             else
-              puts "Creating topic #{name}..."
-              Admin.create_topic(
-                name,
-                topic.declaratives.partitions,
-                topic.declaratives.replication_factor,
-                topic.declaratives.details
-              )
+              supervised("Creating topic #{name}") do
+                Admin.create_topic(
+                  name,
+                  topic.declaratives.partitions,
+                  topic.declaratives.replication_factor,
+                  topic.declaratives.details
+                )
+              end
+
               puts "#{green('Created')} topic #{name}."
               any_created = true
             end

data/lib/karafka/cli/topics/delete.rb

@@ -13,8 +13,10 @@ module Karafka
             name = topic.name
 
             if existing_topics_names.include?(name)
-              puts "Deleting topic #{name}..."
-              Admin.delete_topic(name)
+              supervised("Deleting topic #{name}") do
+                Admin.delete_topic(name)
+              end
+
               puts "#{green('Deleted')} topic #{name}."
               any_deleted = true
             else

data/lib/karafka/cli/topics/help.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Karafka
+  class Cli
+    class Topics < Cli::Base
+      # Declarative topics CLI sub-help
+      class Help < Base
+        # Displays help information for all available topics management commands
+        def call
+          puts <<~HELP
+            Karafka topics commands:
+              align        # Aligns configuration of all declarative topics based on definitions
+              create       # Creates topics with appropriate settings
+              delete       # Deletes all topics defined in the routes
+              help         # Describes available topics management commands
+              migrate      # Creates missing topics, repartitions existing and aligns configuration
+              plan         # Plans migration process and prints changes to be applied
+              repartition  # Adds additional partitions to topics with fewer partitions than expected
+              reset        # Deletes and re-creates all topics
+
+            Options:
+              --detailed-exitcode  # Provides detailed exit codes (0=no changes, 1=error, 2=changes applied)
+
+            Examples:
+              karafka topics create
+              karafka topics plan --detailed-exitcode
+              karafka topics migrate
+              karafka topics align
+
+            Note: All admin operations run on the default cluster only.
+          HELP
+
+          # We return false to indicate with exit code 0 that no changes were applied
+          false
+        end
+      end
+    end
+  end
+end

data/lib/karafka/cli/topics/repartition.rb

@@ -21,8 +21,10 @@ module Karafka
             existing_count = existing_partitions.fetch(name, false)
 
             if existing_count && existing_count < desired_count
-              puts "Increasing number of partitions to #{desired_count} on topic #{name}..."
-              Admin.create_partitions(name, desired_count)
+              supervised("Increasing number of partitions to #{desired_count} on topic #{name}") do
+                Admin.create_partitions(name, desired_count)
+              end
+
               change = desired_count - existing_count
               puts "#{green('Created')} #{change} additional partitions on topic #{name}."
               any_repartitioned = true

data/lib/karafka/cli/topics.rb

@@ -27,10 +27,13 @@ module Karafka
       # crashes
       CHANGES_EXIT_CODE = 2
 
-      private_constant :NO_CHANGES_EXIT_CODE, :CHANGES_EXIT_CODE
+      # Used when there was an error during execution.
+      ERROR_EXIT_CODE = 1
+
+      private_constant :NO_CHANGES_EXIT_CODE, :CHANGES_EXIT_CODE, :ERROR_EXIT_CODE
 
       # @param action [String] action we want to take
-      def call(action = 'missing')
+      def call(action = 'help')
         detailed_exit_code = options.fetch(:detailed_exitcode, false)
 
         command = case action
@@ -48,8 +51,10 @@ module Karafka
                     Topics::Align
                   when 'plan'
                     Topics::Plan
+                  when 'help'
+                    Topics::Help
                   else
-                    raise ::ArgumentError, "Invalid topics action: #{action}"
+                    raise Errors::UnrecognizedCommandError, "Unrecognized topics action: #{action}"
                   end
 
         changes = command.new.call
@@ -57,6 +62,8 @@ module Karafka
         return unless detailed_exit_code
 
         changes ? exit(CHANGES_EXIT_CODE) : exit(NO_CHANGES_EXIT_CODE)
+      rescue Errors::CommandValidationError
+        exit(ERROR_EXIT_CODE)
       end
     end
   end

data/lib/karafka/cli.rb

@@ -21,6 +21,8 @@ module Karafka
           args = action ? [action] : []
 
           command.new.call(*args)
+        elsif command_name.nil?
+          Help.new.call
         else
           raise(
             Karafka::Errors::UnrecognizedCommandError,

data/lib/karafka/connection/client.rb

@@ -9,6 +9,11 @@ module Karafka
     # closed consumer instance as it causes Ruby VM process to crash.
     class Client
       include ::Karafka::Core::Helpers::Time
+      include Helpers::ConfigImporter.new(
+        logger: %i[logger],
+        tick_interval: %i[internal tick_interval],
+        shutdown_timeout: %i[shutdown_timeout]
+      )
 
       attr_reader :rebalance_manager
 
@@ -65,9 +70,8 @@ module Karafka
         @closed = false
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
-        @tick_interval = ::Karafka::App.config.internal.tick_interval
         @rebalance_manager = RebalanceManager.new(@subscription_group.id, @buffer)
-        @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(@subscription_group)
+        @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(@subscription_group, id)
 
         @interval_runner = Helpers::IntervalRunner.new do
           events_poll
@@ -221,10 +225,14 @@ module Karafka
       # @param offset [Integer, nil] offset of the message on which we want to pause (this message
       #   will be reprocessed after getting back to processing) or nil if we want to pause and
       #   resume from the consecutive offset (+1 from the last message passed to us by librdkafka)
+      # @param timeout [Integer] number of ms timeout of pause. It is used only for
+      #   instrumentation and not in the pause itself as pausing on this level is infinite always.
       # @note This will pause indefinitely and requires manual `#resume`
       # @note When `#internal_seek` is not involved (when offset is `nil`) we will not purge the
       #   librdkafka buffers and continue from the last cursor offset
-      def pause(topic, partition, offset = nil)
+      # @note We accept the timeout value on this layer to have a cohesive pause/resume
+      #   instrumentation, where all the details are available. It is especially needed, when
+      def pause(topic, partition, offset = nil, timeout = 0)
         @mutex.synchronize do
           # Do not pause if the client got closed, would not change anything
           return if @closed
@@ -243,7 +251,8 @@ module Karafka
             subscription_group: @subscription_group,
             topic: topic,
             partition: partition,
-            offset: offset
+            offset: offset,
+            timeout: timeout
           )
 
           @paused_tpls[topic][partition] = tpl
@@ -313,7 +322,7 @@ module Karafka
           @unsubscribing = true
 
           # Give 50% of time for the final close before we reach the forceful
-          max_wait = ::Karafka::App.config.shutdown_timeout * COOP_UNSUBSCRIBE_FACTOR
+          max_wait = shutdown_timeout * COOP_UNSUBSCRIBE_FACTOR
           used = 0
           stopped_at = monotonic_now
 
@@ -472,15 +481,27 @@ module Karafka
           message.offset = detected_partition&.offset || raise(Errors::InvalidTimeBasedOffsetError)
         end
 
+        # Those two are librdkafka hardcoded values
+        message.offset = -1 if message.offset.to_s == 'latest'
+        message.offset = -2 if message.offset.to_s == 'earliest'
+
         # Never seek if we would get the same location as we would get without seeking
         # This prevents us from the expensive buffer purges that can lead to increased network
         # traffic and can cost a lot of money
         #
         # This code adds around 0.01 ms per seek but saves from many user unexpected behaviours in
         # seeking and pausing
-        return if message.offset == topic_partition_position(message.topic, message.partition)
+        position = topic_partition_position(message.topic, message.partition)
+
+        # Always seek if current position cannot be fetched or is negative. Offset seek can also
+        # be negative (-1 or -2) and we should not compare it with the position because they are
+        # special (earliest or latest)
+        return kafka.seek(message) if position.negative?
+        # If offset is the same as the next position, we don't have to seek to get there, hence
+        # only in such case we can do nothing.
+        return kafka.seek(message) if message.offset != position
 
-        kafka.seek(message)
+        nil
       end
 
       # Commits the stored offsets in a sync way and closes the consumer.
@@ -573,7 +594,7 @@ module Karafka
         # We should not run a single poll longer than the tick frequency. Otherwise during a single
         # `#batch_poll` we would not be able to run `#events_poll` often enough effectively
         # blocking events from being handled.
-        poll_tick = timeout > @tick_interval ? @tick_interval : timeout
+        poll_tick = timeout > tick_interval ? tick_interval : timeout
 
         result = kafka.poll(poll_tick)
 
@@ -655,7 +676,7 @@ module Karafka
       # Builds a new rdkafka consumer instance based on the subscription group configuration
       # @return [Rdkafka::Consumer]
       def build_consumer
-        ::Rdkafka::Config.logger = ::Karafka::App.config.logger
+        ::Rdkafka::Config.logger = logger
 
         # We need to refresh the setup of this subscription group in case we started running in a
         # swarm. The initial configuration for validation comes from the parent node, but it needs