karafka 2.3.4 → 2.4.0.beta1

data/lib/karafka/admin/configs.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Admin
+    # Namespace for admin operations related to configuration management
+    #
+    # At the moment Karafka supports configuration management for brokers and topics
+    #
+    # You can describe configuration as well as alter it.
+    #
+    # Altering is done in the incremental way.
+    module Configs
+      class << self
+        # Fetches given resources configurations from Kafka
+        #
+        # @param resources [Resource, Array<Resource>] single resource we want to describe or
+        #   list of resources we are interested in. It is useful to provide multiple resources
+        #   when you need data from multiple topics, etc. Karafka will make one query for all the
+        #   data instead of doing one per topic.
+        #
+        # @return [Array<Resource>] array with resources containing their configuration details
+        #
+        # @note Even if you request one resource, result will always be an array with resources
+        #
+        # @example Describe topic named "example" and print its config
+        #   resource = Karafka::Admin::Configs::Resource.new(type: :topic, name: 'example')
+        #   results = Karafka::Admin::Configs.describe(resource)
+        #   results.first.configs.each do |config|
+        #     puts "#{config.name} - #{config.value}"
+        #   end
+        def describe(*resources)
+          operate_on_resources(
+            :describe_configs,
+            resources
+          )
+        end
+
+        # Alters given resources based on the alteration operations accumulated in the provided
+        # resources
+        #
+        # @param resources [Resource, Array<Resource>] single resource we want to alter or
+        #   list of resources.
+        #
+        # @note This operation is not transactional and can work only partially if some config
+        #   options are not valid. Always make sure, your alterations are correct.
+        #
+        # @note We call it `#alter` despite using the Kafka incremental alter API because the
+        #   regular alter is deprecated.
+        #
+        # @example Alter the `delete.retention.ms` and set it to 8640001
+        #   resource = Karafka::Admin::Configs::Resource.new(type: :topic, name: 'example')
+        #   resource.set('delete.retention.ms', '8640001')
+        #   Karafka::Admin::Configs.alter(resource)
+        def alter(*resources)
+          operate_on_resources(
+            :incremental_alter_configs,
+            resources
+          )
+        end
+
+        private
+
+        # @param action [Symbol] runs given action via Rdkafka Admin
+        # @param resources [Array<Resource>] resources on which we want to operate
+        def operate_on_resources(action, resources)
+          resources = Array(resources).flatten
+
+          result = with_admin_wait do |admin|
+            admin.public_send(
+              action,
+              resources.map(&:to_native_hash)
+            )
+          end
+
+          result.resources.map do |rd_kafka_resource|
+            # Create back a resource
+            resource = Resource.new(
+              name: rd_kafka_resource.name,
+              type: rd_kafka_resource.type
+            )
+
+            rd_kafka_resource.configs.each do |rd_kafka_config|
+              resource.configs << Config.from_rd_kafka(rd_kafka_config)
+            end
+
+            resource.configs.sort_by!(&:name)
+            resource.configs.freeze
+
+            resource
+          end
+        end
+
+        # Yields admin instance, allows to run Acl operations and awaits on the final result
+        # Makes sure that admin is closed afterwards.
+        def with_admin_wait
+          Admin.with_admin do |admin|
+            yield(admin).wait(max_wait_timeout: Karafka::App.config.admin.max_wait_time)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/admin.rb

@@ -132,7 +132,7 @@ module Karafka
 
           with_re_wait(
             -> { handler.wait(max_wait_timeout: app_config.admin.max_wait_time) },
-            -> { topic(name).fetch(:partition_count) >= partitions }
+            -> { topic_info(name).fetch(:partition_count) >= partitions }
           )
         end
       end
@@ -168,7 +168,7 @@ module Karafka
           if partitions_with_offsets.is_a?(Hash)
             tpl_base[topic] = partitions_with_offsets
           else
-            topic(topic)[:partition_count].times do |partition|
+            topic_info(topic)[:partition_count].times do |partition|
               tpl_base[topic][partition] = partitions_with_offsets
             end
           end
@@ -183,76 +183,52 @@ module Karafka
         tpl_base.each do |topic, partitions_with_offsets|
           partitions_with_offsets.each do |partition, offset|
             target = offset.is_a?(Time) ? time_tpl : tpl
-            # We reverse and uniq to make sure that potentially duplicated references are removed
-            # in such a way that the newest stays
-            target.to_h[topic] ||= []
-            target.to_h[topic] << Rdkafka::Consumer::Partition.new(partition, offset)
-            target.to_h[topic].reverse!
-            target.to_h[topic].uniq!(&:partition)
-            target.to_h[topic].reverse!
+            target.add_topic_and_partitions_with_offsets(topic, [[partition, offset]])
           end
         end
 
-        # We set this that way so we can impersonate this consumer group and seek where we want
-        mapped_consumer_group_id = app_config.consumer_mapper.call(consumer_group_id)
-        settings = { 'group.id': mapped_consumer_group_id }
-
-        # This error can occur when we query a broker that is not a coordinator because something
-        # was changing in the cluster. We should be able to safely restart our seeking request
-        # when this happens without any issues
-        #
-        # We wrap the consumer creation, so we retry with a new consumer instance
-        with_rdkafka_retry(codes: %i[not_coordinator]) do
-          with_consumer(settings) do |consumer|
-            # If we have any time based stuff to resolve, we need to do it prior to commits
-            unless time_tpl.empty?
-              real_offsets = consumer.offsets_for_times(time_tpl)
-
-              real_offsets.to_h.each do |name, results|
-                results.each do |result|
-                  raise(Errors::InvalidTimeBasedOffsetError) unless result
-
-                  partition = result.partition
-
-                  # Negative offset means we're beyond last message and we need to query for the
-                  # high watermark offset to get the most recent offset and move there
-                  if result.offset.negative?
-                    _, offset = consumer.query_watermark_offsets(name, result.partition)
-                  else
-                    # If we get an offset, it means there existed a message close to this time
-                    # location
-                    offset = result.offset
-                  end
-
-                  # Since now we have proper offsets, we can add this to the final tpl for commit
-                  tpl.to_h[name] ||= []
-                  tpl.to_h[name] << Rdkafka::Consumer::Partition.new(partition, offset)
-                  tpl.to_h[name].reverse!
-                  tpl.to_h[name].uniq!(&:partition)
-                  tpl.to_h[name].reverse!
+        settings = { 'group.id': consumer_group_id }
+
+        with_consumer(settings) do |consumer|
+          # If we have any time based stuff to resolve, we need to do it prior to commits
+          unless time_tpl.empty?
+            real_offsets = consumer.offsets_for_times(time_tpl)
+
+            real_offsets.to_h.each do |name, results|
+              results.each do |result|
+                raise(Errors::InvalidTimeBasedOffsetError) unless result
+
+                partition = result.partition
+
+                # Negative offset means we're beyond last message and we need to query for the
+                # high watermark offset to get the most recent offset and move there
+                if result.offset.negative?
+                  _, offset = consumer.query_watermark_offsets(name, result.partition)
+                else
+                  # If we get an offset, it means there existed a message close to this time
+                  # location
+                  offset = result.offset
                 end
+
+                # Since now we have proper offsets, we can add this to the final tpl for commit
+                tpl.add_topic_and_partitions_with_offsets(name, [[partition, offset]])
               end
             end
-
-            consumer.commit(tpl, false)
           end
+
+          consumer.commit_offsets(tpl, async: false)
         end
       end
 
       # Removes given consumer group (if exists)
       #
-      # @param consumer_group_id [String] consumer group name without the mapper name (if any used)
-      #
-      # @note Please note, Karafka will apply the consumer group mapper on the provided consumer
-      #   group.
+      # @param consumer_group_id [String] consumer group name
       #
       # @note This method should not be used on a running consumer group as it will not yield any
       #   results.
       def delete_consumer_group(consumer_group_id)
-        mapped_consumer_group_id = app_config.consumer_mapper.call(consumer_group_id)
-
         with_admin do |admin|
-          handler = admin.delete_group(mapped_consumer_group_id)
+          handler = admin.delete_group(consumer_group_id)
           handler.wait(max_wait_timeout: app_config.admin.max_wait_time)
         end
       end
@@ -264,13 +240,118 @@ module Karafka
       # @return [Array<Integer, Integer>] low watermark offset and high watermark offset
       def read_watermark_offsets(name, partition)
         with_consumer do |consumer|
-          # For newly created topics or in cases where we're trying to get them but there is no
-          # leader, this can fail. It happens more often for new topics under KRaft, however we
-          # still want to make sure things operate as expected even then
-          with_rdkafka_retry(codes: %i[not_leader_for_partition]) do
-            consumer.query_watermark_offsets(name, partition)
+          consumer.query_watermark_offsets(name, partition)
+        end
+      end
+
+      # Reads lags and offsets for given topics in the context of consumer groups defined in the
+      #   routing
+      # @param consumer_groups_with_topics [Hash<String, Array<String>>] hash with consumer groups
+      #   names with array of topics to query per consumer group inside
+      # @param active_topics_only [Boolean] if set to false, when we use routing topics, will
+      #   select also topics that are marked as inactive in routing
+      # @return [Hash<String, Hash<Integer, <Hash<Integer>>>>] hash where the top level keys are
+      #   the consumer groups and values are hashes with topics and inside partitions with lags
+      #   and offsets
+      #
+      # @note For topics that do not exist, topic details will be set to an empty hash
+      #
+      # @note For topics that exist but were never consumed by a given CG we set `-1` as lag and
+      #   the offset on each of the partitions that were not consumed.
+      #
+      # @note This lag reporting is for committed lags and is "Kafka-centric", meaning that this
+      #   represents lags from Kafka perspective and not the consumer. They may differ.
+      def read_lags_with_offsets(consumer_groups_with_topics = {}, active_topics_only: true)
+        # We first fetch all the topics with partitions count that exist in the cluster so we
+        # do not query for topics that do not exist and so we can get partitions count for all
+        # the topics we may need. The non-existent and not consumed will be filled at the end
+        existing_topics = cluster_info.topics.map do |topic|
+          [topic[:topic_name], topic[:partition_count]]
+        end.to_h.freeze
+
+        # If no expected CGs, we use all from routing that have active topics
+        if consumer_groups_with_topics.empty?
+          consumer_groups_with_topics = Karafka::App.routes.map do |cg|
+            cg_topics = cg.topics.select do |cg_topic|
+              active_topics_only ? cg_topic.active? : true
+            end
+
+            [cg.id, cg_topics.map(&:name)]
+          end.to_h
+        end
+
+        # We make a copy because we will remove once with non-existing topics
+        # We keep original requested consumer groups with topics for later backfilling
+        cgs_with_topics = consumer_groups_with_topics.dup
+        cgs_with_topics.transform_values!(&:dup)
+
+        # We can query only topics that do exist, this is why we are cleaning those that do not
+        # exist
+        cgs_with_topics.each_value do |requested_topics|
+          requested_topics.delete_if { |topic| !existing_topics.include?(topic) }
+        end
+
+        groups_lags = Hash.new { |h, k| h[k] = {} }
+        groups_offs = Hash.new { |h, k| h[k] = {} }
+
+        cgs_with_topics.each do |cg, topics|
+          # Do not add to tpl topics that do not exist
+          next if topics.empty?
+
+          tpl = Rdkafka::Consumer::TopicPartitionList.new
+
+          with_consumer('group.id': cg) do |consumer|
+            topics.each { |topic| tpl.add_topic(topic, existing_topics[topic]) }
+
+            commit_offsets = consumer.committed(tpl)
+
+            commit_offsets.to_h.each do |topic, partitions|
+              groups_offs[cg][topic] = {}
+
+              partitions.each do |partition|
+                # -1 when no offset is stored
+                groups_offs[cg][topic][partition.partition] = partition.offset || -1
+              end
+            end
+
+            consumer.lag(commit_offsets).each do |topic, partitions_lags|
+              groups_lags[cg][topic] = partitions_lags
+            end
+          end
+        end
+
+        consumer_groups_with_topics.each do |cg, topics|
+          groups_lags[cg]
+
+          topics.each do |topic|
+            groups_lags[cg][topic] ||= {}
+
+            next unless existing_topics.key?(topic)
+
+            # We backfill because there is a case where our consumer group would consume for
+            # example only one partition out of 20, rest needs to get -1
+            existing_topics[topic].times do |partition_id|
+              groups_lags[cg][topic][partition_id] ||= -1
+            end
+          end
+        end
+
+        merged = Hash.new { |h, k| h[k] = {} }
+
+        groups_lags.each do |cg, topics|
+          topics.each do |topic, partitions|
+            merged[cg][topic] = {}
+
+            partitions.each do |partition, lag|
+              merged[cg][topic][partition] = {
+                offset: groups_offs.fetch(cg).fetch(topic).fetch(partition),
+                lag: lag
+              }
+            end
           end
         end
+
+        merged
       end
 
       # @return [Rdkafka::Metadata] cluster metadata info
@@ -278,6 +359,24 @@ module Karafka
         with_admin(&:metadata)
       end
 
+      # Returns basic topic metadata
+      #
+      # @param topic_name [String] name of the topic we're interested in
+      # @return [Hash] topic metadata info hash
+      # @raise [Rdkafka::RdkafkaError] `unknown_topic_or_part` if requested topic is not found
+      #
+      # @note This query is much more efficient than doing a full `#cluster_info` + topic lookup
+      #   because it does not have to query for all the topics data but just the topic we're
+      #   interested in
+      def topic_info(topic_name)
+        with_admin do |admin|
+          admin
+            .metadata(topic_name)
+            .topics
+            .find { |topic| topic[:topic_name] == topic_name }
+        end
+      end
+
       # Creates consumer instance and yields it. After usage it closes the consumer instance
       # This API can be used in other pieces of code and allows for low-level consumer usage
       #
@@ -286,7 +385,12 @@ module Karafka
       # @note We always ship and yield a proxied consumer because admin API performance is not
       #   that relevant. That is, there are no high frequency calls that would have to be delegated
       def with_consumer(settings = {})
-        consumer = config(:consumer, settings).consumer
+        bind_id = SecureRandom.uuid
+
+        consumer = config(:consumer, settings).consumer(native_kafka_auto_start: false)
+        bind_oauth(bind_id, consumer)
+
+        consumer.start
         proxy = ::Karafka::Connection::Proxy.new(consumer)
         yield(proxy)
       ensure
@@ -301,30 +405,56 @@ module Karafka
         end
 
         consumer&.close
+
+        unbind_oauth(bind_id)
       end
 
       # Creates admin instance and yields it. After usage it closes the admin instance
       def with_admin
-        admin = config(:producer, {}).admin
-        yield(admin)
+        bind_id = SecureRandom.uuid
+
+        admin = config(:producer, {}).admin(native_kafka_auto_start: false)
+        bind_oauth(bind_id, admin)
+
+        admin.start
+        proxy = ::Karafka::Connection::Proxy.new(admin)
+        yield(proxy)
       ensure
         admin&.close
+
+        unbind_oauth(bind_id)
       end
 
       private
 
+      # Adds a new callback for given rdkafka instance for oauth token refresh (if needed)
+      #
+      # @param id [String, Symbol] unique (for the lifetime of instance) id that we use for
+      #   callback referencing
+      # @param instance [Rdkafka::Consumer, Rdkafka::Admin] rdkafka instance to be used to set
+      #   appropriate oauth token when needed
+      def bind_oauth(id, instance)
+        ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.add(
+          id,
+          Instrumentation::Callbacks::OauthbearerTokenRefresh.new(
+            instance
+          )
+        )
+      end
+
+      # Removes the callback from no longer used instance
+      #
+      # @param id [String, Symbol] unique (for the lifetime of instance) id that we use for
+      #   callback referencing
+      def unbind_oauth(id)
+        ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.delete(id)
+      end
+
       # @return [Array<String>] topics names
       def topics_names
         cluster_info.topics.map { |topic| topic.fetch(:topic_name) }
       end
 
-      # Finds details about given topic
-      # @param name [String] topic name
-      # @return [Hash] topic details
-      def topic(name)
-        cluster_info.topics.find { |topic| topic[:topic_name] == name }
-      end
-
       # There are some cases where rdkafka admin operations finish successfully but without the
       # callback being triggered to materialize the post-promise object. Until this is fixed we
       # can figure out, that operation we wanted to do finished successfully by checking that the
@@ -351,44 +481,15 @@ module Karafka
         raise
       end
 
-      # Handles retries for rdkafka related errors that we specify in `:codes`.
-      #
-      # Some operations temporarily fail, especially for cases where we changed something fast
-      # like topic creation or repartitioning. In cases like this it is ok to retry operations that
-      # do not change the state as it will usually recover.
-      #
-      # @param codes [Array<Symbol>] librdkafka error codes on which we want to retry
-      # @param max_attempts [Integer] number of attempts (including initial) after which we should
-      #   give up
-      #
-      # @note This code implements a simple backoff that increases with each attempt.
-      def with_rdkafka_retry(codes:, max_attempts: 5)
-        attempt ||= 0
-        attempt += 1
-
-        yield
-      rescue Rdkafka::RdkafkaError => e
-        raise unless codes.include?(e.code)
-        raise if attempt >= max_attempts
-
-        sleep(max_attempts)
-
-        retry
-      end
-
       # @param type [Symbol] type of config we want
       # @param settings [Hash] extra settings for config (if needed)
       # @return [::Rdkafka::Config] rdkafka config
       def config(type, settings)
-        mapped_admin_group_id = app_config.consumer_mapper.call(
-          app_config.admin.group_id
-        )
-
         app_config
           .kafka
           .then(&:dup)
           .merge(app_config.admin.kafka)
-          .tap { |config| config[:'group.id'] = mapped_admin_group_id }
+          .tap { |config| config[:'group.id'] = app_config.admin.group_id }
           # We merge after setting the group id so it can be altered if needed
           # In general in admin we only should alter it when we need to impersonate a given
           # consumer group or do something similar

data/lib/karafka/base_consumer.rb

@@ -217,7 +217,7 @@ module Karafka
         subscription_group: topic.subscription_group,
         offset: offset,
         timeout: coordinator.pause_tracker.current_timeout,
-        attempt: coordinator.pause_tracker.attempt
+        attempt: attempt
       )
     end
 
@@ -297,7 +297,7 @@ module Karafka
         partition: partition,
         offset: coordinator.seek_offset,
         timeout: coordinator.pause_tracker.current_timeout,
-        attempt: coordinator.pause_tracker.attempt
+        attempt: attempt
       )
     end
   end

data/lib/karafka/cli/info.rb

@@ -5,6 +5,12 @@ module Karafka
   class Cli
     # Info Karafka Cli action
     class Info < Base
+      include Helpers::ConfigImporter.new(
+        concurrency: %i[concurrency],
+        license: %i[license],
+        client_id: %i[client_id]
+      )
+
       desc 'Prints configuration details and other options of your application'
 
       # Nice karafka banner
@@ -29,8 +35,6 @@ module Karafka
 
       # @return [Array<String>] core framework related info
       def core_info
-        config = Karafka::App.config
-
         postfix = Karafka.pro? ? ' + Pro' : ''
 
         [
@@ -39,8 +43,8 @@ module Karafka
           "Rdkafka version: #{::Rdkafka::VERSION}",
           "Consumer groups count: #{Karafka::App.consumer_groups.size}",
           "Subscription groups count: #{Karafka::App.subscription_groups.values.flatten.size}",
-          "Workers count: #{Karafka::App.config.concurrency}",
-          "Application client id: #{config.client_id}",
+          "Workers count: #{concurrency}",
+          "Application client id: #{client_id}",
           "Boot file: #{Karafka.boot_file}",
           "Environment: #{Karafka.env}"
         ]
@@ -48,12 +52,10 @@ module Karafka
 
       # @return [Array<String>] license related info
       def license_info
-        config = Karafka::App.config
-
         if Karafka.pro?
           [
             'License: Commercial',
-            "License entity: #{config.license.entity}"
+            "License entity: #{license.entity}"
           ]
         else
           [

data/lib/karafka/cli/server.rb

@@ -5,6 +5,10 @@ module Karafka
   class Cli
     # Server Karafka Cli action
     class Server < Base
+      include Helpers::ConfigImporter.new(
+        activity_manager: %i[internal routing activity_manager]
+      )
+
       # Types of things we can include / exclude from the routing via the CLI options
       SUPPORTED_TYPES = ::Karafka::Routing::ActivityManager::SUPPORTED_TYPES
 
@@ -90,23 +94,19 @@ module Karafka
 
       # Registers things we want to include (if defined)
       def register_inclusions
-        activities = ::Karafka::App.config.internal.routing.activity_manager
-
         SUPPORTED_TYPES.each do |type|
           names = options[type] || []
 
-          names.each { |name| activities.include(type, name) }
+          names.each { |name| activity_manager.include(type, name) }
         end
       end
 
       # Registers things we want to exclude (if defined)
       def register_exclusions
-        activities = ::Karafka::App.config.internal.routing.activity_manager
-
-        activities.class::SUPPORTED_TYPES.each do |type|
+        activity_manager.class::SUPPORTED_TYPES.each do |type|
           names = options[:"exclude_#{type}"] || []
 
-          names.each { |name| activities.exclude(type, name) }
+          names.each { |name| activity_manager.exclude(type, name) }
         end
       end
     end