karafka 2.3.4 → 2.4.0.beta1

data/lib/karafka/connection/client.rb

@@ -30,8 +30,11 @@ module Karafka
       #
       # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
       #   with all the configuration details needed for us to create a client
+      # @param batch_poll_breaker [Proc] proc that when evaluated to false will cause the batch
+      #   poll loop to finish early. This improves the shutdown and dynamic multiplication as it
+      #   allows us to early break on long polls.
       # @return [Karafka::Connection::Client]
-      def initialize(subscription_group)
+      def initialize(subscription_group, batch_poll_breaker)
         @id = SecureRandom.hex(6)
         # Name is set when we build consumer
         @name = ''
@@ -41,7 +44,14 @@ module Karafka
         @tick_interval = ::Karafka::App.config.internal.tick_interval
         @rebalance_manager = RebalanceManager.new(@subscription_group.id)
         @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(@subscription_group)
-        @events_poller = Helpers::IntervalRunner.new { events_poll }
+
+        @interval_runner = Helpers::IntervalRunner.new do
+          events_poll
+          # events poller returns nil when not running often enough, hence we don't use the
+          # boolean to be explicit
+          batch_poll_breaker.call ? :run : :stop
+        end
+
         # There are few operations that can happen in parallel from the listener threads as well
         # as from the workers. They are not fully thread-safe because they may be composed out of
         # few calls to Kafka or out of few internal state changes. That is why we mutex them.
@@ -98,7 +108,8 @@ module Karafka
             break
           end
 
-          @events_poller.call
+          # If we were signaled from the outside to break the loop, we should
+          break if @interval_runner.call == :stop
 
           # Track time spent on all of the processing and polling
           time_poll.checkpoint
@@ -295,7 +306,7 @@ module Karafka
         ) do
           close
 
-          @events_poller.reset
+          @interval_runner.reset
           @closed = false
           @paused_tpls.clear
         end
@@ -344,7 +355,7 @@ module Karafka
       # @note It is recommended to use this only on rebalances to get positions with metadata
       #   when working with metadata as this is synchronous
       def committed(tpl = nil)
-        Proxy.new(kafka).committed(tpl)
+        @wrapped_kafka.committed(tpl)
       end
 
       private
@@ -356,13 +367,7 @@ module Karafka
       # @param metadata [String, nil] offset storage metadata or nil if none
       # @return [Boolean] true if we could store the offset (if we still own the partition)
       def internal_store_offset(message, metadata)
-        kafka.store_offset(message, metadata)
-        true
-      rescue Rdkafka::RdkafkaError => e
-        return false if e.code == :assignment_lost
-        return false if e.code == :state
-
-        raise e
+        @wrapped_kafka.store_offset(message, metadata)
       end
 
       # Non thread-safe message committing method
@@ -372,23 +377,7 @@ module Karafka
       #   even when no stored, because with sync commit, it refreshes the ownership state of the
       #   consumer in a sync way.
       def internal_commit_offsets(async: true)
-        kafka.commit(nil, async)
-
-        true
-      rescue Rdkafka::RdkafkaError => e
-        case e.code
-        when :assignment_lost
-          return false
-        when :unknown_member_id
-          return false
-        when :no_offset
-          return true
-        when :coordinator_load_in_progress
-          sleep(1)
-          retry
-        end
-
-        raise e
+        @wrapped_kafka.commit_offsets(async: async)
       end
 
       # Non-mutexed seek that should be used only internally. Outside we expose `#seek` that is
@@ -409,12 +398,10 @@ module Karafka
             message.partition => message.offset
           )
 
-          proxy = Proxy.new(kafka)
-
           # Now we can overwrite the seek message offset with our resolved offset and we can
           # then seek to the appropriate message
           # We set the timeout to 2_000 to make sure that remote clusters handle this well
-          real_offsets = proxy.offsets_for_times(tpl)
+          real_offsets = @wrapped_kafka.offsets_for_times(tpl)
           detected_partition = real_offsets.to_h.dig(message.topic, message.partition)
 
           # There always needs to be an offset. In case we seek into the future, where there
@@ -445,12 +432,16 @@ module Karafka
 
         return unless @kafka
 
+        sg_id = @subscription_group.id
+
         # Remove callbacks runners that were registered
-        ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
-        ::Karafka::Core::Instrumentation.error_callbacks.delete(@subscription_group.id)
+        ::Karafka::Core::Instrumentation.statistics_callbacks.delete(sg_id)
+        ::Karafka::Core::Instrumentation.error_callbacks.delete(sg_id)
+        ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.delete(sg_id)
 
         kafka.close
         @kafka = nil
+        @wrapped_kafka = nil
         @buffer.clear
         # @note We do not clear rebalance manager here as we may still have revocation info
         # here that we want to consider valid prior to running another reconnection
@@ -571,6 +562,12 @@ module Karafka
 
           # No sense in retrying when no topic/partition and we're no longer running
           retryable = false unless Karafka::App.running?
+        # If we detect the end of partition which can happen if `enable.partition.eof` is set to
+        # true, we can just return nil fast. This will fast yield whatever set of messages we
+        # already have instead of waiting. This can be used for better latency control when we do
+        # not expect a lof of lag and want to quickly move to processing.
+        when :partition_eof
+          return nil
         end
 
         if early_report || !retryable
@@ -614,7 +611,8 @@ module Karafka
         # new messages. This allows us to report statistics while data is still being processed
         config.consumer_poll_set = false
 
-        consumer = config.consumer
+        # Do not start native kafka so we can inject the oauth bearer callbacks if needed
+        consumer = config.consumer(native_kafka_auto_start: false)
         @name = consumer.name
 
         # Register statistics runner for this particular type of callbacks
@@ -637,9 +635,21 @@ module Karafka
           )
         )
 
+        ::Karafka::Core::Instrumentation.oauthbearer_token_refresh_callbacks.add(
+          @subscription_group.id,
+          Instrumentation::Callbacks::OauthbearerTokenRefresh.new(
+            consumer
+          )
+        )
+
         # Subscription needs to happen after we assigned the rebalance callbacks just in case of
         # a race condition
-        consumer.subscribe(*@subscription_group.subscriptions)
+        subscriptions = @subscription_group.subscriptions
+        assignments = @subscription_group.assignments(consumer)
+
+        consumer.subscribe(*subscriptions) if subscriptions
+        consumer.assign(assignments) if assignments
+
         consumer
       end
 
@@ -659,7 +669,15 @@ module Karafka
 
       # @return [Rdkafka::Consumer] librdkafka consumer instance
       def kafka
-        @kafka ||= build_consumer
+        return @kafka if @kafka
+
+        @kafka = build_consumer
+        @wrapped_kafka = Proxy.new(@kafka)
+        # We start it only after everything is configured so oauth or any other early-run client
+        # related operations can occur. Otherwise, if all kafka referencing setup would not be
+        # done, we could not intercept the invocations to kafka via client methods.
+        @kafka.start
+        @kafka
       end
     end
   end

data/lib/karafka/connection/listener.rb

@@ -36,7 +36,7 @@ module Karafka
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
         @coordinators = Processing::CoordinatorsBuffer.new(subscription_group.topics)
-        @client = Client.new(@subscription_group)
+        @client = Client.new(@subscription_group, -> { running? })
         @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
         @jobs_builder = proc_config.jobs_builder
         @partitioner = proc_config.partitioner_class.new(subscription_group)
@@ -46,11 +46,18 @@ module Karafka
         # We can do this that way because we always first schedule jobs using messages before we
         # fetch another batch.
         @messages_buffer = MessagesBuffer.new(subscription_group)
-        @usage_tracker = TimeTrackers::PartitionUsage.new
         @mutex = Mutex.new
         @status = Status.new
 
         @jobs_queue.register(@subscription_group.id)
+
+        # This makes sure that even if we tick more often than the interval time due to frequent
+        # unlocks from short-lived jobs or async queues synchronization, events handling and jobs
+        # scheduling still happens with the expected frequency
+        @interval_runner = Helpers::IntervalRunner.new do
+          @events_poller.call
+          @scheduler.on_manage
+        end
       end
 
       # Runs the main listener fetch loop.
@@ -103,7 +110,8 @@ module Karafka
         end
 
         @status.start!
-        async_call
+
+        async_call("karafka.listener##{@subscription_group.id}")
       end
 
       # Stops the jobs queue, triggers shutdown on all the executors (sync), commits offsets and
@@ -277,15 +285,14 @@ module Karafka
 
         revoked_partitions.each do |topic, partitions|
           partitions.each do |partition|
-            @usage_tracker.revoke(topic, partition)
             @coordinators.revoke(topic, partition)
 
             # There may be a case where we have lost partition of which data we have never
             # processed (if it was assigned and revoked really fast), thus we may not have it
             # here. In cases like this, we do not run a revocation job
             @executors.find_all(topic, partition).each do |executor|
-              job = @jobs_builder.revoked(executor)
-              jobs << job
+              executor.coordinator.increment(:revoked)
+              jobs << @jobs_builder.revoked(executor)
             end
 
             # We need to remove all the executors of a given topic partition that we have lost, so
@@ -308,6 +315,7 @@ module Karafka
         jobs = []
 
         @executors.each do |executor|
+          executor.coordinator.increment(:shutdown)
           job = @jobs_builder.shutdown(executor)
           jobs << job
         end
@@ -328,8 +336,6 @@ module Karafka
         idle_jobs = []
 
         @messages_buffer.each do |topic, partition, messages|
-          @usage_tracker.track(topic, partition)
-
           coordinator = @coordinators.find_or_create(topic, partition)
           # Start work coordination for this topic partition
           coordinator.start(messages)
@@ -337,12 +343,13 @@ module Karafka
           # We do not increment coordinator for idle job because it's not a user related one
           # and it will not go through a standard lifecycle. Same applies to revoked and shutdown
           if messages.empty?
+            coordinator.increment(:idle)
             executor = @executors.find_or_create(topic, partition, 0, coordinator)
             idle_jobs << @jobs_builder.idle(executor)
           else
             @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
+              coordinator.increment(:consume)
               executor = @executors.find_or_create(topic, partition, group_id, coordinator)
-              coordinator.increment
               consume_jobs << @jobs_builder.consume(executor, partition_messages)
             end
           end
@@ -391,11 +398,11 @@ module Karafka
           interval = topic.periodic_job.interval
 
           partitions.each do |partition|
-            # Skip if we were operating on a given topic partition recently
-            next if @usage_tracker.active?(topic_name, partition, interval)
-
             coordinator = @coordinators.find_or_create(topic_name, partition)
 
+            # Skip if we were operating on a given topic partition recently
+            next if coordinator.active_within?(interval)
+
             # Do not tick if we do not want to tick during pauses
             next if coordinator.paused? && !topic.periodic_job.during_pause?
 
@@ -405,10 +412,8 @@ module Karafka
             # run (ok) but attempt 1 means, there was an error and we will retry
             next if coordinator.attempt.positive? && !topic.periodic_job.during_retry?
 
-            # Track so we do not run periodic job again too soon
-            @usage_tracker.track(topic_name, partition)
-
             @executors.find_all_or_create(topic_name, partition, coordinator).each do |executor|
+              coordinator.increment(:periodic)
               jobs << @jobs_builder.periodic(executor)
             end
           end
@@ -423,8 +428,7 @@ module Karafka
       # Waits for all the jobs from a given subscription group to finish before moving forward
       def wait
         @jobs_queue.wait(@subscription_group.id) do
-          @events_poller.call
-          @scheduler.on_manage
+          @interval_runner.call
         end
       end
 
@@ -462,6 +466,7 @@ module Karafka
         @events_poller.reset
         @client.reset
         @coordinators.reset
+        @interval_runner.reset
         @executors = Processing::ExecutorsBuffer.new(@client, @subscription_group)
       end
     end

data/lib/karafka/connection/proxy.rb

@@ -11,12 +11,16 @@ module Karafka
     # it would be needed
     class Proxy < SimpleDelegator
       # Errors on which we want to retry
-      RETRYABLE_ERRORS = %i[
+      # Includes temporary errors related to node not being (or not yet being) coordinator or a
+      # leader to a given set of partitions. Usually goes away after a retry
+      RETRYABLE_DEFAULT_ERRORS = %i[
         all_brokers_down
         timed_out
+        not_coordinator
+        not_leader_for_partition
       ].freeze
 
-      private_constant :RETRYABLE_ERRORS
+      private_constant :RETRYABLE_DEFAULT_ERRORS
 
       attr_accessor :wrapped
 
@@ -42,6 +46,9 @@ module Karafka
       def query_watermark_offsets(topic, partition)
         l_config = @config.query_watermark_offsets
 
+        # For newly created topics or in cases where we're trying to get them but there is no
+        # leader, this can fail. It happens more often for new topics under KRaft, however we
+        # still want to make sure things operate as expected even then
         with_broker_errors_retry(
           # required to be in seconds, not ms
           wait_time: l_config.wait_time / 1_000.to_f,
@@ -85,6 +92,87 @@ module Karafka
         end
       end
 
+      # When we cannot store an offset, it means we no longer own the partition
+      #
+      # Non thread-safe offset storing method
+      # @param message [Karafka::Messages::Message]
+      # @param metadata [String, nil] offset storage metadata or nil if none
+      # @return [Boolean] true if we could store the offset (if we still own the partition)
+      def store_offset(message, metadata = nil)
+        @wrapped.store_offset(message, metadata)
+
+        true
+      rescue Rdkafka::RdkafkaError => e
+        return false if e.code == :assignment_lost
+        return false if e.code == :state
+
+        raise e
+      end
+
+      # Non thread-safe message committing method
+      # @param tpl [Rdkafka::Consumer::TopicPartitionList, nil] tpl or nil
+      # @param async [Boolean] should the commit happen async or sync (async by default)
+      # @return [Boolean] true if offset commit worked, false if we've lost the assignment
+      # @note We do **not** consider `no_offset` as any problem and we allow to commit offsets
+      #   even when no stored, because with sync commit, it refreshes the ownership state of the
+      #   consumer in a sync way.
+      def commit_offsets(tpl = nil, async: true)
+        c_config = @config.commit
+
+        with_broker_errors_retry(
+          wait_time: c_config.wait_time / 1_000.to_f,
+          max_attempts: c_config.max_attempts
+        ) do
+          @wrapped.commit(tpl, async)
+        end
+
+        true
+      rescue Rdkafka::RdkafkaError => e
+        case e.code
+        when :assignment_lost
+          return false
+        when :unknown_member_id
+          return false
+        when :no_offset
+          return true
+        when :coordinator_load_in_progress
+          sleep(1)
+          retry
+        end
+
+        raise e
+      end
+
+      # @param tpl [Rdkafka::Consumer::TopicPartitionList] list of topics and partitions for which
+      #   we want to get the lag on the defined CG
+      # @return [Hash<String, Hash>] hash with topics and their partitions lags
+      def lag(tpl)
+        l_config = @config.committed
+
+        with_broker_errors_retry(
+          # required to be in seconds, not ms
+          wait_time: l_config.wait_time / 1_000.to_f,
+          max_attempts: l_config.max_attempts
+        ) do
+          @wrapped.lag(tpl, l_config.timeout)
+        end
+      end
+
+      # @param topic_name [String, nil] Name of the topic we're interested in or nil if we want to
+      #   get info on all topics
+      # @return [Rdkafka::Metadata] rdkafka metadata object with the requested details
+      def metadata(topic_name = nil)
+        m_config = @config.metadata
+
+        with_broker_errors_retry(
+          # required to be in seconds, not ms
+          wait_time: m_config.wait_time / 1_000.to_f,
+          max_attempts: m_config.max_attempts
+        ) do
+          @wrapped.metadata(topic_name, m_config.timeout)
+        end
+      end
+
       private
 
       # Runs expected block of code with few retries on all_brokers_down
@@ -94,13 +182,14 @@ module Karafka
       #   completely.
       # @param wait_time [Integer, Float] how many seconds should we wait. It uses `#sleep` of Ruby
       #   so it needs time in seconds.
-      def with_broker_errors_retry(max_attempts:, wait_time: 1)
+      # @param errors [Array<Symbol>] rdkafka errors we want to retry on
+      def with_broker_errors_retry(max_attempts:, wait_time: 1, errors: RETRYABLE_DEFAULT_ERRORS)
         attempt ||= 0
         attempt += 1
 
         yield
       rescue Rdkafka::RdkafkaError => e
-        raise unless RETRYABLE_ERRORS.include?(e.code)
+        raise unless errors.include?(e.code)
 
         if attempt <= max_attempts
           sleep(wait_time)

data/lib/karafka/connection/status.rb

@@ -5,6 +5,11 @@ module Karafka
   module Connection
     # Listener connection status representation
     class Status
+      include Helpers::ConfigImporter.new(
+        monitor: %i[monitor],
+        conductor: %i[internal connection conductor]
+      )
+
       # Available states and their transitions.
       STATES = {
         pending: :pending!,
@@ -26,7 +31,8 @@ module Karafka
               return if @status && STATES.keys.index(:#{state}) <= STATES.keys.index(@status)
 
               @status = :#{state}
-              @conductor.signal
+              conductor.signal
+              monitor.instrument("connection.listener.#{state}", caller: self)
             end
           end
 
@@ -39,7 +45,6 @@ module Karafka
 
       def initialize
         @mutex = Mutex.new
-        @conductor = Karafka::App.config.internal.connection.conductor
         pending!
       end
 
@@ -49,11 +54,18 @@ module Karafka
       def stop!
         if pending?
           @status = :stopping
+          conductor.signal
+          monitor.instrument('connection.listener.stopping', caller: self)
+
           stopped!
         elsif stopped?
           nil
+        elsif stopping?
+          nil
         else
           @status = :stopping
+          conductor.signal
+          monitor.instrument('connection.listener.stopping', caller: self)
         end
       end
 

data/lib/karafka/contracts/config.rb

@@ -25,13 +25,14 @@ module Karafka
 
       required(:client_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
       required(:concurrency) { |val| val.is_a?(Integer) && val.positive? }
-      required(:consumer_mapper) { |val| !val.nil? }
       required(:consumer_persistence) { |val| [true, false].include?(val) }
       required(:pause_timeout) { |val| val.is_a?(Integer) && val.positive? }
       required(:pause_max_timeout) { |val| val.is_a?(Integer) && val.positive? }
       required(:pause_with_exponential_backoff) { |val| [true, false].include?(val) }
+      required(:strict_topics_namespacing) { |val| [true, false].include?(val) }
       required(:shutdown_timeout) { |val| val.is_a?(Integer) && val.positive? }
       required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
+      required(:group_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
 
       nested(:swarm) do
@@ -39,6 +40,12 @@ module Karafka
         required(:node) { |val| val == false || val.is_a?(Karafka::Swarm::Node) }
       end
 
+      nested(:oauth) do
+        required(:token_provider_listener) do |val|
+          val == false || val.respond_to?(:on_oauthbearer_token_refresh)
+        end
+      end
+
       nested(:admin) do
         # Can be empty because inherits values from the root kafka
         required(:kafka) { |val| val.is_a?(Hash) }
@@ -74,11 +81,17 @@ module Karafka
           required(:conductor) { |val| !val.nil? }
 
           nested(:proxy) do
+            nested(:commit) do
+              required(:max_attempts) { |val| val.is_a?(Integer) && val.positive? }
+              required(:wait_time) { |val| val.is_a?(Integer) && val.positive? }
+            end
+
             # All of them have the same requirements
             %i[
               query_watermark_offsets
               offsets_for_times
               committed
+              metadata
             ].each do |scope|
               nested(scope) do
                 required(:timeout) { |val| val.is_a?(Integer) && val.positive? }

data/lib/karafka/contracts/topic.rb

@@ -12,7 +12,7 @@ module Karafka
         ).fetch('en').fetch('validations').fetch('topic')
       end
 
-      required(:deserializer) { |val| !val.nil? }
+      required(:deserializers) { |val| !val.nil? }
       required(:id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
       required(:max_messages) { |val| val.is_a?(Integer) && val >= 1 }

data/lib/karafka/deserializers/headers.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Deserializers
+    # Default message headers deserializer
+    class Headers
+      # @param metadata [Karafka::Messages::Metadata] metadata object from which we obtain the
+      #   `#raw_headers`
+      # @return [Hash] expected message headers hash
+      def call(metadata)
+        metadata.raw_headers
+      end
+    end
+  end
+end

data/lib/karafka/deserializers/key.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Deserializers
+    # Default message key deserializer
+    class Key
+      # @param metadata [Karafka::Messages::Metadata] metadata object from which we obtain the
+      #   `#raw_key`
+      # @return [String, nil] expected message key in a string format or nil if no key
+      def call(metadata)
+        metadata.raw_key
+      end
+    end
+  end
+end

data/lib/karafka/deserializers/payload.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Karafka
+  # Module for all supported by default deserializers.
+  module Deserializers
+    # Default Karafka Json deserializer for loading JSON data in payload.
+    class Payload
+      # @param message [Karafka::Messages::Message] Message object that we want to deserialize
+      # @return [Hash] hash with deserialized JSON data
+      def call(message)
+        # nil payload can be present for example for tombstone messages
+        message.raw_payload.nil? ? nil : ::JSON.parse(message.raw_payload)
+      end
+    end
+  end
+end

data/lib/karafka/embedded.rb

@@ -8,6 +8,8 @@ module Karafka
       # so it won't interrupt other things running
       def start
         Thread.new do
+          Thread.current.name = 'karafka.embedded'
+
           Karafka::Process.tags.add(:execution_mode, 'embedded')
           Karafka::Server.start
         end

data/lib/karafka/helpers/async.rb

@@ -22,16 +22,19 @@ module Karafka
         def included(base)
           base.extend ::Forwardable
 
-          base.def_delegators :@thread, :join, :terminate, :alive?
+          base.def_delegators :@thread, :join, :terminate, :alive?, :name
         end
       end
 
       # Runs the `#call` method in a new thread
-      def async_call
+      # @param thread_name [String] name that we want to assign to the thread when we start it
+      def async_call(thread_name = '')
         MUTEX.synchronize do
           return if @thread&.alive?
 
           @thread = Thread.new do
+            Thread.current.name = thread_name
+
             Thread.current.abort_on_exception = true
 
             call

data/lib/karafka/helpers/colorize.rb

@@ -21,6 +21,12 @@ module Karafka
       def yellow(string)
         "\033[1;33m#{string}\033[0m"
       end
+
+      # @param string [String] string we want to have in grey
+      # @return [String] grey string
+      def grey(string)
+        "\e[38;5;244m#{string}\e[0m"
+      end
     end
   end
 end

data/lib/karafka/instrumentation/callbacks/oauthbearer_token_refresh.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Instrumentation
+    module Callbacks
+      # Callback that is triggered when oauth token needs to be refreshed.
+      class OauthbearerTokenRefresh
+        # @param bearer [Rdkafka::Consumer, Rdkafka::Admin] given rdkafka instance. It is needed as
+        #   we need to have a reference to call `#oauthbearer_set_token` or
+        #   `#oauthbearer_set_token_failure` upon the event.
+        def initialize(bearer)
+          @bearer = bearer
+        end
+
+        # @param _rd_config [Rdkafka::Config]
+        # @param bearer_name [String] name of the bearer for which we refresh
+        def call(_rd_config, bearer_name)
+          return unless @bearer.name == bearer_name
+
+          ::Karafka.monitor.instrument(
+            'oauthbearer.token_refresh',
+            bearer: @bearer,
+            caller: self
+          )
+        end
+      end
+    end
+  end
+end