karafka 2.0.0 → 2.1.0

data/lib/karafka/connection/client.rb

@@ -14,17 +14,25 @@ module Karafka
       # @note Consumer name may change in case we regenerate it
       attr_reader :name
 
+      # @return [String] id of the client
+      attr_reader :id
+
       # How many times should we retry polling in case of a failure
-      MAX_POLL_RETRIES = 10
+      MAX_POLL_RETRIES = 20
+
+      # We want to make sure we never close several clients in the same moment to prevent
+      # potential race conditions and other issues
+      SHUTDOWN_MUTEX = Mutex.new
 
-      private_constant :MAX_POLL_RETRIES
+      private_constant :MAX_POLL_RETRIES, :SHUTDOWN_MUTEX
 
       # Creates a new consumer instance.
       #
       # @param subscription_group [Karafka::Routing::SubscriptionGroup] subscription group
       #   with all the configuration details needed for us to create a client
-      # @return [Karafka::Connection::Rdk::Consumer]
+      # @return [Karafka::Connection::Client]
       def initialize(subscription_group)
+        @id = SecureRandom.hex(6)
         # Name is set when we build consumer
         @name = ''
         @mutex = Mutex.new
@@ -153,7 +161,7 @@ module Karafka
 
         pause_msg = Messages::Seek.new(topic, partition, offset)
 
-        internal_commit_offsets(async: false)
+        internal_commit_offsets(async: true)
 
         # Here we do not use our cached tpls because we should not try to pause something we do
         # not own anymore.
@@ -161,6 +169,15 @@ module Karafka
 
         return unless tpl
 
+        Karafka.monitor.instrument(
+          'client.pause',
+          caller: self,
+          subscription_group: @subscription_group,
+          topic: topic,
+          partition: partition,
+          offset: offset
+        )
+
         @paused_tpls[topic][partition] = tpl
 
         @kafka.pause(tpl)
@@ -179,19 +196,26 @@ module Karafka
 
         return if @closed
 
-        # Always commit synchronously offsets if any when we resume
-        # This prevents resuming without offset in case it would not be committed prior
-        # We can skip performance penalty since resuming should not happen too often
-        internal_commit_offsets(async: false)
+        # We now commit offsets on rebalances, thus we can do it async just to make sure
+        internal_commit_offsets(async: true)
 
         # If we were not able, let's try to reuse the one we have (if we have)
         tpl = topic_partition_list(topic, partition) || @paused_tpls[topic][partition]
 
         return unless tpl
+
         # If we did not have it, it means we never paused this partition, thus no resume should
         # happen in the first place
         return unless @paused_tpls[topic].delete(partition)
 
+        Karafka.monitor.instrument(
+          'client.resume',
+          caller: self,
+          subscription_group: @subscription_group,
+          topic: topic,
+          partition: partition
+        )
+
         @kafka.resume(tpl)
       ensure
         @mutex.unlock
@@ -238,6 +262,17 @@ module Karafka
         end
       end
 
+      # Runs a single poll ignoring all the potential errors
+      # This is used as a keep-alive in the shutdown stage and any errors that happen here are
+      # irrelevant from the shutdown process perspective
+      #
+      # This is used only to trigger rebalance callbacks
+      def ping
+        poll(100)
+      rescue Rdkafka::RdkafkaError
+        nil
+      end
+
       private
 
       # When we cannot store an offset, it means we no longer own the partition
@@ -267,32 +302,41 @@ module Karafka
 
         true
       rescue Rdkafka::RdkafkaError => e
-        return false if e.code == :assignment_lost
-        return true if e.code == :no_offset
+        case e.code
+        when :assignment_lost
+          return false
+        when :no_offset
+          return true
+        when :coordinator_load_in_progress
+          sleep(1)
+          retry
+        end
 
         raise e
       end
 
       # Commits the stored offsets in a sync way and closes the consumer.
       def close
-        # Once client is closed, we should not close it again
-        # This could only happen in case of a race-condition when forceful shutdown happens
-        # and triggers this from a different thread
-        return if @closed
-
-        @mutex.synchronize do
-          internal_commit_offsets(async: false)
-
-          @closed = true
-
-          # Remove callbacks runners that were registered
-          ::Karafka::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
-          ::Karafka::Instrumentation.error_callbacks.delete(@subscription_group.id)
-
-          @kafka.close
-          @buffer.clear
-          # @note We do not clear rebalance manager here as we may still have revocation info here
-          # that we want to consider valid prior to running another reconnection
+        # Allow only one client to be closed at the same time
+        SHUTDOWN_MUTEX.synchronize do
+          # Make sure that no other operations are happening on this client when we close it
+          @mutex.synchronize do
+            # Once client is closed, we should not close it again
+            # This could only happen in case of a race-condition when forceful shutdown happens
+            # and triggers this from a different thread
+            return if @closed
+
+            @closed = true
+
+            # Remove callbacks runners that were registered
+            ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@subscription_group.id)
+            ::Karafka::Core::Instrumentation.error_callbacks.delete(@subscription_group.id)
+
+            @kafka.close
+            @buffer.clear
+            # @note We do not clear rebalance manager here as we may still have revocation info
+            # here that we want to consider valid prior to running another reconnection
+          end
         end
       end
 
@@ -310,7 +354,7 @@ module Karafka
         Rdkafka::Consumer::TopicPartitionList.new({ topic => [rdkafka_partition] })
       end
 
-      # Performs a single poll operation.
+      # Performs a single poll operation and handles retries and error
       #
       # @param timeout [Integer] timeout for a single poll
       # @return [Rdkafka::Consumer::Message, nil] fetched message or nil if nothing polled
@@ -323,41 +367,56 @@ module Karafka
 
         @kafka.poll(timeout)
       rescue ::Rdkafka::RdkafkaError => e
-        # We return nil, so we do not restart until running the whole loop
-        # This allows us to run revocation jobs and other things and we will pick up new work
-        # next time after dispatching all the things that are needed
-        #
-        # If we would retry here, the client reset would become transparent and we would not have
-        # a chance to take any actions
+        early_report = false
+
+        retryable = time_poll.attempts <= MAX_POLL_RETRIES && time_poll.retryable?
+
+        # There are retryable issues on which we want to report fast as they are source of
+        # problems and can mean some bigger system instabilities
+        # Those are mainly network issues and exceeding the max poll interval
+        # We want to report early on max poll interval exceeding because it may mean that the
+        # underlying processing is taking too much time and it is not LRJ
         case e.code
         when :max_poll_exceeded # -147
-          reset
-          return nil
-        when :transport # -195
-          reset
-          return nil
-        when :rebalance_in_progress # -27
-          reset
-          return nil
-        when :not_coordinator # 16
-          reset
-          return nil
+          early_report = true
         when :network_exception # 13
-          reset
-          return nil
-        when :unknown_topic_or_part
-          # This is expected and temporary until rdkafka catches up with metadata
-          return nil
+          early_report = true
+        when :transport # -195
+          early_report = true
+        # @see
+        # https://github.com/confluentinc/confluent-kafka-dotnet/issues/1366#issuecomment-821842990
+        # This will be raised each time poll detects a non-existing topic. When auto creation is
+        # on, we can safely ignore it
+        when :unknown_topic_or_part # 3
+          return nil if @subscription_group.kafka[:'allow.auto.create.topics']
+
+          early_report = true
+
+          # No sense in retrying when no topic/partition and we're no longer running
+          retryable = false unless Karafka::App.running?
         end
 
-        raise if time_poll.attempts > MAX_POLL_RETRIES
-        raise unless time_poll.retryable?
+        if early_report || !retryable
+          Karafka.monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            type: 'connection.client.poll.error'
+          )
+        end
+
+        raise unless retryable
+
+        # Most of the errors can be safely ignored as librdkafka will recover from them
+        # @see https://github.com/edenhill/librdkafka/issues/1987#issuecomment-422008750
+        # @see https://github.com/edenhill/librdkafka/wiki/Error-handling
 
         time_poll.checkpoint
         time_poll.backoff
 
-        # On unknown errors we do our best to retry and handle them before raising
-        retry
+        # poll may not only return message but also can run callbacks and if they changed,
+        # despite the errors we need to delegate to the other app parts
+        @rebalance_manager.changed? ? nil : retry
       end
 
       # Builds a new rdkafka consumer instance based on the subscription group configuration
@@ -370,24 +429,22 @@ module Karafka
         @name = consumer.name
 
         # Register statistics runner for this particular type of callbacks
-        ::Karafka::Instrumentation.statistics_callbacks.add(
+        ::Karafka::Core::Instrumentation.statistics_callbacks.add(
           @subscription_group.id,
           Instrumentation::Callbacks::Statistics.new(
             @subscription_group.id,
             @subscription_group.consumer_group_id,
-            @name,
-            ::Karafka::App.config.monitor
+            @name
           )
         )
 
         # Register error tracking callback
-        ::Karafka::Instrumentation.error_callbacks.add(
+        ::Karafka::Core::Instrumentation.error_callbacks.add(
           @subscription_group.id,
           Instrumentation::Callbacks::Error.new(
             @subscription_group.id,
             @subscription_group.consumer_group_id,
-            @name,
-            ::Karafka::App.config.monitor
+            @name
           )
         )
 

data/lib/karafka/connection/consumer_group_coordinator.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Connection
+    # This object represents a collective status of execution of group of listeners running inside
+    # of one consumer group but in separate subscription groups.
+    #
+    # There are cases when we do not want to close a given client when others from the same
+    # consumer group are running because it can cause instabilities due to early shutdown of some
+    # of the clients out of same consumer group.
+    #
+    # We also want to make sure, we close one consumer at a time while others can continue polling.
+    #
+    # This prevents a scenario, where a rebalance is not acknowledged and we loose assignment
+    # without having a chance to commit changes.
+    class ConsumerGroupCoordinator
+      # @param group_size [Integer] number of separate subscription groups in a consumer group
+      def initialize(group_size)
+        @shutdown_lock = Mutex.new
+        @group_size = group_size
+        @finished = Set.new
+      end
+
+      # @return [Boolean] true if all the subscription groups from a given consumer group are
+      #   finished
+      def finished?
+        @finished.size == @group_size
+      end
+
+      # @return [Boolean] can we start shutdown on a given listener
+      # @note If true, will also obtain a lock so no-one else will be closing the same time we do
+      def shutdown?
+        finished? && @shutdown_lock.try_lock
+      end
+
+      # Unlocks the shutdown lock
+      def unlock
+        @shutdown_lock.unlock if @shutdown_lock.owned?
+      end
+
+      # Marks given listener as finished
+      # @param listener_id [String]
+      def finish_work(listener_id)
+        @finished << listener_id
+      end
+    end
+  end
+end

data/lib/karafka/connection/listener.rb

@@ -14,16 +14,18 @@ module Karafka
       # @return [String] id of this listener
       attr_reader :id
 
+      # @param consumer_group_coordinator [Karafka::Connection::ConsumerGroupCoordinator]
       # @param subscription_group [Karafka::Routing::SubscriptionGroup]
       # @param jobs_queue [Karafka::Processing::JobsQueue] queue where we should push work
       # @return [Karafka::Connection::Listener] listener instance
-      def initialize(subscription_group, jobs_queue)
+      def initialize(consumer_group_coordinator, subscription_group, jobs_queue)
         proc_config = ::Karafka::App.config.internal.processing
 
-        @id = SecureRandom.uuid
+        @id = SecureRandom.hex(6)
+        @consumer_group_coordinator = consumer_group_coordinator
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
-        @coordinators = Processing::CoordinatorsBuffer.new
+        @coordinators = Processing::CoordinatorsBuffer.new(subscription_group.topics)
         @client = Client.new(@subscription_group)
         @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
         @jobs_builder = proc_config.jobs_builder
@@ -46,8 +48,8 @@ module Karafka
         Karafka.monitor.instrument(
           'connection.listener.before_fetch_loop',
           caller: self,
-          subscription_group: @subscription_group,
-          client: @client
+          client: @client,
+          subscription_group: @subscription_group
         )
 
         fetch_loop
@@ -67,7 +69,6 @@ module Karafka
           @stopped = true
           @executors.clear
           @coordinators.reset
-          @client.commit_offsets!
           @client.stop
         end
       end
@@ -83,11 +84,13 @@ module Karafka
       #   Kafka connections / Internet connection issues / Etc. Business logic problems should not
       #   propagate this far.
       def fetch_loop
-        until Karafka::App.stopping?
+        # Run the main loop as long as we are not stopping or moving into quiet mode
+        until Karafka::App.stopping? || Karafka::App.quieting? || Karafka::App.quiet?
           Karafka.monitor.instrument(
             'connection.listener.fetch_loop',
             caller: self,
-            client: @client
+            client: @client,
+            subscription_group: @subscription_group
           )
 
           resume_paused_partitions
@@ -95,6 +98,8 @@ module Karafka
           Karafka.monitor.instrument(
             'connection.listener.fetch_loop.received',
             caller: self,
+            client: @client,
+            subscription_group: @subscription_group,
             messages_buffer: @messages_buffer
           ) do
             # We need to fetch data before we revoke lost partitions details as during the polling
@@ -118,13 +123,9 @@ module Karafka
           build_and_schedule_consumption_jobs
 
           wait
-
-          # We don't use the `#commit_offsets!` here for performance reasons. This can be achieved
-          # if needed by using manual offset management.
-          @client.commit_offsets
         end
 
-        # If we are stopping we will no longer schedule any jobs despite polling.
+        # If we are stopping we will no longer schedule any regular jobs despite polling.
         # We need to keep polling not to exceed the `max.poll.interval` for long-running
         # non-blocking jobs and we need to allow them to finish. We however do not want to
         # enqueue any new jobs. It's worth keeping in mind that it is the end user responsibility
@@ -133,7 +134,14 @@ module Karafka
         #
         # We do not care about resuming any partitions or lost jobs as we do not plan to do
         # anything with them as we're in the shutdown phase.
-        wait_with_poll
+        #
+        # What we do care however is the ability to still run revocation jobs in case anything
+        # would change in the cluster. We still want to notify the long-running jobs about changes
+        # that occurred in the cluster.
+        wait_pinging(
+          wait_until: -> { @jobs_queue.empty?(@subscription_group.id) },
+          after_ping: -> { build_and_schedule_revoke_lost_partitions_jobs }
+        )
 
         # We do not want to schedule the shutdown jobs prior to finishing all the jobs
         # (including non-blocking) as there might be a long-running job with a shutdown and then
@@ -141,7 +149,25 @@ module Karafka
         # as it could create a race-condition.
         build_and_schedule_shutdown_jobs
 
-        wait_with_poll
+        # Wait until all the shutdown jobs are done
+        wait_pinging(wait_until: -> { @jobs_queue.empty?(@subscription_group.id) })
+
+        # Once all the work is done, we need to decrement counter of active subscription groups
+        # within this consumer group
+        @consumer_group_coordinator.finish_work(id)
+
+        # Wait if we're in the process of finishing started work or finished all the work and
+        # just sitting and being quiet
+        wait_pinging(wait_until: -> { !(Karafka::App.quieting? || Karafka::App.quiet?) })
+
+        # We need to wait until all the work in the whole consumer group (local to the process)
+        # is done. Otherwise we may end up with locks and `Timed out LeaveGroupRequest in flight`
+        # warning notifications.
+        wait_pinging(wait_until: -> { @consumer_group_coordinator.shutdown? })
+
+        # This extra ping will make sure we've refreshed the rebalance state after other instances
+        # potentially shutdown. This will prevent us from closing with a dangling callback
+        @client.ping
 
         shutdown
 
@@ -159,6 +185,8 @@ module Karafka
         restart
 
         sleep(1) && retry
+      ensure
+        @consumer_group_coordinator.unlock
       end
 
       # Resumes processing of partitions that were paused due to an error.
@@ -185,7 +213,9 @@ module Karafka
             # processed (if it was assigned and revoked really fast), thus we may not have it
             # here. In cases like this, we do not run a revocation job
             @executors.find_all(topic, partition).each do |executor|
-              jobs << @jobs_builder.revoked(executor)
+              job = @jobs_builder.revoked(executor)
+              job.before_enqueue
+              jobs << job
             end
 
             # We need to remove all the executors of a given topic partition that we have lost, so
@@ -204,8 +234,10 @@ module Karafka
       def build_and_schedule_shutdown_jobs
         jobs = []
 
-        @executors.each do |_, _, executor|
-          jobs << @jobs_builder.shutdown(executor)
+        @executors.each do |executor|
+          job = @jobs_builder.shutdown(executor)
+          job.before_enqueue
+          jobs << job
         end
 
         @scheduler.schedule_shutdown(@jobs_queue, jobs)
@@ -231,20 +263,25 @@ module Karafka
 
         @messages_buffer.each do |topic, partition, messages|
           coordinator = @coordinators.find_or_create(topic, partition)
-
           # Start work coordination for this topic partition
           coordinator.start(messages)
 
-          @partitioner.call(topic, messages) do |group_id, partition_messages|
-            # Count the job we're going to create here
-            coordinator.increment
-
-            executor = @executors.find_or_create(topic, partition, group_id)
-
-            jobs << @jobs_builder.consume(executor, partition_messages, coordinator)
+          # We do not increment coordinator for idle job because it's not a user related one
+          # and it will not go through a standard lifecycle. Same applies to revoked and shutdown
+          if messages.empty?
+            executor = @executors.find_or_create(topic, partition, 0, coordinator)
+            jobs << @jobs_builder.idle(executor)
+          else
+            @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
+              executor = @executors.find_or_create(topic, partition, group_id, coordinator)
+              coordinator.increment
+              jobs << @jobs_builder.consume(executor, partition_messages)
+            end
           end
         end
 
+        jobs.each(&:before_enqueue)
+
         @scheduler.schedule_consumption(@jobs_queue, jobs)
       end
 
@@ -254,10 +291,21 @@ module Karafka
       end
 
       # Waits without blocking the polling
-      # This should be used only when we no longer plan to use any incoming data and we can safely
-      # discard it
-      def wait_with_poll
-        @client.batch_poll until @jobs_queue.empty?(@subscription_group.id)
+      #
+      # This should be used only when we no longer plan to use any incoming messages data and we
+      # can safely discard it. We can however use the rebalance information if needed.
+      #
+      # @param wait_until [Proc] until this evaluates to true, we will poll data
+      # @param after_ping [Proc] code that we want to run after each ping (if any)
+      #
+      # @note Performance of this is not relevant (in regards to blocks) because it is used only
+      #   on shutdown and quiet, hence not in the running mode
+      def wait_pinging(wait_until:, after_ping: -> {})
+        until wait_until.call
+          @client.ping
+          after_ping.call
+          sleep(0.2)
+        end
       end
 
       # We can stop client without a problem, as it will reinitialize itself when running the

data/lib/karafka/connection/listeners_batch.rb

@@ -6,11 +6,27 @@ module Karafka
     class ListenersBatch
       include Enumerable
 
+      attr_reader :coordinators
+
       # @param jobs_queue [JobsQueue]
       # @return [ListenersBatch]
       def initialize(jobs_queue)
-        @batch = App.subscription_groups.map do |subscription_group|
-          Connection::Listener.new(subscription_group, jobs_queue)
+        @coordinators = []
+
+        @batch = App.subscription_groups.flat_map do |_consumer_group, subscription_groups|
+          consumer_group_coordinator = Connection::ConsumerGroupCoordinator.new(
+            subscription_groups.size
+          )
+
+          @coordinators << consumer_group_coordinator
+
+          subscription_groups.map do |subscription_group|
+            Connection::Listener.new(
+              consumer_group_coordinator,
+              subscription_group,
+              jobs_queue
+            )
+          end
         end
       end
 

data/lib/karafka/connection/rebalance_manager.rb

@@ -14,6 +14,9 @@ module Karafka
     #   that are lost, are those that got revoked but did not get re-assigned back. We do not
     #   expose this concept outside and we normalize to have them revoked, as it is irrelevant
     #   from the rest of the code perspective as only those that are lost are truly revoked.
+    #
+    # @note For cooperative-sticky `#assigned_partitions` holds only the recently assigned
+    #   partitions, not all the partitions that are owned
     class RebalanceManager
       # Empty array for internal usage not to create new objects
       EMPTY_ARRAY = [].freeze

data/lib/karafka/contracts/config.rb

@@ -12,7 +12,7 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('config')
       end

data/lib/karafka/contracts/consumer_group.rb

@@ -7,13 +7,13 @@ module Karafka
       configure do |config|
         config.error_messages = YAML.safe_load(
           File.read(
-            File.join(Karafka.gem_root, 'config', 'errors.yml')
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
           )
         ).fetch('en').fetch('validations').fetch('consumer_group')
       end
 
-      required(:id) { |id| id.is_a?(String) && Contracts::TOPIC_REGEXP.match?(id) }
-      required(:topics) { |topics| topics.is_a?(Array) && !topics.empty? }
+      required(:id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
+      required(:topics) { |val| val.is_a?(Array) && !val.empty? }
 
       virtual do |data, errors|
         next unless errors.empty?
@@ -27,16 +27,29 @@ module Karafka
 
       virtual do |data, errors|
         next unless errors.empty?
+        next unless ::Karafka::App.config.strict_topics_namespacing
 
-        fetched_errors = []
-
-        data.fetch(:topics).each do |topic|
-          ConsumerGroupTopic.new.call(topic).errors.each do |key, value|
-            fetched_errors << [[topic, key].flatten, value]
+        names = data.fetch(:topics).map { |topic| topic[:name] }
+        names_hash = names.each_with_object({}) { |n, h| h[n] = true }
+        error_occured = false
+        names.each do |n|
+          # Skip topic names that are not namespaced
+          next unless n.chars.find { |c| ['.', '_'].include?(c) }
+
+          if n.chars.include?('.')
+            # Check underscore styled topic
+            underscored_topic = n.tr('.', '_')
+            error_occured = names_hash[underscored_topic] ? true : false
+          else
+            # Check dot styled topic
+            dot_topic = n.tr('_', '.')
+            error_occured = names_hash[dot_topic] ? true : false
           end
         end
 
-        fetched_errors
+        next unless error_occured
+
+        [[%i[topics], :topics_namespaced_names_not_unique]]
       end
     end
   end