karafka 2.4.0 → 2.4.18

data/lib/karafka/cli/base.rb

@@ -41,22 +41,38 @@ module Karafka
       class << self
         # Loads proper environment with what is needed to run the CLI
         def load
+          rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
+          is_rails = Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
+
+          # If the boot file is disabled and this is a Rails app, we assume that user moved the
+          # karafka app configuration to initializers or other Rails loading related place.
+          # It is not recommended but some users tend to do this. In such cases we just try to load
+          # the Rails stuff hoping that it will also load Karafka stuff
+          if Karafka.boot_file.to_s == 'false' && is_rails
+            require rails_env_rb
+
+            return
+          end
+
           # If there is a boot file, we need to require it as we expect it to contain
           # Karafka app setup, routes, etc
           if File.exist?(::Karafka.boot_file)
-            rails_env_rb = File.join(Dir.pwd, 'config/environment.rb')
-
             # Load Rails environment file that starts Rails, so we can reference consumers and
             # other things from `karafka.rb` file. This will work only for Rails, for non-rails
             # a manual setup is needed
-            require rails_env_rb if Kernel.const_defined?(:Rails) && File.exist?(rails_env_rb)
-
+            require rails_env_rb if is_rails
             require Karafka.boot_file.to_s
+
+            return
+          end
+
           # However when it is unavailable, we still want to be able to run help command
           # and install command as they don't require configured app itself to run
-          elsif %w[-h install].none? { |cmd| cmd == ARGV[0] }
-            raise ::Karafka::Errors::MissingBootFileError, ::Karafka.boot_file
-          end
+          return if %w[-h install].any? { |cmd| cmd == ARGV[0] }
+
+          # All other commands except help and install do require an existing boot file if it was
+          # declared
+          raise ::Karafka::Errors::MissingBootFileError, ::Karafka.boot_file
         end
 
         # Allows to set options for Thor cli
@@ -96,7 +112,7 @@ module Karafka
                 *[names, option[2], option[1]].flatten
               ) { |value| options[option[0]] = value }
             end
-          end.parse!
+          end.parse(ARGV)
 
           options
         end

data/lib/karafka/cli/install.rb

@@ -21,7 +21,7 @@ module Karafka
 
       # Where should we map proper files from templates
       INSTALL_FILES_MAP = {
-        'karafka.rb.erb' => Karafka.boot_file.basename,
+        'karafka.rb.erb' => Karafka.boot_file,
         'application_consumer.rb.erb' => 'app/consumers/application_consumer.rb',
         'example_consumer.rb.erb' => 'app/consumers/example_consumer.rb'
       }.freeze
@@ -51,6 +51,7 @@ module Karafka
 
         INSTALL_FILES_MAP.each do |source, target|
           pathed_target = Karafka.root.join(target)
+          FileUtils.mkdir_p File.dirname(pathed_target)
 
           template = File.read(Karafka.core_root.join("templates/#{source}"))
           render = ::ERB.new(template, trim_mode: '-').result(binding)

data/lib/karafka/cli/server.rb

@@ -89,6 +89,7 @@ module Karafka
         register_inclusions
         register_exclusions
 
+        Karafka::Server.execution_mode = :standalone
         Karafka::Server.run
       end
 

data/lib/karafka/cli/swarm.rb

@@ -23,6 +23,7 @@ module Karafka
         server.register_inclusions
         server.register_exclusions
 
+        Karafka::Server.execution_mode = :supervisor
         Karafka::Swarm::Supervisor.new.run
       end
     end

data/lib/karafka/cli/topics/align.rb

@@ -89,9 +89,19 @@ module Karafka
             desired_configs.transform_keys!(&:to_s)
 
             topic_with_configs.configs.each do |config|
-              next unless desired_configs.key?(config.name)
+              names = config.synonyms.map(&:name) << config.name
 
-              desired_config = desired_configs.fetch(config.name)
+              # We move forward only if given topic config is for altering
+              next if (desired_configs.keys & names).empty?
+
+              desired_config = nil
+
+              # We then find last defined value in our configs for a given attribute
+              # Since attributes can have synonyms, we select last one, which will represent the
+              # last defined value in case someone defined same multiple times
+              desired_configs.each do |name, value|
+                desired_config = value if names.include?(name)
+              end
 
               # Do not migrate if existing and desired values are the same
               next if desired_config == config.value

data/lib/karafka/cli/topics/plan.rb

@@ -16,7 +16,10 @@ module Karafka
             return false
           end
 
+          changes = false
+
           unless topics_to_create.empty?
+            changes = true
             puts 'Following topics will be created:'
             puts
 
@@ -33,20 +36,54 @@ module Karafka
           end
 
           unless topics_to_repartition.empty?
-            puts 'Following topics will be repartitioned:'
-            puts
+            upscale = {}
+            downscale = {}
 
             topics_to_repartition.each do |topic, partitions|
               from = partitions
               to = topic.declaratives.partitions
 
-              puts "  #{yellow('~')} #{topic.name}:"
-              puts "    #{yellow('~')} partitions: \"#{red(from)}\" #{grey('=>')} \"#{green(to)}\""
+              if from < to
+                upscale[topic] = partitions
+              else
+                downscale[topic] = partitions
+              end
+            end
+
+            unless upscale.empty?
+              changes = true
+              puts 'Following topics will be repartitioned:'
+              puts
+
+              upscale.each do |topic, partitions|
+                from = partitions
+                to = topic.declaratives.partitions
+                y = yellow('~')
+                puts "  #{y} #{topic.name}:"
+                puts "    #{y} partitions: \"#{red(from)}\" #{grey('=>')} \"#{green(to)}\""
+                puts
+              end
+            end
+
+            unless downscale.empty?
+              puts(
+                'Following topics repartitioning will be ignored as downscaling is not supported:'
+              )
               puts
+
+              downscale.each do |topic, partitions|
+                from = partitions
+                to = topic.declaratives.partitions
+
+                puts "  #{grey('~')} #{topic.name}:"
+                puts "    #{grey('~')} partitions: \"#{grey(from)}\" #{grey('=>')} \"#{grey(to)}\""
+                puts
+              end
             end
           end
 
           unless topics_to_alter.empty?
+            changes = true
             puts 'Following topics will have configuration changes:'
             puts
 
@@ -65,7 +102,7 @@ module Karafka
             end
           end
 
-          true
+          changes
         end
 
         private
@@ -143,14 +180,25 @@ module Karafka
               }
 
               scoped = @topics_to_alter[declarative][declarative_name]
+              # declarative name can be a synonym. In such cases we remap it during the discovery
+              # below
+              final_name = declarative_name
 
               topic_c.configs.each do |config|
-                next unless declarative_name == config.name
+                names = config.synonyms.map(&:name) << config.name
 
+                next unless names.include?(declarative_name)
+
+                # Always use a non-synonym name if differs
+                final_name = config.name
                 scoped[:action] = :change
                 scoped[:from] = config.value
               end
 
+              # Aligns the name in case synonym was used
+              target = @topics_to_alter[declarative].delete(declarative_name)
+              @topics_to_alter[declarative][final_name] = target
+
               # Remove change definitions that would migrate to the same value as present
               @topics_to_alter[declarative].delete_if do |_name, details|
                 details[:from] == details[:to]

data/lib/karafka/cli/topics.rb

@@ -10,26 +10,53 @@ module Karafka
       )
 
       desc 'Allows for the topics management'
+
+      option(
+        :detailed_exitcode,
+        'Exists with 0 when no changes, 1 when error and 2 when changes present or applied',
+        TrueClass,
+        %w[
+          --detailed_exitcode
+        ]
+      )
+
+      # We exit with 0 if no changes happened
+      NO_CHANGES_EXIT_CODE = 0
+
+      # When any changes happened (or could happen) we return 2 because 1 is default when Ruby
+      # crashes
+      CHANGES_EXIT_CODE = 2
+
+      private_constant :NO_CHANGES_EXIT_CODE, :CHANGES_EXIT_CODE
+
       # @param action [String] action we want to take
       def call(action = 'missing')
-        case action
-        when 'create'
-          Topics::Create.new.call
-        when 'delete'
-          Topics::Delete.new.call
-        when 'reset'
-          Topics::Reset.new.call
-        when 'repartition'
-          Topics::Repartition.new.call
-        when 'migrate'
-          Topics::Migrate.new.call
-        when 'align'
-          Topics::Align.new.call
-        when 'plan'
-          Topics::Plan.new.call
-        else
-          raise ::ArgumentError, "Invalid topics action: #{action}"
-        end
+        detailed_exit_code = options.fetch(:detailed_exitcode, false)
+
+        command = case action
+                  when 'create'
+                    Topics::Create
+                  when 'delete'
+                    Topics::Delete
+                  when 'reset'
+                    Topics::Reset
+                  when 'repartition'
+                    Topics::Repartition
+                  when 'migrate'
+                    Topics::Migrate
+                  when 'align'
+                    Topics::Align
+                  when 'plan'
+                    Topics::Plan
+                  else
+                    raise ::ArgumentError, "Invalid topics action: #{action}"
+                  end
+
+        changes = command.new.call
+
+        return unless detailed_exit_code
+
+        changes ? exit(CHANGES_EXIT_CODE) : exit(NO_CHANGES_EXIT_CODE)
       end
     end
   end

data/lib/karafka/connection/client.rb

@@ -8,6 +8,8 @@ module Karafka
     # It is threadsafe and provides some security measures so we won't end up operating on a
     # closed consumer instance as it causes Ruby VM process to crash.
     class Client
+      include ::Karafka::Core::Helpers::Time
+
       attr_reader :rebalance_manager
 
       # @return [Karafka::Routing::SubscriptionGroup] subscription group to which this client
@@ -24,7 +26,29 @@ module Karafka
       # How many times should we retry polling in case of a failure
       MAX_POLL_RETRIES = 20
 
-      private_constant :MAX_POLL_RETRIES
+      # How much time of the total shutdown time can we wait for our manual unsubscribe before
+      # attempting to close without unsubscribe. We try to wait for 50% of the shutdown time
+      # before we move to a regular unsubscribe.
+      COOP_UNSUBSCRIBE_FACTOR = 0.5
+
+      # Errors upon which we early report that something is off without retrying prior to the
+      # report
+      EARLY_REPORT_ERRORS = [
+        :inconsistent_group_protocol, # 23
+        :max_poll_exceeded, # -147
+        :network_exception, # 13
+        :transport, # -195
+        :topic_authorization_failed, # 29
+        :group_authorization_failed, # 30
+        :cluster_authorization_failed, # 31
+        :illegal_generation,
+        # this will not recover as fencing is permanent
+        :fenced, # -144
+        # This can happen for many reasons, including issues with static membership being fenced
+        :fatal # -150
+      ].freeze
+
+      private_constant :MAX_POLL_RETRIES, :COOP_UNSUBSCRIBE_FACTOR, :EARLY_REPORT_ERRORS
 
       # Creates a new consumer instance.
       #
@@ -42,7 +66,7 @@ module Karafka
         @subscription_group = subscription_group
         @buffer = RawMessagesBuffer.new
         @tick_interval = ::Karafka::App.config.internal.tick_interval
-        @rebalance_manager = RebalanceManager.new(@subscription_group.id)
+        @rebalance_manager = RebalanceManager.new(@subscription_group.id, @buffer)
         @rebalance_callback = Instrumentation::Callbacks::Rebalance.new(@subscription_group)
 
         @interval_runner = Helpers::IntervalRunner.new do
@@ -91,8 +115,21 @@ module Karafka
           # Fetch message within our time boundaries
           response = poll(time_poll.remaining)
 
-          # Put a message to the buffer if there is one
-          @buffer << response if response && response != :tick_time
+          # We track when last polling happened so we can provide means to detect upcoming
+          # `max.poll.interval.ms` limit
+          @buffer.polled
+
+          case response
+          when :tick_time
+            nil
+          # We get a hash only in case of eof error
+          when Hash
+            @buffer.eof(response[:topic], response[:partition])
+          when nil
+            nil
+          else
+            @buffer << response
+          end
 
           # Upon polling rebalance manager might have been updated.
           # If partition revocation happens, we need to remove messages from revoked partitions
@@ -104,7 +141,7 @@ module Karafka
             # Since rebalances do not occur often, we can run events polling as well without
             # any throttling
             events_poll
-            remove_revoked_and_duplicated_messages
+
             break
           end
 
@@ -115,10 +152,11 @@ module Karafka
           time_poll.checkpoint
 
           # Finally once we've (potentially) removed revoked, etc, if no messages were returned
-          # and it was not an early poll exist, we can break.
+          # and it was not an early poll exist, we can break. We also break if we got the eof
+          # signaling to propagate it asap
           # Worth keeping in mind, that the rebalance manager might have been updated despite no
           # messages being returned during a poll
-          break unless response
+          break if response.nil? || response.is_a?(Hash)
         end
 
         @buffer
@@ -257,14 +295,36 @@ module Karafka
 
       # Gracefully stops topic consumption.
       def stop
-        # In case of cooperative-sticky, there is a bug in librdkafka that may hang it.
-        # To mitigate it we first need to unsubscribe so we will not receive any assignments and
-        # only then we should be good to go.
+        # librdkafka has several constant issues when shutting down during rebalance. This is
+        # an issue that gets back every few versions of librdkafka in a limited scope, for example
+        # for cooperative-sticky or in a general scope. This is why we unsubscribe and wait until
+        # we no longer have any assignments. That way librdkafka consumer shutdown should never
+        # happen with rebalance associated with the given consumer instance. Since we do not want
+        # to wait forever, we also impose a limit on how long should we wait. This prioritizes
+        # shutdown stability over endless wait.
+        #
+        # The `@unsubscribing` ensures that when there would be a direct close attempt, it
+        # won't get into this loop again. This can happen when supervision decides it should close
+        # things faster
+        #
+        # @see https://github.com/confluentinc/librdkafka/issues/4792
         # @see https://github.com/confluentinc/librdkafka/issues/4527
-        if @subscription_group.kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
+        if unsubscribe?
+          @unsubscribing = true
+
+          # Give 50% of time for the final close before we reach the forceful
+          max_wait = ::Karafka::App.config.shutdown_timeout * COOP_UNSUBSCRIBE_FACTOR
+          used = 0
+          stopped_at = monotonic_now
+
           unsubscribe
 
           until assignment.empty?
+            used += monotonic_now - stopped_at
+            stopped_at = monotonic_now
+
+            break if used >= max_wait
+
             sleep(0.1)
 
             ping
@@ -402,7 +462,9 @@ module Karafka
           # then seek to the appropriate message
           # We set the timeout to 2_000 to make sure that remote clusters handle this well
           real_offsets = @wrapped_kafka.offsets_for_times(tpl)
-          detected_partition = real_offsets.to_h.dig(message.topic, message.partition)
+          # We always ask for one partition, so result will contain array with only one element
+          # that is the partition we were interested it regardless its number
+          detected_partition = real_offsets.to_h.dig(message.topic, 0)
 
           # There always needs to be an offset. In case we seek into the future, where there
           # are no offsets yet, we get -1 which indicates the most recent offset
@@ -545,11 +607,7 @@ module Karafka
         # We want to report early on max poll interval exceeding because it may mean that the
         # underlying processing is taking too much time and it is not LRJ
         case e.code
-        when :max_poll_exceeded # -147
-          early_report = true
-        when :network_exception # 13
-          early_report = true
-        when :transport # -195
+        when *EARLY_REPORT_ERRORS
           early_report = true
         # @see
         # https://github.com/confluentinc/confluent-kafka-dotnet/issues/1366#issuecomment-821842990
@@ -563,11 +621,12 @@ module Karafka
           # No sense in retrying when no topic/partition and we're no longer running
           retryable = false unless Karafka::App.running?
         # If we detect the end of partition which can happen if `enable.partition.eof` is set to
-        # true, we can just return nil fast. This will fast yield whatever set of messages we
+        # true, we can just return fast. This will fast yield whatever set of messages we
         # already have instead of waiting. This can be used for better latency control when we do
         # not expect a lof of lag and want to quickly move to processing.
+        # We can also pass the eof notion to the consumers for improved decision making.
         when :partition_eof
-          return nil
+          return e.details
         end
 
         if early_report || !retryable
@@ -647,24 +706,15 @@ module Karafka
         subscriptions = @subscription_group.subscriptions
         assignments = @subscription_group.assignments(consumer)
 
-        consumer.subscribe(*subscriptions) if subscriptions
-        consumer.assign(assignments) if assignments
-
-        consumer
-      end
-
-      # We may have a case where in the middle of data polling, we've lost a partition.
-      # In a case like this we should remove all the pre-buffered messages from list partitions as
-      # we are no longer responsible in a given process for processing those messages and they
-      # should have been picked up by a different process.
-      def remove_revoked_and_duplicated_messages
-        @rebalance_manager.lost_partitions.each do |topic, partitions|
-          partitions.each do |partition|
-            @buffer.delete(topic, partition)
-          end
+        if subscriptions
+          consumer.subscribe(*subscriptions)
+          @mode = :subscribe
+        elsif assignments
+          consumer.assign(assignments)
+          @mode = :assign
         end
 
-        @buffer.uniq!
+        consumer
       end
 
       # @return [Rdkafka::Consumer] librdkafka consumer instance
@@ -679,6 +729,23 @@ module Karafka
         @kafka.start
         @kafka
       end
+
+      # Decides whether or not we should unsubscribe prior to closing.
+      #
+      # We cannot do it when there is a static group membership assignment as it would be
+      # reassigned.
+      # We cannot do it also for assign mode because then there are no subscriptions
+      # We also do not do it if there are no assignments at all as it does not make sense
+      #
+      # @return [Boolean] should we unsubscribe prior to shutdown
+      def unsubscribe?
+        return false if @unsubscribing
+        return false if @subscription_group.kafka.key?(:'group.instance.id')
+        return false if @mode != :subscribe
+        return false if assignment.empty?
+
+        true
+      end
     end
   end
 end

data/lib/karafka/connection/listener.rb

@@ -253,7 +253,9 @@ module Karafka
 
         reset
 
-        sleep(1) && retry
+        # Ruby sleep is in seconds
+        sleep_time = ::Karafka::App.config.internal.connection.reset_backoff / 10_000.0
+        sleep(sleep_time) && retry
       end
 
       # Resumes processing of partitions that were paused due to an error.
@@ -330,28 +332,58 @@ module Karafka
       # given scheduler. It also handles the idle jobs when filtering API removed all messages
       # and we need to run house-keeping
       def build_and_schedule_flow_jobs
-        return if @messages_buffer.empty?
-
         consume_jobs = []
         idle_jobs = []
+        eofed_jobs = []
+
+        @messages_buffer.each do |topic, partition, messages, eof, last_polled_at|
+          # In case we did not receive any new messages without eof we skip.
+          # We may yield empty array here in case we have reached eof without new messages but in
+          # such cases, we can run an eof job
+          next if messages.empty? && !eof
 
-        @messages_buffer.each do |topic, partition, messages|
           coordinator = @coordinators.find_or_create(topic, partition)
-          # Start work coordination for this topic partition
+          coordinator.eofed = eof
+          coordinator.last_polled_at = last_polled_at
+
+          # If we did not receive any messages and we did receive eof signal, we run the eofed
+          # jobs so user can take actions on reaching eof
+          if messages.empty? && eof
+            # If user wants to run the eofed jobs on eof we do it. Otherwise we just allow it to
+            # pass through. This allows to configure if user actually wants to have `#eofed`
+            # logic or if he wants to only use fast eof work yield
+            if coordinator.topic.eofed?
+              @executors.find_all_or_create(topic, partition, coordinator).each do |executor|
+                coordinator.increment(:eofed)
+                eofed_jobs << @jobs_builder.eofed(executor)
+              end
+            end
+
+            next
+          end
+
           coordinator.start(messages)
 
+          # If it is not an eof and there are no new messages, we just run house-keeping
+          #
           # We do not increment coordinator for idle job because it's not a user related one
           # and it will not go through a standard lifecycle. Same applies to revoked and shutdown
           if messages.empty?
+            # Start work coordination for this topic partition
             coordinator.increment(:idle)
             executor = @executors.find_or_create(topic, partition, 0, coordinator)
             idle_jobs << @jobs_builder.idle(executor)
-          else
-            @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
-              coordinator.increment(:consume)
-              executor = @executors.find_or_create(topic, partition, group_id, coordinator)
-              consume_jobs << @jobs_builder.consume(executor, partition_messages)
-            end
+
+            next
+          end
+
+          # If there are messages, it is irrelevant if eof or not as consumption needs to happen
+          #
+          # Start work coordination for this topic partition
+          @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
+            coordinator.increment(:consume)
+            executor = @executors.find_or_create(topic, partition, group_id, coordinator)
+            consume_jobs << @jobs_builder.consume(executor, partition_messages)
           end
         end
 
@@ -367,6 +399,11 @@ module Karafka
           consume_jobs.each(&:before_schedule)
           @scheduler.on_schedule_consumption(consume_jobs)
         end
+
+        unless eofed_jobs.empty?
+          eofed_jobs.each(&:before_schedule)
+          @scheduler.on_schedule_eofed(eofed_jobs)
+        end
       end
 
       # Builds and schedules periodic jobs for topics partitions for which no messages were

data/lib/karafka/connection/messages_buffer.rb

@@ -23,9 +23,13 @@ module Karafka
       def initialize(subscription_group)
         @subscription_group = subscription_group
         @size = 0
+
         @groups = Hash.new do |topic_groups, topic|
           topic_groups[topic] = Hash.new do |partition_groups, partition|
-            partition_groups[partition] = []
+            partition_groups[partition] = {
+              eof: false,
+              messages: []
+            }
           end
         end
       end
@@ -33,24 +37,31 @@ module Karafka
       # Remaps raw messages from the raw messages buffer to Karafka messages
       # @param raw_messages_buffer [RawMessagesBuffer] buffer with raw messages
       def remap(raw_messages_buffer)
-        clear unless @size.zero?
+        clear
 
         # Since it happens "right after" we've received the messages, it is close enough it time
         # to be used as the moment we received messages.
         received_at = Time.now
+        last_polled_at = raw_messages_buffer.last_polled_at
 
-        raw_messages_buffer.each do |topic, partition, messages|
+        raw_messages_buffer.each do |topic, partition, messages, eof|
           @size += messages.count
 
           ktopic = @subscription_group.topics.find(topic)
 
-          @groups[topic][partition] = messages.map do |message|
+          built_messages = messages.map do |message|
             Messages::Builders::Message.call(
               message,
               ktopic,
               received_at
             )
           end
+
+          @groups[topic][partition] = {
+            eof: eof,
+            messages: built_messages,
+            last_polled_at: last_polled_at
+          }
         end
       end
 
@@ -59,10 +70,12 @@ module Karafka
       # @yieldparam [String] topic name
       # @yieldparam [Integer] partition number
       # @yieldparam [Array<Karafka::Messages::Message>] messages from a given topic partition
+      # @yieldparam [Boolean] true if eof, false otherwise
+      # @yieldparam [Float] last polled at monotonic clock time
       def each
         @groups.each do |topic, partitions|
-          partitions.each do |partition, messages|
-            yield(topic, partition, messages)
+          partitions.each do |partition, details|
+            yield(topic, partition, details[:messages], details[:eof], details[:last_polled_at])
           end
         end
       end