karafka 2.4.0 → 2.4.18

data/lib/karafka/connection/proxy.rb

@@ -16,8 +16,11 @@ module Karafka
       RETRYABLE_DEFAULT_ERRORS = %i[
         all_brokers_down
         timed_out
+        transport
+        network_exception
         not_coordinator
         not_leader_for_partition
+        coordinator_load_in_progress
       ].freeze
 
       private_constant :RETRYABLE_DEFAULT_ERRORS

data/lib/karafka/connection/raw_messages_buffer.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   module Connection
-    # Buffer for raw librdkafka messages.
+    # Buffer for raw librdkafka messages and eof status.
     #
     # When message is added to this buffer, it gets assigned to an array with other messages from
     # the same topic and partition.
@@ -12,14 +12,24 @@ module Karafka
     # @note We store data here in groups per topic partition to handle the revocation case, where
     #    we may need to remove messages from a single topic partition.
     class RawMessagesBuffer
+      include Karafka::Core::Helpers::Time
+
       attr_reader :size
 
+      # @return [Float] last polling time in milliseconds (monotonic)
+      attr_reader :last_polled_at
+
       # @return [Karafka::Connection::MessagesBuffer] buffer instance
       def initialize
         @size = 0
+        @last_polled_at = monotonic_now
+
         @groups = Hash.new do |topic_groups, topic|
           topic_groups[topic] = Hash.new do |partition_groups, partition|
-            partition_groups[partition] = []
+            partition_groups[partition] = {
+              eof: false,
+              messages: []
+            }
           end
         end
       end
@@ -30,7 +40,21 @@ module Karafka
       # @return [Array<Rdkafka::Consumer::Message>] given partition topic sub-buffer array
       def <<(message)
         @size += 1
-        @groups[message.topic][message.partition] << message
+        partition_state = @groups[message.topic][message.partition]
+        partition_state[:messages] << message
+        partition_state[:eof] = false
+      end
+
+      # Marks given topic partition as one that reached eof
+      # @param topic [String] topic that reached eof
+      # @param partition [Integer] partition that reached eof
+      def eof(topic, partition)
+        @groups[topic][partition][:eof] = true
+      end
+
+      # Marks the last polling time that can be accessed via `#last_polled_at`
+      def polled
+        @last_polled_at = monotonic_now
       end
 
       # Allows to iterate over all the topics and partitions messages
@@ -38,10 +62,11 @@ module Karafka
       # @yieldparam [String] topic name
       # @yieldparam [Integer] partition number
       # @yieldparam [Array<Rdkafka::Consumer::Message>] topic partition aggregated results
+      # @yieldparam [Boolean] has polling of this partition reach eof
       def each
         @groups.each do |topic, partitions|
-          partitions.each do |partition, messages|
-            yield(topic, partition, messages)
+          partitions.each do |partition, details|
+            yield(topic, partition, details[:messages], details[:eof])
           end
         end
       end
@@ -69,8 +94,8 @@ module Karafka
       # again and we do want to ensure as few duplications as possible
       def uniq!
         @groups.each_value do |partitions|
-          partitions.each_value do |messages|
-            messages.uniq!(&:offset)
+          partitions.each_value do |details|
+            details[:messages].uniq!(&:offset)
           end
         end
 
@@ -83,6 +108,11 @@ module Karafka
       #   we save ourselves some objects allocations. We cannot clear the underlying arrays as they
       #   may be used in other threads for data processing, thus if we would clear it, we could
       #   potentially clear a raw messages array for a job that is in the jobs queue.
+      #
+      # @note We do not clear the eof assignments because they can span across batch pollings.
+      #   Since eof is not raised non-stop and is silenced after an eof poll, if we would clean it
+      #   here we would loose the notion of it. The reset state for it should happen when we do
+      #   discover new messages for given topic partition.
       def clear
         @size = 0
         @groups.each_value(&:clear)
@@ -92,8 +122,12 @@ module Karafka
 
       # Updates the messages count if we performed any operations that could change the state
       def recount!
-        @size = @groups.each_value.sum do |partitions|
-          partitions.each_value.map(&:count).sum
+        @size = 0
+
+        @groups.each_value do |partitions|
+          partitions.each_value do |details|
+            @size += details[:messages].size
+          end
         end
       end
     end

data/lib/karafka/connection/rebalance_manager.rb

@@ -2,7 +2,9 @@
 
 module Karafka
   module Connection
-    # Manager for tracking changes in the partitions assignment after the assignment is done.
+    # Manager for tracking changes in the partitions assignment after the assignment is done and
+    # for ensuring, that proper buffer related operations that may be impacted by the rebalance
+    # state are applied.
     #
     # We need tracking of those to clean up consumers that will no longer process given partitions
     # as they were taken away.
@@ -30,13 +32,15 @@ module Karafka
       private_constant :EMPTY_ARRAY
 
       # @param subscription_group_id [String] subscription group id
+      # @param buffer [Karafka::Connection::RawMessagesBuffer]
       # @return [RebalanceManager]
-      def initialize(subscription_group_id)
+      def initialize(subscription_group_id, buffer)
         @assigned_partitions = {}
         @revoked_partitions = {}
         @changed = false
         @active = false
         @subscription_group_id = subscription_group_id
+        @buffer = buffer
 
         # Connects itself to the instrumentation pipeline so rebalances can be tracked
         ::Karafka.monitor.subscribe(self)
@@ -64,17 +68,6 @@ module Karafka
         @active
       end
 
-      # We consider as lost only partitions that were taken away and not re-assigned back to us
-      def lost_partitions
-        lost_partitions = {}
-
-        revoked_partitions.each do |topic, partitions|
-          lost_partitions[topic] = partitions - assigned_partitions.fetch(topic, EMPTY_ARRAY)
-        end
-
-        lost_partitions
-      end
-
       # Callback that kicks in inside of rdkafka, when new partitions were assigned.
       #
       # @private
@@ -99,6 +92,24 @@ module Karafka
         @active = true
         @revoked_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
         @changed = true
+
+        remove_revoked_and_duplicated_messages
+      end
+
+      private
+
+      # We may have a case where in the middle of data polling, we've lost a partition.
+      # In a case like this we should remove all the pre-buffered messages from list partitions as
+      # we are no longer responsible in a given process for processing those messages and they
+      # should have been picked up by a different process.
+      def remove_revoked_and_duplicated_messages
+        @revoked_partitions.each do |topic, partitions|
+          partitions.each do |partition|
+            @buffer.delete(topic, partition)
+          end
+        end
+
+        @buffer.uniq!
       end
     end
   end

data/lib/karafka/contracts/config.rb

@@ -34,6 +34,7 @@ module Karafka
       required(:max_wait_time) { |val| val.is_a?(Integer) && val.positive? }
       required(:group_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
+      required(:strict_declarative_topics) { |val| [true, false].include?(val) }
 
       nested(:swarm) do
         required(:nodes) { |val| val.is_a?(Integer) && val.positive? }
@@ -79,6 +80,7 @@ module Karafka
         nested(:connection) do
           required(:manager) { |val| !val.nil? }
           required(:conductor) { |val| !val.nil? }
+          required(:reset_backoff) { |val| val.is_a?(Integer) && val >= 1_000 }
 
           nested(:proxy) do
             nested(:commit) do
@@ -115,6 +117,8 @@ module Karafka
           required(:partitioner_class) { |val| !val.nil? }
           required(:strategy_selector) { |val| !val.nil? }
           required(:expansions_selector) { |val| !val.nil? }
+          required(:executor_class) { |val| !val.nil? }
+          required(:worker_job_call_wrapper) { |val| val == false || val.respond_to?(:wrap) }
         end
 
         nested(:active_job) do

data/lib/karafka/contracts/consumer_group.rb

@@ -25,6 +25,23 @@ module Karafka
         [[%i[topics], :names_not_unique]]
       end
 
+      # Prevent same topics subscriptions in one CG with different consumer classes
+      # This should prevent users from accidentally creating multi-sg one CG setup with weird
+      # different consumer usage. If you need to consume same topic twice, use distinct CGs.
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        topics_consumers = Hash.new { |h, k| h[k] = Set.new }
+
+        data.fetch(:topics).map do |topic|
+          topics_consumers[topic[:name]] << topic[:consumer]
+        end
+
+        next if topics_consumers.values.map(&:size).all? { |count| count == 1 }
+
+        [[%i[topics], :many_consumers_same_topic]]
+      end
+
       virtual do |data, errors|
         next unless errors.empty?
         next unless ::Karafka::App.config.strict_topics_namespacing

data/lib/karafka/contracts/routing.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Contracts
+    # Ensures that routing wide rules are obeyed
+    class Routing < Base
+      configure do |config|
+        config.error_messages = YAML.safe_load(
+          File.read(
+            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+          )
+        ).fetch('en').fetch('validations').fetch('routing')
+      end
+
+      # Ensures, that when declarative topics strict requirement is on, all topics have
+      # declarative definition (including DLQ topics)
+      # @note It will ignore routing pattern topics because those topics are virtual
+      virtual do |data, errors|
+        next unless errors.empty?
+        # Do not validate declaratives unless required and explicitly enabled
+        next unless Karafka::App.config.strict_declarative_topics
+
+        # Collects declarative topics. Please note, that any topic that has a `#topic` reference,
+        # will be declarative by default unless explicitly disabled. This however does not apply
+        # to the DLQ definitions
+        dec_topics = Set.new
+        # All topics including the DLQ topics names that are marked as active
+        topics = Set.new
+
+        data.each do |consumer_group|
+          consumer_group[:topics].each do |topic|
+            pat = topic[:patterns]
+            # Ignore pattern topics because they won't exist and should not be declarative
+            # managed
+            topics << topic[:name] if !pat || !pat[:active]
+
+            dlq = topic[:dead_letter_queue]
+            topics << dlq[:topic] if dlq[:active]
+
+            dec = topic[:declaratives]
+
+            dec_topics << topic[:name] if dec[:active]
+          end
+        end
+
+        missing_dec = topics - dec_topics
+
+        next if missing_dec.empty?
+
+        missing_dec.map do |topic_name|
+          [
+            [:topics, topic_name],
+            :without_declarative_definition
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/karafka/contracts/topic.rb

@@ -51,6 +51,20 @@ module Karafka
         end
       end
 
+      # When users redefine kafka scope settings per topic, they often forget to define the
+      # basic stuff as they assume it is auto-inherited. It is not (unless inherit flag used),
+      # leaving them with things like bootstrap.servers undefined. This checks that bootstrap
+      # servers are defined so we can catch those issues before they cause more problems.
+      virtual do |data, errors|
+        next unless errors.empty?
+
+        kafka = data.fetch(:kafka)
+
+        next if kafka.key?(:'bootstrap.servers')
+
+        [[%w[kafka bootstrap.servers], :missing]]
+      end
+
       virtual do |data, errors|
         next unless errors.empty?
         next unless ::Karafka::App.config.strict_topics_namespacing

data/lib/karafka/embedded.rb

@@ -2,15 +2,33 @@
 
 module Karafka
   # Allows to start and stop Karafka as part of a different process
+  # Following limitations and restrictions apply:
+  #
+  # - `#start` cannot be called from a trap context - non blocking
+  # - `#quiet` - can be called from a trap context - non blocking
+  # - `#stop` - can be called from a trap context - blocking
   module Embedded
     class << self
+      # Lock for ensuring we do not control embedding in parallel
+      MUTEX = Mutex.new
+
+      private_constant :MUTEX
+
       # Starts Karafka without supervision and without ownership of signals in a background thread
       # so it won't interrupt other things running
       def start
+        MUTEX.synchronize do
+          # Prevent from double-starting
+          return if @started
+
+          @started = true
+        end
+
         Thread.new do
           Thread.current.name = 'karafka.embedded'
 
-          Karafka::Process.tags.add(:execution_mode, 'embedded')
+          Karafka::Process.tags.add(:execution_mode, 'mode:embedded')
+          Karafka::Server.execution_mode = :embedded
           Karafka::Server.start
         end
       end
@@ -19,9 +37,32 @@ module Karafka
       #
       # @note This method is blocking because we want to wait until Karafka is stopped with final
       #   process shutdown
+      #
+      # @note This method **is** safe to run from a trap context.
       def stop
-        # Stop needs to be blocking to wait for all the things to finalize
-        Karafka::Server.stop
+        # Prevent from double stopping
+        unless @stopping
+          Thread.new do
+            Thread.current.name = 'karafka.embedded.stopping'
+
+            stop = false
+
+            # We spawn a new thread because `#stop` may be called from a trap context
+            MUTEX.synchronize do
+              break if @stopping
+
+              @stopping = true
+              stop = true
+            end
+
+            next unless stop
+
+            Karafka::Server.stop
+          end
+        end
+
+        # Since we want to have this blocking, we wait for the background thread
+        sleep(0.1) until Karafka::App.terminated?
       end
 
       # Quiets Karafka upon any event
@@ -29,6 +70,8 @@ module Karafka
       # @note This method is not blocking and will not wait for Karafka to fully quiet.
       # It will trigger the quiet procedure but won't wait.
       #
+      # @note This method **can** be called from a trap context.
+      #
       # @note Please keep in mind you need to `#stop` to actually stop the server anyhow.
       def quiet
         Karafka::Server.quiet

data/lib/karafka/errors.rb

@@ -82,10 +82,11 @@ module Karafka
     AssignmentLostError = Class.new(BaseError)
 
     # Raised if optional dependencies like karafka-web are required in a version that is not
-    # supported by the current framework version.
+    # supported by the current framework version or when an optional dependency is missing.
     #
     # Because we do not want to require web out of the box and we do not want to lock web with
-    # karafka 1:1, we do such a sanity check
+    # karafka 1:1, we do such a sanity check. This also applies to cases where some external
+    # optional dependencies are needed but not available.
     DependencyConstraintsError = Class.new(BaseError)
 
     # Raised when we were not able to open pidfd for given pid

data/lib/karafka/helpers/async.rb

@@ -22,13 +22,22 @@ module Karafka
         def included(base)
           base.extend ::Forwardable
 
-          base.def_delegators :@thread, :join, :terminate, :alive?, :name
+          base.def_delegators :@thread, :join, :terminate, :name
+        end
+      end
+
+      # @return [Boolean] true if thread is present and is running, false otherwise
+      def alive?
+        MUTEX.synchronize do
+          return false unless @thread
+
+          @thread.alive?
         end
       end
 
       # Runs the `#call` method in a new thread
       # @param thread_name [String] name that we want to assign to the thread when we start it
-      def async_call(thread_name = '')
+      def async_call(thread_name)
         MUTEX.synchronize do
           return if @thread&.alive?
 

data/lib/karafka/helpers/config_importer.rb

@@ -25,6 +25,19 @@ module Karafka
           RUBY
         end
       end
+
+      # @param model [Object] object to which we want to add the config fetcher on a class level
+      def extended(model)
+        super
+
+        @attributes.each do |name, path|
+          model.class_eval <<~RUBY, __FILE__, __LINE__ + 1
+            def self.#{name}
+              @#{name} ||= ::Karafka::App.config.#{path.join('.')}
+            end
+          RUBY
+        end
+      end
     end
   end
 end

data/lib/karafka/instrumentation/assignments_tracker.rb

@@ -31,8 +31,13 @@ module Karafka
       def current
         assignments = {}
 
-        @assignments.each do |topic, partitions|
-          assignments[topic] = partitions.dup.freeze
+        # Since the `@assignments` state can change during a rebalance, if we would iterate over
+        # it exactly during state change, we would end up with the following error:
+        #   RuntimeError: can't add a new key into hash during iteration
+        @mutex.synchronize do
+          @assignments.each do |topic, partitions|
+            assignments[topic] = partitions.dup.freeze
+          end
         end
 
         assignments.freeze

data/lib/karafka/instrumentation/logger_listener.rb

@@ -41,7 +41,7 @@ module Karafka
         return unless log_polling?
 
         listener = event[:caller]
-        time = event[:time]
+        time = event[:time].round(2)
         messages_count = event[:messages_buffer].size
 
         message = "[#{listener.id}] Polled #{messages_count} messages in #{time}ms"
@@ -69,14 +69,14 @@ module Karafka
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_worker_processed(event)
         job = event[:job]
-        time = event[:time]
+        time = event[:time].round(2)
         job_type = job.class.to_s.split('::').last
         consumer = job.executor.topic.consumer
         topic = job.executor.topic.name
         partition = job.executor.partition
         info <<~MSG.tr("\n", ' ').strip!
           [#{job.id}] #{job_type} job for #{consumer}
-          on #{topic}/#{partition} finished in #{time}ms
+          on #{topic}/#{partition} finished in #{time} ms
         MSG
       end
 
@@ -127,6 +127,21 @@ module Karafka
         MSG
       end
 
+      # Prints info about seeking to a particular location
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_consumer_consuming_seek(event)
+        topic = event[:topic]
+        partition = event[:partition]
+        seek_offset = event[:message].offset
+        consumer = event[:caller]
+
+        info <<~MSG.tr("\n", ' ').strip!
+          [#{consumer.id}] Seeking from #{consumer.class}
+          on topic #{topic}/#{partition} to offset #{seek_offset}
+        MSG
+      end
+
       # Logs info about system signals that Karafka received and prints backtrace for threads in
       # case of ttin
       #
@@ -275,6 +290,12 @@ module Karafka
         details = (error.backtrace || []).join("\n")
 
         case type
+        when 'consumer.initialized.error'
+          error "Consumer initialized error: #{error}"
+          error details
+        when 'consumer.wrap.error'
+          error "Consumer wrap failed due to an error: #{error}"
+          error details
         when 'consumer.consume.error'
           error "Consumer consuming error: #{error}"
           error details
@@ -290,6 +311,9 @@ module Karafka
         when 'consumer.tick.error'
           error "Consumer on tick failed due to an error: #{error}"
           error details
+        when 'consumer.eofed.error'
+          error "Consumer on eofed failed due to an error: #{error}"
+          error details
         when 'consumer.after_consume.error'
           error "Consumer on after_consume failed due to an error: #{error}"
           error details
@@ -306,7 +330,24 @@ module Karafka
           fatal "Runner crashed due to an error: #{error}"
           fatal details
         when 'app.stopping.error'
-          error 'Forceful Karafka server stop'
+          # Counts number of workers and listeners that were still active when forcing the
+          # shutdown. Please note, that unless all listeners are closed, workers will not finalize
+          # their operations as well.
+          # We need to check if listeners and workers are assigned as during super early stages of
+          # boot they are not.
+          listeners = Server.listeners ? Server.listeners.count(&:active?) : 0
+          workers = Server.workers ? Server.workers.count(&:alive?) : 0
+
+          message = <<~MSG.tr("\n", ' ').strip!
+            Forceful Karafka server stop with:
+            #{workers} active workers and
+            #{listeners} active listeners
+          MSG
+
+          error message
+        when 'app.forceful_stopping.error'
+          error "Forceful shutdown error occurred: #{error}"
+          error details
         when 'librdkafka.error'
           error "librdkafka internal error occurred: #{error}"
           error details

data/lib/karafka/instrumentation/notifications.rb

@@ -48,11 +48,16 @@ module Karafka
         connection.listener.stopping
         connection.listener.stopped
 
+        consumer.initialize
+        consumer.initialized
+
         consumer.before_schedule_consume
         consumer.consume
         consumer.consumed
         consumer.consuming.pause
         consumer.consuming.retry
+        consumer.consuming.seek
+        consumer.consuming.transaction
 
         consumer.before_schedule_idle
         consumer.idle
@@ -65,10 +70,17 @@ module Karafka
         consumer.tick
         consumer.ticked
 
+        consumer.before_schedule_eofed
+        consumer.eof
+        consumer.eofed
+
         consumer.before_schedule_shutdown
         consumer.shutting_down
         consumer.shutdown
 
+        consumer.wrap
+        consumer.wrapped
+
         dead_letter_queue.dispatched
 
         filtering.throttled