karafka 2.4.7 → 2.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c94c3a4e646233f535e63bcf7a65c88d656f534bfccd21cad87be59b525adb1
-  data.tar.gz: 63b6ad0491880325d6ac0e5a7367f2a938e8a36245bd1cff48d3de73da6266c6
+  metadata.gz: c7e8e51a5c0c4ded0d074965cee2090e64ca77e43443f4f36ab03cc3a21ddfd6
+  data.tar.gz: c6e912c518d301f55974a9e4deb491ebe4c3e073e6748fe62ce1003eaea7bed7
 SHA512:
-  metadata.gz: 6c04bcd1f7ed17855140e519a82a52c0cae93a3c41126120ab797bbcf172315f7f7720f12e233cc85eed4af2dd437a3364f7f5613f730c9b2e32f8b79225570f
-  data.tar.gz: 3441c4ac2230c5903dacd210b54c80c05f72c2804490229334d10656672c4ef9d9b3160c12cdd4d22916d55f1687a730980b0b3fc21099a003f36238f58025eb
+  metadata.gz: 8769a192c7ebb852250afd96611e115807172bf320d4e9d513bffbcc66f5570ca637aee1b8f3b68e46350b37054935ea905893139bc264a02949f975b39f2041
+  data.tar.gz: 13dc8e118850aace7127bae5fe667b6f73118d220b61b1d2b590cc18e71243b3c1467d43df9ae134cd7b26c0274a100e34d8a07db0c6091bc02fd9c95ffb24f0

data/CHANGELOG.md

@@ -1,7 +1,15 @@
 # Karafka Framework Changelog
 
+## 2.4.8 (2024-08-09)
+- **[Feature]** Introduce ability to react to `#eof` either from `#consume` or from `#eofed` when EOF without new messages.
+- [Enhancement] Provide `Consumer#eofed?` to indicate reaching EOF.
+- [Enhancement] Always immediately report on `inconsistent_group_protocol` error.
+- [Enhancement] Reduce virtual partitioning to 1 partition when any partitioner execution in a partitioned batch crashes.
+- [Enhancement] Provide `KARAFKA_REQUIRE_RAILS` to disable default Rails `require` to run Karafka without Rails despite having Rails in the Gemfile.
+- [Enhancement] Increase final listener recovery from 1 to 60 seconds to prevent constant rebalancing. This is the last resort recovery and should never happen unless critical errors occur.
+
 ## 2.4.7 (2024-08-01)
-- [Enhancement] Introduce `Karafka::Server.mode` to check in what mode Karafka process operates (`standalone`, `swarm`, `supervisor`, `embedded`).
+- [Enhancement] Introduce `Karafka::Server.execution_mode` to check in what mode Karafka process operates (`standalone`, `swarm`, `supervisor`, `embedded`).
 - [Enhancement] Ensure `max.poll.interval.ms` is always present and populate it with librdkafka default.
 - [Enhancement] Introduce a shutdown time limit for unsubscription wait.
 - [Enhancement] Tag with `mode:swarm` each of the running swarm consumers.

data/Gemfile

@@ -12,7 +12,7 @@ gemspec
 group :integrations do
   gem 'activejob', require: false
   gem 'karafka-testing', '>= 2.4.0', require: false
-  gem 'karafka-web', '>= 0.9.0', require: false
+  gem 'karafka-web', '>= 0.10.0.beta1', require: false
   gem 'rspec', require: false
 end
 

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.7)
+    karafka (2.4.8)
       base64 (~> 0.2)
       karafka-core (>= 2.4.3, < 2.5.0)
+      karafka-rdkafka (>= 0.17.2)
       waterdrop (>= 2.7.3, < 3.0.0)
       zeitwerk (~> 2.3)
 
@@ -29,7 +30,7 @@ GEM
     concurrent-ruby (1.3.3)
     connection_pool (2.4.1)
     diff-lcs (1.5.1)
-    docile (1.4.0)
+    docile (1.4.1)
     drb (2.2.1)
     erubi (1.13.0)
     factory_bot (6.4.6)
@@ -41,26 +42,26 @@ GEM
       concurrent-ruby (~> 1.0)
     karafka-core (2.4.4)
       karafka-rdkafka (>= 0.15.0, < 0.18.0)
-    karafka-rdkafka (0.17.1)
+    karafka-rdkafka (0.17.3)
       ffi (~> 1.15)
       mini_portile2 (~> 2.6)
       rake (> 12)
     karafka-testing (2.4.6)
       karafka (>= 2.4.0, < 2.5.0)
       waterdrop (>= 2.7.0)
-    karafka-web (0.9.1)
+    karafka-web (0.10.0.rc1)
       erubi (~> 1.4)
-      karafka (>= 2.4.0, < 2.5.0)
+      karafka (>= 2.4.7, < 2.5.0)
       karafka-core (>= 2.4.0, < 2.5.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
     mini_portile2 (2.8.7)
-    minitest (5.24.0)
+    minitest (5.24.1)
     mutex_m (0.2.0)
     ostruct (0.6.0)
-    rack (3.1.5)
+    rack (3.1.7)
     rake (13.2.1)
-    roda (3.81.0)
+    roda (3.82.0)
       rack
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
@@ -100,7 +101,7 @@ DEPENDENCIES
   factory_bot
   karafka!
   karafka-testing (>= 2.4.0)
-  karafka-web (>= 0.9.0)
+  karafka-web (>= 0.10.0.beta1)
   ostruct
   rspec
   simplecov

data/config/locales/errors.yml

@@ -49,6 +49,7 @@ en:
 
       internal.connection.manager_format: needs to be present
       internal.connection.conductor_format: needs to be present
+      internal.connection.reset_backoff_format: needs to be an integer bigger or equal to 1000
       internal.connection.proxy.query_watermark_offsets.timeout_format: needs to be an integer bigger than 0
       internal.connection.proxy.query_watermark_offsets.max_attempts_format: needs to be an integer bigger than 0
       internal.connection.proxy.query_watermark_offsets.wait_time_format: needs to be an integer bigger than 0
@@ -115,6 +116,8 @@ en:
 
       active_format: needs to be either true or false
 
+      eofed.active_format: needs to be either true or false
+
       declaratives.partitions_format: needs to be more or equal to 1
       declaratives.active_format: needs to be true
       declaratives.replication_factor_format: needs to be more or equal to 1

data/karafka.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'base64', '~> 0.2'
   spec.add_dependency 'karafka-core', '>= 2.4.3', '< 2.5.0'
+  spec.add_dependency 'karafka-rdkafka', '>= 0.17.2'
   spec.add_dependency 'waterdrop', '>= 2.7.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 

data/lib/karafka/base_consumer.rb

@@ -9,7 +9,7 @@ module Karafka
 
     extend Forwardable
 
-    def_delegators :@coordinator, :topic, :partition
+    def_delegators :@coordinator, :topic, :partition, :eofed?
 
     def_delegators :producer, :produce_async, :produce_sync, :produce_many_async,
                    :produce_many_sync
@@ -100,6 +100,24 @@ module Karafka
       retry_after_pause
     end
 
+    # Can be used to run code prior to scheduling of eofed execution
+    def on_before_schedule_eofed
+      handle_before_schedule_eofed
+    end
+
+    # Trigger method for running on eof without messages
+    def on_eofed
+      handle_eofed
+    rescue StandardError => e
+      Karafka.monitor.instrument(
+        'error.occurred',
+        error: e,
+        caller: self,
+        seek_offset: coordinator.seek_offset,
+        type: 'consumer.eofed.error'
+      )
+    end
+
     # Can be used to run code prior to scheduling of idle execution
     #
     # @private
@@ -166,6 +184,10 @@ module Karafka
       raise NotImplementedError, 'Implement this in a subclass'
     end
 
+    # Method that will be executed when a given topic partition reaches eof without any new
+    # incoming messages alongside
+    def eofed; end
+
     # Method that will be executed when a given topic partition is revoked. You can use it for
     # some teardown procedures (closing file handler, etc).
     def revoked; end

data/lib/karafka/connection/client.rb

@@ -31,7 +31,21 @@ module Karafka
       # before we move to a regular unsubscribe.
       COOP_UNSUBSCRIBE_FACTOR = 0.5
 
-      private_constant :MAX_POLL_RETRIES, :COOP_UNSUBSCRIBE_FACTOR
+      # Errors upon which we early report that something is off without retrying prior to the
+      # report
+      EARLY_REPORT_ERRORS = [
+        :inconsistent_group_protocol, # 23
+        :max_poll_exceeded, # -147
+        :network_exception, # 13
+        :transport, # -195
+        :topic_authorization_failed, # 29
+        :group_authorization_failed, # 30
+        :cluster_authorization_failed, # 31
+        # This can happen for many reasons, including issues with static membership being fenced
+        :fatal # -150
+      ].freeze
+
+      private_constant :MAX_POLL_RETRIES, :COOP_UNSUBSCRIBE_FACTOR, :EARLY_REPORT_ERRORS
 
       # Creates a new consumer instance.
       #
@@ -98,8 +112,17 @@ module Karafka
           # Fetch message within our time boundaries
           response = poll(time_poll.remaining)
 
-          # Put a message to the buffer if there is one
-          @buffer << response if response && response != :tick_time
+          case response
+          when :tick_time
+            nil
+          # We get a hash only in case of eof error
+          when Hash
+            @buffer.eof(response[:topic], response[:partition])
+          when nil
+            nil
+          else
+            @buffer << response
+          end
 
           # Upon polling rebalance manager might have been updated.
           # If partition revocation happens, we need to remove messages from revoked partitions
@@ -122,10 +145,11 @@ module Karafka
           time_poll.checkpoint
 
           # Finally once we've (potentially) removed revoked, etc, if no messages were returned
-          # and it was not an early poll exist, we can break.
+          # and it was not an early poll exist, we can break. We also break if we got the eof
+          # signaling to propagate it asap
           # Worth keeping in mind, that the rebalance manager might have been updated despite no
           # messages being returned during a poll
-          break unless response
+          break if response.nil? || response.is_a?(Hash)
         end
 
         @buffer
@@ -576,20 +600,7 @@ module Karafka
         # We want to report early on max poll interval exceeding because it may mean that the
         # underlying processing is taking too much time and it is not LRJ
         case e.code
-        when :max_poll_exceeded # -147
-          early_report = true
-        when :network_exception # 13
-          early_report = true
-        when :transport # -195
-          early_report = true
-        when :topic_authorization_failed # 29
-          early_report = true
-        when :group_authorization_failed # 30
-          early_report = true
-        when :cluster_authorization_failed # 31
-          early_report = true
-        # This can happen for many reasons, including issues with static membership being fenced
-        when :fatal # -150
+        when *EARLY_REPORT_ERRORS
           early_report = true
         # @see
         # https://github.com/confluentinc/confluent-kafka-dotnet/issues/1366#issuecomment-821842990
@@ -603,11 +614,12 @@ module Karafka
           # No sense in retrying when no topic/partition and we're no longer running
           retryable = false unless Karafka::App.running?
         # If we detect the end of partition which can happen if `enable.partition.eof` is set to
-        # true, we can just return nil fast. This will fast yield whatever set of messages we
+        # true, we can just return fast. This will fast yield whatever set of messages we
         # already have instead of waiting. This can be used for better latency control when we do
         # not expect a lof of lag and want to quickly move to processing.
+        # We can also pass the eof notion to the consumers for improved decision making.
         when :partition_eof
-          return nil
+          return e.details
         end
 
         if early_report || !retryable

data/lib/karafka/connection/listener.rb

@@ -253,7 +253,9 @@ module Karafka
 
         reset
 
-        sleep(1) && retry
+        # Ruby sleep is in seconds
+        sleep_time = ::Karafka::App.config.internal.connection.reset_backoff / 10_000.0
+        sleep(sleep_time) && retry
       end
 
       # Resumes processing of partitions that were paused due to an error.
@@ -330,28 +332,57 @@ module Karafka
       # given scheduler. It also handles the idle jobs when filtering API removed all messages
       # and we need to run house-keeping
       def build_and_schedule_flow_jobs
-        return if @messages_buffer.empty?
-
         consume_jobs = []
         idle_jobs = []
+        eofed_jobs = []
+
+        @messages_buffer.each do |topic, partition, messages, eof|
+          # In case we did not receive any new messages without eof we skip.
+          # We may yield empty array here in case we have reached eof without new messages but in
+          # such cases, we can run an eof job
+          next if messages.empty? && !eof
 
-        @messages_buffer.each do |topic, partition, messages|
           coordinator = @coordinators.find_or_create(topic, partition)
-          # Start work coordination for this topic partition
+          coordinator.eofed = eof
+
+          # If we did not receive any messages and we did receive eof signal, we run the eofed
+          # jobs so user can take actions on reaching eof
+          if messages.empty? && eof
+            # If user wants to run the eofed jobs on eof we do it. Otherwise we just allow it to
+            # pass through. This allows to configure if user actually wants to have `#eofed`
+            # logic or if he wants to only use fast eof work yield
+            if coordinator.topic.eofed?
+              @executors.find_all_or_create(topic, partition, coordinator).each do |executor|
+                coordinator.increment(:eofed)
+                eofed_jobs << @jobs_builder.eofed(executor)
+              end
+            end
+
+            next
+          end
+
           coordinator.start(messages)
 
+          # If it is not an eof and there are no new messages, we just run house-keeping
+          #
           # We do not increment coordinator for idle job because it's not a user related one
           # and it will not go through a standard lifecycle. Same applies to revoked and shutdown
           if messages.empty?
+            # Start work coordination for this topic partition
             coordinator.increment(:idle)
             executor = @executors.find_or_create(topic, partition, 0, coordinator)
             idle_jobs << @jobs_builder.idle(executor)
-          else
-            @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
-              coordinator.increment(:consume)
-              executor = @executors.find_or_create(topic, partition, group_id, coordinator)
-              consume_jobs << @jobs_builder.consume(executor, partition_messages)
-            end
+
+            next
+          end
+
+          # If there are messages, it is irrelevant if eof or not as consumption needs to happen
+          #
+          # Start work coordination for this topic partition
+          @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
+            coordinator.increment(:consume)
+            executor = @executors.find_or_create(topic, partition, group_id, coordinator)
+            consume_jobs << @jobs_builder.consume(executor, partition_messages)
           end
         end
 
@@ -367,6 +398,11 @@ module Karafka
           consume_jobs.each(&:before_schedule)
           @scheduler.on_schedule_consumption(consume_jobs)
         end
+
+        unless eofed_jobs.empty?
+          eofed_jobs.each(&:before_schedule)
+          @scheduler.on_schedule_eofed(eofed_jobs)
+        end
       end
 
       # Builds and schedules periodic jobs for topics partitions for which no messages were

data/lib/karafka/connection/messages_buffer.rb

@@ -23,9 +23,13 @@ module Karafka
       def initialize(subscription_group)
         @subscription_group = subscription_group
         @size = 0
+
         @groups = Hash.new do |topic_groups, topic|
           topic_groups[topic] = Hash.new do |partition_groups, partition|
-            partition_groups[partition] = []
+            partition_groups[partition] = {
+              eof: false,
+              messages: []
+            }
           end
         end
       end
@@ -33,24 +37,29 @@ module Karafka
       # Remaps raw messages from the raw messages buffer to Karafka messages
       # @param raw_messages_buffer [RawMessagesBuffer] buffer with raw messages
       def remap(raw_messages_buffer)
-        clear unless @size.zero?
+        clear
 
         # Since it happens "right after" we've received the messages, it is close enough it time
         # to be used as the moment we received messages.
         received_at = Time.now
 
-        raw_messages_buffer.each do |topic, partition, messages|
+        raw_messages_buffer.each do |topic, partition, messages, eof|
           @size += messages.count
 
           ktopic = @subscription_group.topics.find(topic)
 
-          @groups[topic][partition] = messages.map do |message|
+          built_messages = messages.map do |message|
             Messages::Builders::Message.call(
               message,
               ktopic,
               received_at
             )
           end
+
+          @groups[topic][partition] = {
+            eof: eof,
+            messages: built_messages
+          }
         end
       end
 
@@ -59,10 +68,11 @@ module Karafka
       # @yieldparam [String] topic name
       # @yieldparam [Integer] partition number
       # @yieldparam [Array<Karafka::Messages::Message>] messages from a given topic partition
+      # @yieldparam [Boolean] true if eof, false otherwise
       def each
         @groups.each do |topic, partitions|
-          partitions.each do |partition, messages|
-            yield(topic, partition, messages)
+          partitions.each do |partition, details|
+            yield(topic, partition, details[:messages], details[:eof])
           end
         end
       end

data/lib/karafka/connection/raw_messages_buffer.rb

@@ -2,7 +2,7 @@
 
 module Karafka
   module Connection
-    # Buffer for raw librdkafka messages.
+    # Buffer for raw librdkafka messages and eof status.
     #
     # When message is added to this buffer, it gets assigned to an array with other messages from
     # the same topic and partition.
@@ -17,9 +17,13 @@ module Karafka
       # @return [Karafka::Connection::MessagesBuffer] buffer instance
       def initialize
         @size = 0
+
         @groups = Hash.new do |topic_groups, topic|
           topic_groups[topic] = Hash.new do |partition_groups, partition|
-            partition_groups[partition] = []
+            partition_groups[partition] = {
+              eof: false,
+              messages: []
+            }
           end
         end
       end
@@ -30,7 +34,16 @@ module Karafka
       # @return [Array<Rdkafka::Consumer::Message>] given partition topic sub-buffer array
       def <<(message)
         @size += 1
-        @groups[message.topic][message.partition] << message
+        partition_state = @groups[message.topic][message.partition]
+        partition_state[:messages] << message
+        partition_state[:eof] = false
+      end
+
+      # Marks given topic partition as one that reached eof
+      # @param topic [String] topic that reached eof
+      # @param partition [Integer] partition that reached eof
+      def eof(topic, partition)
+        @groups[topic][partition][:eof] = true
       end
 
       # Allows to iterate over all the topics and partitions messages
@@ -38,10 +51,11 @@ module Karafka
       # @yieldparam [String] topic name
       # @yieldparam [Integer] partition number
       # @yieldparam [Array<Rdkafka::Consumer::Message>] topic partition aggregated results
+      # @yieldparam [Boolean] has polling of this partition reach eof
       def each
         @groups.each do |topic, partitions|
-          partitions.each do |partition, messages|
-            yield(topic, partition, messages)
+          partitions.each do |partition, details|
+            yield(topic, partition, details[:messages], details[:eof])
           end
         end
       end
@@ -69,8 +83,8 @@ module Karafka
       # again and we do want to ensure as few duplications as possible
       def uniq!
         @groups.each_value do |partitions|
-          partitions.each_value do |messages|
-            messages.uniq!(&:offset)
+          partitions.each_value do |details|
+            details[:messages].uniq!(&:offset)
           end
         end
 
@@ -83,6 +97,11 @@ module Karafka
       #   we save ourselves some objects allocations. We cannot clear the underlying arrays as they
       #   may be used in other threads for data processing, thus if we would clear it, we could
       #   potentially clear a raw messages array for a job that is in the jobs queue.
+      #
+      # @note We do not clear the eof assignments because they can span across batch pollings.
+      #   Since eof is not raised non-stop and is silenced after an eof poll, if we would clean it
+      #   here we would loose the notion of it. The reset state for it should happen when we do
+      #   discover new messages for given topic partition.
       def clear
         @size = 0
         @groups.each_value(&:clear)
@@ -92,8 +111,12 @@ module Karafka
 
       # Updates the messages count if we performed any operations that could change the state
       def recount!
-        @size = @groups.each_value.sum do |partitions|
-          partitions.each_value.map(&:count).sum
+        @size = 0
+
+        @groups.each_value do |partitions|
+          partitions.each_value do |details|
+            @size += details[:messages].size
+          end
         end
       end
     end

data/lib/karafka/contracts/config.rb

@@ -79,6 +79,7 @@ module Karafka
         nested(:connection) do
           required(:manager) { |val| !val.nil? }
           required(:conductor) { |val| !val.nil? }
+          required(:reset_backoff) { |val| val.is_a?(Integer) && val >= 1_000 }
 
           nested(:proxy) do
             nested(:commit) do

data/lib/karafka/instrumentation/logger_listener.rb

@@ -290,6 +290,9 @@ module Karafka
         when 'consumer.tick.error'
           error "Consumer on tick failed due to an error: #{error}"
           error details
+        when 'consumer.eofed.error'
+          error "Consumer on eofed failed due to an error: #{error}"
+          error details
         when 'consumer.after_consume.error'
           error "Consumer on after_consume failed due to an error: #{error}"
           error details

data/lib/karafka/instrumentation/notifications.rb

@@ -65,6 +65,10 @@ module Karafka
         consumer.tick
         consumer.ticked
 
+        consumer.before_schedule_eofed
+        consumer.eof
+        consumer.eofed
+
         consumer.before_schedule_shutdown
         consumer.shutting_down
         consumer.shutdown

data/lib/karafka/pro/processing/jobs/eofed_non_blocking.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Processing
+      module Jobs
+        # Non-Blocking version of the Eofed job
+        # We use this version for LRJ topics for cases where saturated resources would not allow
+        # to run this job for extended period of time. Under such scenarios, if we would not use
+        # a non-blocking one, we would reach max.poll.interval.ms.
+        class EofedNonBlocking < ::Karafka::Processing::Jobs::Eofed
+          # @param args [Array] any arguments accepted by `::Karafka::Processing::Jobs::Eofed`
+          def initialize(*args)
+            super
+            @non_blocking = true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/processing/jobs_builder.rb

@@ -33,6 +33,18 @@ module Karafka
           end
         end
 
+        # @param executor [Karafka::Pro::Processing::Executor]
+        # @return [Karafka::Processing::Jobs::Eofed] eofed job for non LRJ
+        # @return [Karafka::Processing::Jobs::EofedBlocking] eofed job that is
+        #   non-blocking, so when revocation job is scheduled for LRJ it also will not block
+        def eofed(executor)
+          if executor.topic.long_running_job?
+            Jobs::EofedNonBlocking.new(executor)
+          else
+            super
+          end
+        end
+
         # @param executor [Karafka::Pro::Processing::Executor]
         # @return [Karafka::Processing::Jobs::Revoked] revocation job for non LRJ
         # @return [Karafka::Processing::Jobs::RevokedNonBlocking] revocation job that is

data/lib/karafka/pro/processing/partitioner.rb

@@ -31,6 +31,7 @@ module Karafka
           # - a virtual partitioner
           # - more than one thread to process the data
           # - collective is not collapsed via coordinator
+          # - none of the partitioner executions raised an error
           #
           # With one thread it is not worth partitioning the work as the work itself will be
           # assigned to one thread (pointless work)
@@ -41,18 +42,36 @@ module Karafka
           # This is great because it allows us to run things without the parallelization that adds
           # a bit of uncertainty and allows us to use DLQ and safely skip messages if needed.
           if vps.active? && vps.max_partitions > 1 && !coordinator.collapsed?
-            groupings = messages.group_by do |msg|
-              # We need to reduce it to the max concurrency, so the group_id is not a direct effect
-              # of the end user action. Otherwise the persistence layer for consumers would cache
-              # it forever and it would cause memory leaks
-              #
-              # This also needs to be consistent because the aggregation here needs to warrant,
-              # that the same partitioned message will always be assigned to the same virtual
-              # partition. Otherwise in case of a window aggregation with VP spanning across
-              # several polls, the data could not be complete.
-              vps.reducer.call(
-                vps.partitioner.call(msg)
+            # If we cannot virtualize even one message from a given batch due to user errors, we
+            # reduce the whole set into one partition and emit error. This should still allow for
+            # user flow but should mitigate damages by not virtualizing
+            begin
+              groupings = messages.group_by do |msg|
+                # We need to reduce it to the max concurrency, so the group_id is not a direct
+                # effect of the end user action. Otherwise the persistence layer for consumers
+                # would cache it forever and it would cause memory leaks
+                #
+                # This also needs to be consistent because the aggregation here needs to warrant,
+                # that the same partitioned message will always be assigned to the same virtual
+                # partition. Otherwise in case of a window aggregation with VP spanning across
+                # several polls, the data could not be complete.
+                vps.reducer.call(
+                  vps.partitioner.call(msg)
+                )
+              end
+            rescue StandardError => e
+              # This should not happen. If you are seeing this it means your partitioner code
+              # failed and raised an error. We highly recommend mitigating partitioner level errors
+              # on the user side because this type of collapse should be considered a last resort
+              Karafka.monitor.instrument(
+                'error.occurred',
+                caller: self,
+                error: e,
+                messages: messages,
+                type: 'virtual_partitions.partitioner.error'
               )
+
+              groupings = { 0 => messages }
             end
 
             groupings.each do |key, messages_group|

data/lib/karafka/pro/processing/schedulers/default.rb

@@ -68,6 +68,7 @@ module Karafka
           alias on_schedule_shutdown schedule_fifo
           alias on_schedule_idle schedule_fifo
           alias on_schedule_periodic schedule_fifo
+          alias on_schedule_eofed schedule_fifo
 
           # This scheduler does not have anything to manage as it is a pass through and has no
           # state

data/lib/karafka/processing/coordinator.rb

@@ -15,6 +15,10 @@ module Karafka
 
       attr_reader :pause_tracker, :seek_offset, :topic, :partition
 
+      # This can be set directly on the listener because it can be triggered on first run without
+      # any messages
+      attr_accessor :eofed
+
       def_delegators :@pause_tracker, :attempt, :paused?
 
       # @param topic [Karafka::Routing::Topic]
@@ -32,6 +36,7 @@ module Karafka
         @mutex = Mutex.new
         @marked = false
         @failure = false
+        @eofed = false
         @changed_at = monotonic_now
       end
 
@@ -146,6 +151,11 @@ module Karafka
         @revoked
       end
 
+      # @return [Boolean] did we reach end of partition when polling data
+      def eofed?
+        @eofed
+      end
+
       # @return [Boolean] was the new seek offset assigned at least once. This is needed because
       #   by default we assign seek offset of a first message ever, however this is insufficient
       #   for DLQ in a scenario where the first message would be broken. We would never move

data/lib/karafka/processing/executor.rb

@@ -104,6 +104,18 @@ module Karafka
         consumer.on_idle
       end
 
+      # Runs the code needed before eofed work is scheduled
+      def before_schedule_eofed
+        consumer.on_before_schedule_eofed
+      end
+
+      # Runs consumed eofed operation.
+      # This may run even when there were no messages received prior. This will however not
+      # run when eof is received together with messages as in such case `#consume` will run
+      def eofed
+        consumer.on_eofed
+      end
+
       # Runs code needed before revoked job is scheduled
       def before_schedule_revoked
         consumer.on_before_schedule_revoked if @consumer

data/lib/karafka/processing/jobs/eofed.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Processing
+    module Jobs
+      # Job that runs the eofed operation when we receive eof without messages alongside.
+      class Eofed < Base
+        # @param executor [Karafka::Processing::Executor] executor that is suppose to run the job
+        # @return [Eofed]
+        def initialize(executor)
+          @executor = executor
+          super()
+        end
+
+        # Runs code prior to scheduling this eofed job
+        def before_schedule
+          executor.before_schedule_eofed
+        end
+
+        # Runs the eofed job via an executor.
+        def call
+          executor.eofed
+        end
+      end
+    end
+  end
+end

data/lib/karafka/processing/jobs_builder.rb

@@ -12,6 +12,12 @@ module Karafka
         Jobs::Consume.new(executor, messages)
       end
 
+      # @param executor [Karafka::Processing::Executor]
+      # @return [Karafka::Processing::Jobs::Eofed] eofed job
+      def eofed(executor)
+        Jobs::Eofed.new(executor)
+      end
+
       # @param executor [Karafka::Processing::Executor]
       # @return [Karafka::Processing::Jobs::Revoked] revocation job
       def revoked(executor)

data/lib/karafka/processing/schedulers/default.rb

@@ -24,6 +24,7 @@ module Karafka
         alias on_schedule_revocation on_schedule_consumption
         alias on_schedule_shutdown on_schedule_consumption
         alias on_schedule_idle on_schedule_consumption
+        alias on_schedule_eofed on_schedule_consumption
 
         # This scheduler does not have anything to manage as it is a pass through and has no state
         def on_manage

data/lib/karafka/processing/strategies/default.rb

@@ -17,6 +17,7 @@ module Karafka
         %i[
           consume
           idle
+          eofed
           revoked
           shutdown
         ].each do |action|
@@ -151,6 +152,16 @@ module Karafka
           coordinator.decrement(:idle)
         end
 
+        # Runs the consumer `#eofed` method with reporting
+        def handle_eofed
+          Karafka.monitor.instrument('consumer.eof', caller: self)
+          Karafka.monitor.instrument('consumer.eofed', caller: self) do
+            eofed
+          end
+        ensure
+          coordinator.decrement(:eofed)
+        end
+
         # We need to always un-pause the processing in case we have lost a given partition.
         # Otherwise the underlying librdkafka would not know we may want to continue processing and
         # the pause could in theory last forever

data/lib/karafka/routing/features/eofed/config.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Eofed < Base
+        # Config of this feature
+        Config = Struct.new(
+          :active,
+          keyword_init: true
+        ) { alias_method :active?, :active }
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/eofed/contracts/topic.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Eofed < Base
+        # Eofed related contracts namespace
+        module Contracts
+          # Contract for eofed topic setup
+          class Topic < Karafka::Contracts::Base
+            configure do |config|
+              config.error_messages = YAML.safe_load(
+                File.read(
+                  File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
+                )
+              ).fetch('en').fetch('validations').fetch('topic')
+            end
+
+            nested :eofed do
+              required(:active) { |val| [true, false].include?(val) }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/eofed/topic.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      class Eofed < Base
+        # Routing topic eofed API
+        module Topic
+          # @param active [Boolean] should the `#eofed` job run on eof
+          def eofed(active = false)
+            @eofed ||= Config.new(
+              active: active
+            )
+          end
+
+          # @return [Boolean] Are `#eofed` jobs active
+          def eofed?
+            eofed.active?
+          end
+
+          # @return [Hash] topic setup hash
+          def to_h
+            super.merge(
+              eofed: eofed.to_h
+            ).freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/routing/features/eofed.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Routing
+    module Features
+      # Namespace for feature allowing to enable the `#eofed` jobs.
+      # We do not enable it always because users may only be interested in fast eofed yielding
+      # without running the `#eofed` operation at all. This safes on empty cycles of running
+      # pointless empty jobs.
+      class Eofed < Base
+      end
+    end
+  end
+end

data/lib/karafka/setup/config.rb

@@ -203,6 +203,9 @@ module Karafka
           setting :manager, default: Connection::Manager.new
           # Controls frequency of connections management checks
           setting :conductor, default: Connection::Conductor.new
+          # How long should we wait before a critical listener recovery
+          # Too short may cause endless rebalance loops
+          setting :reset_backoff, default: 60_000
 
           # Settings that are altered by our client proxy layer
           setting :proxy do

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.7'
+  VERSION = '2.4.8'
 end

data/lib/karafka.rb

@@ -71,11 +71,23 @@ module Karafka
 
     # @return [Boolean] Do we run within/with Rails. We use this to initialize Railtie and proxy
     #   the console invocation to Rails
+    #
+    # @note We allow users to disable Rails require because having Rails in the Gemfile does not
+    #   always mean user wants to have it required. User may want to run Karafka without Rails
+    #   even when having both in the same Gemfile.
     def rails?
       return @rails if instance_variable_defined?('@rails')
 
-      # Do not load Rails again if already loaded
-      Object.const_defined?('Rails::Railtie') || require('rails')
+      @rails = Object.const_defined?('Rails::Railtie')
+
+      # If Rails exists we set it immediately based on its presence and return
+      return @rails if @rails
+
+      # If rails is not present and user wants us not to force-load it, we return
+      return @rails if ENV['KARAFKA_REQUIRE_RAILS'] == 'false'
+
+      # If we should try to require it, we try and if no error, it means its there
+      require('rails')
 
       @rails = true
     rescue LoadError

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.4.7
+  version: 2.4.8
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-08-01 00:00:00.000000000 Z
+date: 2024-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base64
@@ -71,6 +71,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 2.5.0
+- !ruby/object:Gem::Dependency
+  name: karafka-rdkafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.17.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.17.2
 - !ruby/object:Gem::Dependency
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
@@ -290,6 +304,7 @@ files:
 - lib/karafka/pro/processing/filters/throttler.rb
 - lib/karafka/pro/processing/filters/virtual_limiter.rb
 - lib/karafka/pro/processing/jobs/consume_non_blocking.rb
+- lib/karafka/pro/processing/jobs/eofed_non_blocking.rb
 - lib/karafka/pro/processing/jobs/periodic.rb
 - lib/karafka/pro/processing/jobs/periodic_non_blocking.rb
 - lib/karafka/pro/processing/jobs/revoked_non_blocking.rb
@@ -444,6 +459,7 @@ files:
 - lib/karafka/processing/inline_insights/tracker.rb
 - lib/karafka/processing/jobs/base.rb
 - lib/karafka/processing/jobs/consume.rb
+- lib/karafka/processing/jobs/eofed.rb
 - lib/karafka/processing/jobs/idle.rb
 - lib/karafka/processing/jobs/revoked.rb
 - lib/karafka/processing/jobs/shutdown.rb
@@ -487,6 +503,10 @@ files:
 - lib/karafka/routing/features/deserializers/config.rb
 - lib/karafka/routing/features/deserializers/contracts/topic.rb
 - lib/karafka/routing/features/deserializers/topic.rb
+- lib/karafka/routing/features/eofed.rb
+- lib/karafka/routing/features/eofed/config.rb
+- lib/karafka/routing/features/eofed/contracts/topic.rb
+- lib/karafka/routing/features/eofed/topic.rb
 - lib/karafka/routing/features/inline_insights.rb
 - lib/karafka/routing/features/inline_insights/config.rb
 - lib/karafka/routing/features/inline_insights/contracts/topic.rb