karafka 2.5.6 → 2.5.9

data/lib/karafka/pro/scheduled_messages/day.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/scheduled_messages/deserializers/headers.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/scheduled_messages/deserializers/payload.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/scheduled_messages/dispatcher.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/scheduled_messages/errors.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/scheduled_messages/max_epoch.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/scheduled_messages/proxy.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/scheduled_messages/schema_validator.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/scheduled_messages/serializer.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/scheduled_messages/setup/config.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/scheduled_messages/state.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/scheduled_messages/tracker.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/scheduled_messages.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 

data/lib/karafka/pro/swarm/liveness_listener.rb

@@ -6,9 +6,14 @@
 # This software is NOT open source. It is source-available commercial software
 # requiring a paid license for use. It is NOT covered by LGPL.
 #
+# The author retains all right, title, and interest in this software,
+# including all copyrights, patents, and other intellectual property rights.
+# No patent rights are granted under this license.
+#
 # PROHIBITED:
 # - Use without a valid commercial license
 # - Redistribution, modification, or derivative works without authorization
+# - Reverse engineering, decompilation, or disassembly of this software
 # - Use as training data for AI/ML models or inclusion in datasets
 # - Scraping, crawling, or automated collection for any purpose
 #
@@ -17,6 +22,9 @@
 # - Runtime retrieval by AI assistants, coding agents, and RAG systems
 #   for the purpose of providing contextual help to Karafka users
 #
+# Receipt, viewing, or possession of this software does not convey or
+# imply any license or right beyond those expressly stated above.
+#
 # License: https://karafka.io/docs/Pro-License-Comm/
 # Contact: contact@karafka.io
 
@@ -65,11 +73,20 @@ module Karafka
           super()
         end
 
-        # Tick on each fetch
+        # Tick on each fetch and report liveness so it works even when statistics are disabled
         #
         # @param _event [Karafka::Core::Monitoring::Event]
         def on_connection_listener_fetch_loop(_event)
           mark_polling_tick
+          report_status
+        end
+
+        # Report liveness during events poll so it works during long processing without statistics.
+        # This event fires periodically during wait even when the listener is blocked on consumer
+        # jobs, preventing the supervisor from killing the node.
+        # @param _event [Karafka::Core::Monitoring::Event]
+        def on_client_events_poll(_event)
+          report_status
         end
 
         {
@@ -99,19 +116,6 @@ module Karafka
           clear_polling_tick
         end
 
-        # Reports the current status once in a while
-        #
-        # @param _event [Karafka::Core::Monitoring::Event]
-        def on_statistics_emitted(_event)
-          periodically do
-            return unless node
-
-            current_status = status
-
-            current_status.positive? ? node.unhealthy(current_status) : node.healthy
-          end
-        end
-
         # Deregister the polling tracker for given listener
         # @param _event [Karafka::Core::Monitoring::Event]
         def on_connection_listener_stopping(_event)
@@ -134,6 +138,17 @@ module Karafka
 
         private
 
+        # Reports the current status to the supervisor periodically
+        def report_status
+          periodically do
+            return unless node
+
+            current_status = status
+
+            current_status.positive? ? node.unhealthy(current_status) : node.healthy
+          end
+        end
+
         # @return [Integer] object id of the current thread
         def thread_id
           Thread.current.object_id

data/lib/karafka/processing/jobs_queue.rb

@@ -180,6 +180,16 @@ module Karafka
         end
       end
 
+      # Returns a snapshot of all jobs currently in processing per group.
+      # Useful for diagnostics during forceful shutdown to understand what is blocking.
+      #
+      # @return [Hash{String => Array<Jobs::Base>}] hash mapping group ids to arrays of jobs
+      def in_processing
+        @mutex.synchronize do
+          @in_processing.transform_values(&:dup).freeze
+        end
+      end
+
       private
 
       # @param group_id [String] id of the group in which jobs we're interested.

data/lib/karafka/server.rb

@@ -126,10 +126,19 @@ module Karafka
 
         raise Errors::ForcefulShutdownError
       rescue Errors::ForcefulShutdownError => e
+        active_listeners = listeners.select(&:active?)
+        alive_workers = workers.select(&:alive?)
+
+        # Collect details about subscription groups that still have jobs in processing
+        in_processing = jobs_queue ? jobs_queue.in_processing : {}
+
         Karafka.monitor.instrument(
           "error.occurred",
           caller: self,
           error: e,
+          active_listeners: active_listeners,
+          alive_workers: alive_workers,
+          in_processing: in_processing,
           type: "app.stopping.error"
         )
 

data/lib/karafka/setup/config_proxy.rb

@@ -119,6 +119,11 @@ module Karafka
     # @see Karafka::Setup::Config.setup
     # @see Karafka::Setup::Config.configure_components
     class ConfigProxy < SimpleDelegator
+      # A frozen object used as a sentinel value to indicate that the producer is called as
+      # a reader (without block) during setup.
+      READ = Object.new.freeze
+      private_constant :READ
+
       # @return [Proc] the stored producer initialization block (defaults to empty lambda)
       attr_reader :producer_initialization_block
 
@@ -195,10 +200,13 @@ module Karafka
       # @example Direct producer assignment
       #   custom_producer = WaterDrop::Producer.new { |c| c.kafka = { 'bootstrap.servers' => 'localhost:9092' } }
       #   config.producer = custom_producer
-      def producer(instance = nil, &block)
+      def producer(instance = READ, &block)
         if block
           # Store the configuration block for later execution
           @producer_initialization_block = block
+        elsif instance == READ
+          # Reading the producer configuration without a block
+          __getobj__.producer
         else
           # Direct assignment - delegate to real config via __getobj__
           __getobj__.producer = instance

data/lib/karafka/swarm/liveness_listener.rb

@@ -19,13 +19,24 @@ module Karafka
         @mutex = Mutex.new
       end
 
-      # Since there may be many statistics emitted from multiple listeners, we do not want to write
-      # statuses that often. Instead we do it only once in a while which should be enough
-      #
-      # While this may provide a small lag in the orphaned detection, it does not really matter
-      # as it will be picked up fast enough.
+      # Report from the fetch loop at the top of each iteration
       # @param _event [Karafka::Core::Monitoring::Event]
-      def on_statistics_emitted(_event)
+      def on_connection_listener_fetch_loop(_event)
+        report_liveness
+      end
+
+      # Report from events poll so liveness works during long processing.
+      # This event fires periodically during wait even when the listener is blocked on consumer
+      # jobs, preventing the supervisor from killing the node.
+      # @param _event [Karafka::Core::Monitoring::Event]
+      def on_client_events_poll(_event)
+        report_liveness
+      end
+
+      private
+
+      # Reports liveness to the supervisor periodically
+      def report_liveness
         periodically do
           Kernel.exit!(orphaned_exit_code) if node.orphaned?
 
@@ -33,8 +44,6 @@ module Karafka
         end
       end
 
-      private
-
       # Wraps the logic with a mutex
       def synchronize(&)
         @mutex.synchronize(&)

data/lib/karafka/swarm/node.rb

@@ -27,18 +27,6 @@ module Karafka
       # @return [Integer] pid of the node
       attr_reader :pid
 
-      # When re-creating a producer in the fork, those are not attributes we want to inherit
-      # from the parent process because they are updated in the fork. If user wants to take those
-      # from the parent process, he should redefine them by overwriting the whole producer.
-      SKIPPABLE_NEW_PRODUCER_ATTRIBUTES = %i[
-        id
-        kafka
-        logger
-        oauth
-      ].freeze
-
-      private_constant :SKIPPABLE_NEW_PRODUCER_ATTRIBUTES
-
       # @param id [Integer] number of the fork. Used for uniqueness setup for group client ids and
       #   other stuff where we need to know a unique reference of the fork in regards to the rest
       #   of them.
@@ -70,24 +58,7 @@ module Karafka
           config.producer.close
 
           old_producer = config.producer
-          old_producer_config = old_producer.config
-
-          # Supervisor producer is closed, hence we need a new one here
-          config.producer = WaterDrop::Producer.new do |p_config|
-            p_config.kafka = Setup::AttributesMap.producer(kafka.dup)
-            p_config.logger = config.logger
-
-            old_producer_config.to_h.each do |key, value|
-              next if SKIPPABLE_NEW_PRODUCER_ATTRIBUTES.include?(key)
-
-              p_config.public_send("#{key}=", value)
-            end
-
-            # Namespaced attributes need to be migrated directly on their config node
-            old_producer_config.oauth.to_h.each do |key, value|
-              p_config.oauth.public_send("#{key}=", value)
-            end
-          end
+          config.producer = ProducerReplacer.new.call(old_producer, kafka, config.logger)
 
           @pid = ::Process.pid
           @reader.close