karafka 2.5.5 → 2.5.7

data/lib/karafka/swarm/node.rb

@@ -27,18 +27,6 @@ module Karafka
       # @return [Integer] pid of the node
       attr_reader :pid
 
-      # When re-creating a producer in the fork, those are not attributes we want to inherit
-      # from the parent process because they are updated in the fork. If user wants to take those
-      # from the parent process, he should redefine them by overwriting the whole producer.
-      SKIPPABLE_NEW_PRODUCER_ATTRIBUTES = %i[
-        id
-        kafka
-        logger
-        oauth
-      ].freeze
-
-      private_constant :SKIPPABLE_NEW_PRODUCER_ATTRIBUTES
-
       # @param id [Integer] number of the fork. Used for uniqueness setup for group client ids and
       #   other stuff where we need to know a unique reference of the fork in regards to the rest
       #   of them.
@@ -70,24 +58,7 @@ module Karafka
           config.producer.close
 
           old_producer = config.producer
-          old_producer_config = old_producer.config
-
-          # Supervisor producer is closed, hence we need a new one here
-          config.producer = WaterDrop::Producer.new do |p_config|
-            p_config.kafka = Setup::AttributesMap.producer(kafka.dup)
-            p_config.logger = config.logger
-
-            old_producer_config.to_h.each do |key, value|
-              next if SKIPPABLE_NEW_PRODUCER_ATTRIBUTES.include?(key)
-
-              p_config.public_send("#{key}=", value)
-            end
-
-            # Namespaced attributes need to be migrated directly on their config node
-            old_producer_config.oauth.to_h.each do |key, value|
-              p_config.oauth.public_send("#{key}=", value)
-            end
-          end
+          config.producer = ProducerReplacer.new.call(old_producer, kafka, config.logger)
 
           @pid = ::Process.pid
           @reader.close
@@ -100,9 +71,9 @@ module Karafka
 
           swarm.node = self
           monitor.subscribe(liveness_listener)
-          monitor.instrument('swarm.node.after_fork', caller: self)
+          monitor.instrument("swarm.node.after_fork", caller: self)
 
-          Karafka::Process.tags.add(:execution_mode, 'mode:swarm')
+          Karafka::Process.tags.add(:execution_mode, "mode:swarm")
           Karafka::Process.tags.add(:swarm_nodeid, "node:#{@id}")
 
           Server.execution_mode.swarm!
@@ -118,7 +89,7 @@ module Karafka
       # Indicates that this node is doing well
       # @note Child API
       def healthy
-        write('0')
+        write("0")
       end
 
       # Indicates, that this node has failed
@@ -127,7 +98,7 @@ module Karafka
       #   have complex health-checking with reporting.
       # @note Child API
       # @note We convert this to string to normalize the API
-      def unhealthy(reason_code = '1')
+      def unhealthy(reason_code = "1")
         write(reason_code.to_s)
       end
 
@@ -192,19 +163,19 @@ module Karafka
       # Sends sigterm to the node
       # @note Parent API
       def stop
-        signal('TERM')
+        signal("TERM")
       end
 
       # Sends sigtstp to the node
       # @note Parent API
       def quiet
-        signal('TSTP')
+        signal("TSTP")
       end
 
       # Terminates node
       # @note Parent API
       def terminate
-        signal('KILL')
+        signal("KILL")
       end
 
       # Sends provided signal to the node

data/lib/karafka/swarm/producer_replacer.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Swarm
+    # Builds a new WaterDrop producer that inherits configuration from an old one
+    #
+    # When a swarm node forks, the parent's producer must be replaced with a new one.
+    # This class encapsulates the logic for building that new producer, inheriting all relevant
+    # settings from the old one while generating fresh connection-level configuration.
+    class ProducerReplacer
+      # Attributes that should not be directly copied from the old producer config because they
+      # are either regenerated fresh (kafka, logger, id) or handled via their own namespaced
+      # migration (oauth, polling, polling.fd).
+      SKIPPABLE_ATTRIBUTES = %i[
+        id
+        kafka
+        logger
+        oauth
+        polling
+        fd
+      ].freeze
+
+      private_constant :SKIPPABLE_ATTRIBUTES
+
+      # Builds a new WaterDrop producer inheriting configuration from the old one
+      #
+      # @param old_producer [WaterDrop::Producer] the old producer to inherit settings from
+      # @param kafka [Hash] app-level kafka configuration
+      # @param logger [Object] logger instance for the new producer
+      # @return [WaterDrop::Producer] new producer with inherited configuration
+      def call(old_producer, kafka, logger)
+        old_producer_config = old_producer.config
+
+        WaterDrop::Producer.new do |p_config|
+          p_config.logger = logger
+
+          migrate_kafka(p_config, old_producer_config, kafka)
+          migrate_root(p_config, old_producer_config)
+          migrate_oauth(p_config, old_producer_config)
+          migrate_polling(p_config, old_producer_config)
+          migrate_polling_fd(p_config, old_producer_config)
+        end
+      end
+
+      private
+
+      # Migrates root-level producer attributes from the old producer, skipping those that are
+      # regenerated fresh or handled by their own namespaced migration
+      #
+      # @param p_config [WaterDrop::Config] new producer config being built
+      # @param old_producer_config [WaterDrop::Config] old producer config to inherit from
+      def migrate_root(p_config, old_producer_config)
+        old_producer_config.to_h.each do |key, value|
+          next if SKIPPABLE_ATTRIBUTES.include?(key)
+
+          p_config.public_send("#{key}=", value)
+        end
+      end
+
+      # Builds fresh kafka config from app-level settings and preserves any producer-specific
+      # kafka settings from the old producer (e.g., enable.idempotence) that aren't in the
+      # base app kafka config
+      #
+      # @param p_config [WaterDrop::Config] new producer config being built
+      # @param old_producer_config [WaterDrop::Config] old producer config to inherit from
+      # @param kafka [Hash] app-level kafka configuration
+      def migrate_kafka(p_config, old_producer_config, kafka)
+        p_config.kafka = Setup::AttributesMap.producer(kafka.dup)
+
+        old_producer_config.kafka.each do |key, value|
+          next if p_config.kafka.key?(key)
+
+          p_config.kafka[key] = value
+        end
+      end
+
+      # Migrates oauth configuration from the old producer
+      #
+      # @param p_config [WaterDrop::Config] new producer config being built
+      # @param old_producer_config [WaterDrop::Config] old producer config to inherit from
+      def migrate_oauth(p_config, old_producer_config)
+        old_producer_config.oauth.to_h.each do |key, value|
+          p_config.oauth.public_send("#{key}=", value)
+        end
+      end
+
+      # Migrates polling configuration from the old producer
+      #
+      # @param p_config [WaterDrop::Config] new producer config being built
+      # @param old_producer_config [WaterDrop::Config] old producer config to inherit from
+      def migrate_polling(p_config, old_producer_config)
+        old_producer_config.polling.to_h.each do |key, value|
+          next if SKIPPABLE_ATTRIBUTES.include?(key)
+
+          p_config.polling.public_send("#{key}=", value)
+        end
+      end
+
+      # Migrates polling fd configuration from the old producer
+      #
+      # @param p_config [WaterDrop::Config] new producer config being built
+      # @param old_producer_config [WaterDrop::Config] old producer config to inherit from
+      def migrate_polling_fd(p_config, old_producer_config)
+        old_producer_config.polling.fd.to_h.each do |key, value|
+          p_config.polling.fd.public_send("#{key}=", value)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/swarm/supervisor.rb

@@ -66,7 +66,7 @@ module Karafka
         process.on_sigquit { stop }
         process.on_sigterm { stop }
         process.on_sigtstp { quiet }
-        process.on_sigttin { signal('TTIN') }
+        process.on_sigttin { signal("TTIN") }
         # Needed to be registered as we want to unlock on child changes
         process.on_sigchld { nil }
         process.on_any_active { unlock }
@@ -87,13 +87,13 @@ module Karafka
       # If anything went wrong during supervision, signal this and die
       # Supervisor is meant to be thin and not cause any issues. If you encounter this case
       # please report it as it should be considered critical
-      rescue StandardError => e
+      rescue => e
         monitor.instrument(
-          'error.occurred',
+          "error.occurred",
           caller: self,
           error: e,
           manager: manager,
-          type: 'swarm.supervisor.error'
+          type: "swarm.supervisor.error"
         )
 
         manager.terminate
@@ -148,11 +148,14 @@ module Karafka
         raise Errors::ForcefulShutdownError
       rescue Errors::ForcefulShutdownError => e
         monitor.instrument(
-          'error.occurred',
+          "error.occurred",
           caller: self,
           error: e,
           manager: manager,
-          type: 'app.stopping.error'
+          active_listeners: [],
+          alive_workers: [],
+          in_processing: {},
+          type: "app.stopping.error"
         )
 
         # Run forceful kill

data/lib/karafka/swarm.rb

@@ -14,7 +14,7 @@ module Karafka
 
         raise(
           Errors::UnsupportedOptionError,
-          'Swarm mode not supported on this platform'
+          "Swarm mode not supported on this platform"
         )
       end
 

data/lib/karafka/time_trackers/pause.rb

@@ -115,7 +115,7 @@ module Karafka
 
         timeout = backoff_factor * @timeout
 
-        @max_timeout && timeout > @max_timeout ? @max_timeout : timeout
+        (@max_timeout && timeout > @max_timeout) ? @max_timeout : timeout
       end
     end
   end

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.5.5'
+  VERSION = "2.5.7"
 end

data/lib/karafka.rb

@@ -1,21 +1,21 @@
 # frozen_string_literal: true
 
-require 'karafka-core'
-require 'delegate'
-require 'English'
-require 'rdkafka'
-require 'waterdrop'
-require 'json'
-require 'forwardable'
-require 'fileutils'
-require 'openssl'
-require 'optparse'
-require 'socket'
-require 'date'
-require 'singleton'
-require 'digest'
-require 'zeitwerk'
-require 'logger'
+require "karafka-core"
+require "delegate"
+require "English"
+require "rdkafka"
+require "waterdrop"
+require "json"
+require "forwardable"
+require "fileutils"
+require "openssl"
+require "optparse"
+require "socket"
+require "date"
+require "singleton"
+require "digest"
+require "zeitwerk"
+require "logger"
 
 # Karafka framework main namespace
 module Karafka
@@ -50,7 +50,7 @@ module Karafka
 
     # @return [Pathname] root path of this gem
     def gem_root
-      Pathname.new(File.expand_path('..', __dir__))
+      Pathname.new(File.expand_path("..", __dir__))
     end
 
     # @return [Pathname] Karafka app root path (user application path)
@@ -58,26 +58,26 @@ module Karafka
       return @root if @root
 
       # If user points to a different root explicitly, use it
-      if ENV['KARAFKA_ROOT_DIR']
-        @root = Pathname.new(ENV['KARAFKA_ROOT_DIR'])
+      if ENV["KARAFKA_ROOT_DIR"]
+        @root = Pathname.new(ENV["KARAFKA_ROOT_DIR"])
 
         return @root
       end
 
-      if defined?(::Bundler)
+      @root = if defined?(::Bundler)
         # By default we infer the project root from bundler.
         # We cannot use the BUNDLE_GEMFILE env directly because it may be altered by things like
         # ruby-lsp. Instead we always fallback to the most outer Gemfile. In most of the cases, it
         # won't matter but in case of some automatic setup alterations like ruby-lsp, the location
         # from which the project starts may not match the original Gemfile.
-        @root = Pathname.new(
+        Pathname.new(
           File.dirname(
             Bundler.with_unbundled_env { Bundler.default_gemfile }
           )
         )
       else
         # Fallback when Bundler is not available: use current directory
-        @root = Pathname.new(Dir.pwd)
+        Pathname.new(Dir.pwd)
       end
 
       @root
@@ -85,7 +85,7 @@ module Karafka
 
     # @return [Pathname] path to Karafka gem root core
     def core_root
-      Pathname.new(File.expand_path('karafka', __dir__))
+      Pathname.new(File.expand_path("karafka", __dir__))
     end
 
     # @return [Boolean] true if there is a valid pro token present
@@ -102,16 +102,16 @@ module Karafka
     def rails?
       return @rails if instance_variable_defined?(:@rails)
 
-      @rails = Object.const_defined?('Rails::Railtie')
+      @rails = Object.const_defined?("Rails::Railtie")
 
       # If Rails exists we set it immediately based on its presence and return
       return @rails if @rails
 
       # If rails is not present and user wants us not to force-load it, we return
-      return @rails if ENV['KARAFKA_REQUIRE_RAILS'] == 'false'
+      return @rails if ENV["KARAFKA_REQUIRE_RAILS"] == "false"
 
       # If we should try to require it, we try and if no error, it means its there
-      require('rails')
+      require("rails")
 
       @rails = true
     rescue LoadError
@@ -128,10 +128,10 @@ module Karafka
     #   KARAFKA_BOOT_FILE='/home/app_path/app.rb'
     #   Karafka.boot_file #=> '/home/app_path/app.rb'
     def boot_file
-      boot_file = Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
+      boot_file = Pathname.new(ENV["KARAFKA_BOOT_FILE"] || File.join(Karafka.root, "karafka.rb"))
 
       return boot_file if boot_file.absolute?
-      return boot_file if boot_file.to_s == 'false'
+      return boot_file if boot_file.to_s == "false"
 
       Pathname.new(
         File.expand_path(
@@ -158,19 +158,19 @@ end
 
 loader = Zeitwerk::Loader.for_gem
 # Do not load Rails extensions by default, this will be handled by Railtie if they are needed
-loader.ignore(Karafka.gem_root.join('lib/active_job'))
+loader.ignore(Karafka.gem_root.join("lib/active_job"))
 # Do not load CurrentAttributes components as they will be loaded if needed
 # @note We have to exclude both the .rb file as well as the whole directory so users can require
 # current attributes only when needed
-loader.ignore(Karafka.gem_root.join('lib/karafka/active_job/current_attributes'))
-loader.ignore(Karafka.gem_root.join('lib/karafka/active_job/current_attributes.rb'))
+loader.ignore(Karafka.gem_root.join("lib/karafka/active_job/current_attributes"))
+loader.ignore(Karafka.gem_root.join("lib/karafka/active_job/current_attributes.rb"))
 # Do not load Railtie. It will load if after everything is ready, so we don't have to load any
 # Karafka components when we require this railtie. Railtie needs to be loaded last.
-loader.ignore(Karafka.gem_root.join('lib/karafka/railtie'))
+loader.ignore(Karafka.gem_root.join("lib/karafka/railtie"))
 # Do not load pro components as they will be loaded if needed and allowed
-loader.ignore(Karafka.core_root.join('pro/'))
+loader.ignore(Karafka.core_root.join("pro/"))
 # Do not load vendors instrumentation components. Those need to be required manually if needed
-loader.ignore(Karafka.core_root.join('instrumentation/vendors'))
+loader.ignore(Karafka.core_root.join("instrumentation/vendors"))
 loader.setup
 loader.eager_load
 
@@ -182,12 +182,12 @@ Karafka::Routing::Features::Base.load_all
 # to make pro components available in case anyone wants to use them as a base to their own
 # custom components. Otherwise inheritance would not work.
 Karafka::Licenser.detect do
-  require 'karafka/pro/loader'
+  require "karafka/pro/loader"
 
   Karafka::Pro::Loader.require_all
 end
 
 # Load railtie after everything else is ready so we know we can rely on it.
-require 'karafka/railtie'
+require "karafka/railtie"
 
 Karafka::Constraints.verify!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.5.5
+  version: 2.5.7
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,14 +35,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.23.1
+        version: 0.24.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.23.1
+        version: 0.24.0
 - !ruby/object:Gem::Dependency
   name: waterdrop
   requirement: !ruby/object:Gem::Requirement
@@ -95,6 +95,7 @@ files:
 - LICENSE-LGPL
 - README.md
 - bin/karafka
+- certs/expired.txt
 - certs/karafka-pro.pem
 - config/locales/errors.yml
 - config/locales/pro_errors.yml
@@ -216,6 +217,8 @@ files:
 - lib/karafka/pro/cli/parallel_segments/base.rb
 - lib/karafka/pro/cli/parallel_segments/collapse.rb
 - lib/karafka/pro/cli/parallel_segments/distribute.rb
+- lib/karafka/pro/cli/topics.rb
+- lib/karafka/pro/cli/topics/health.rb
 - lib/karafka/pro/connection/manager.rb
 - lib/karafka/pro/connection/multiplexing/listener.rb
 - lib/karafka/pro/contracts/base.rb
@@ -539,6 +542,7 @@ files:
 - lib/karafka/swarm/liveness_listener.rb
 - lib/karafka/swarm/manager.rb
 - lib/karafka/swarm/node.rb
+- lib/karafka/swarm/producer_replacer.rb
 - lib/karafka/swarm/supervisor.rb
 - lib/karafka/templates/application_consumer.rb.erb
 - lib/karafka/templates/example_consumer.rb.erb