karafka 2.4.0 → 2.4.18

data/lib/karafka/routing/subscription_group.rb

@@ -120,9 +120,10 @@ module Karafka
       def build_kafka
         kafka = Setup::AttributesMap.consumer(@topics.first.kafka.dup)
 
+        inject_defaults(kafka)
         inject_group_instance_id(kafka)
+        inject_client_id(kafka)
 
-        kafka[:'client.id'] ||= client_id
         kafka[:'group.id'] ||= @consumer_group.id
         kafka[:'auto.offset.reset'] ||= @topics.first.initial_offset
         # Karafka manages the offsets based on the processing state, thus we do not rely on the
@@ -132,6 +133,33 @@ module Karafka
         kafka
       end
 
+      # Injects (if needed) defaults
+      #
+      # @param kafka [Hash] kafka level config
+      def inject_defaults(kafka)
+        Setup::DefaultsInjector.consumer(kafka)
+      end
+
+      # Sets (if needed) the client.id attribute
+      #
+      # @param kafka [Hash] kafka level config
+      def inject_client_id(kafka)
+        # If client id is set directly on librdkafka level, we do nothing and just go with what
+        # end user has configured
+        return if kafka.key?(:'client.id')
+
+        # This mitigates an issue for multiplexing and potentially other cases when running
+        # multiple karafka processes on one machine, where librdkafka goes into an infinite
+        # loop when using cooperative-sticky and upscaling.
+        #
+        # @see https://github.com/confluentinc/librdkafka/issues/4783
+        kafka[:'client.id'] = if kafka[:'partition.assignment.strategy'] == 'cooperative-sticky'
+                                "#{client_id}/#{Time.now.to_f}/#{SecureRandom.hex[0..9]}"
+                              else
+                                client_id
+                              end
+      end
+
       # If we use static group memberships, there can be a case, where same instance id would
       # be set on many subscription groups as the group instance id from Karafka perspective is
       # set per config. Each instance even if they are subscribed to different topics needs to

data/lib/karafka/routing/topic.rb

@@ -54,6 +54,20 @@ module Karafka
         RUBY
       end
 
+      # Often users want to have the same basic cluster setup with small setting alterations
+      # This method allows us to do so by setting `inherit` to `true`. Whe inherit is enabled,
+      # settings will be merged with defaults.
+      #
+      # @param settings [Hash] kafka scope settings. If `:inherit` key is provided, it will
+      #   instruct the assignment to merge with root level defaults
+      #
+      # @note It is set to `false` by default to preserve backwards compatibility
+      def kafka=(settings = {})
+        inherit = settings.delete(:inherit)
+
+        @kafka = inherit ? Karafka::App.config.kafka.merge(settings) : settings
+      end
+
       # @return [String] name of subscription that will go to librdkafka
       def subscription_name
         name
@@ -81,7 +95,16 @@ module Karafka
       # Allows to disable topic by invoking this method and setting it to `false`.
       # @param active [Boolean] should this topic be consumed or not
       def active(active)
-        @active = active
+        # Do not allow for active overrides. Basically if this is set on the topic level, defaults
+        # will not overwrite it and this is desired. Otherwise because of the fact that this is
+        # not a full feature config but just a flag, default value would always overwrite the
+        # per-topic config since defaults application happens after the topic config block
+        unless @active_assigned
+          @active = active
+          @active_assigned = true
+        end
+
+        @active
       end
 
       # @return [Class] consumer class that we should use

data/lib/karafka/runner.rb

@@ -3,17 +3,18 @@
 module Karafka
   # Class used to run the Karafka listeners in separate threads
   class Runner
-    def initialize
-      @manager = App.config.internal.connection.manager
-      @conductor = App.config.internal.connection.conductor
-    end
+    include Helpers::ConfigImporter.new(
+      manager: %i[internal connection manager],
+      conductor: %i[internal connection conductor],
+      jobs_queue_class: %i[internal processing jobs_queue_class]
+    )
 
     # Starts listening on all the listeners asynchronously and handles the jobs queue closing
     # after listeners are done with their work.
     def call
       # Despite possibility of having several independent listeners, we aim to have one queue for
       # jobs across and one workers poll for that
-      jobs_queue = App.config.internal.processing.jobs_queue_class.new
+      jobs_queue = jobs_queue_class.new
 
       workers = Processing::WorkersBatch.new(jobs_queue)
       listeners = Connection::ListenersBatch.new(jobs_queue)
@@ -23,7 +24,7 @@ module Karafka
       Karafka::App.run!
 
       # Register all the listeners so they can be started and managed
-      @manager.register(listeners)
+      manager.register(listeners)
 
       workers.each_with_index { |worker, i| worker.async_call("karafka.worker##{i}") }
 
@@ -32,10 +33,10 @@ module Karafka
       Karafka::Server.listeners = listeners
       Karafka::Server.jobs_queue = jobs_queue
 
-      until @manager.done?
-        @conductor.wait
+      until manager.done?
+        conductor.wait
 
-        @manager.control
+        manager.control
       end
 
       # We close the jobs queue only when no listener threads are working.

data/lib/karafka/server.rb

@@ -3,6 +3,12 @@
 module Karafka
   # Karafka consuming server class
   class Server
+    # How long should we wait on the listeners forceful shutdown when they are stuck beyond the
+    # shutdown timeout before forcing a bypass
+    FORCEFUL_SHUTDOWN_WAIT = 5
+
+    private_constant :FORCEFUL_SHUTDOWN_WAIT
+
     class << self
       # Set of consuming threads. Each consumer thread contains a single consumer
       attr_accessor :listeners
@@ -13,6 +19,17 @@ module Karafka
       # Jobs queue
       attr_accessor :jobs_queue
 
+      # Mode in which the Karafka server is executed. It can be:
+      #
+      # - :standalone - regular karafka consumer process
+      # - :embedded - embedded in a different process and not supervised
+      # - :supervisor - swarm supervisor process
+      # - :swarm - one of swarm processes
+      #
+      # Sometimes it is important to know in what mode we operate, especially from UI perspective
+      # as not everything is possible when operating in non-standalone mode, etc.
+      attr_accessor :execution_mode
+
       # Method which runs app
       def run
         self.listeners = []
@@ -105,9 +122,23 @@ module Karafka
         # We're done waiting, lets kill them!
         workers.each(&:terminate)
         listeners.active.each(&:terminate)
+
         # We always need to shutdown clients to make sure we do not force the GC to close consumer.
         # This can cause memory leaks and crashes.
-        listeners.each(&:shutdown)
+        # We run it in a separate thread in case this would hang and we ignore it after the time
+        # we assigned to it and force shutdown as we prefer to stop the process rather than wait
+        # indefinitely even with risk of VM crash as this is a last resort.
+        Thread.new do
+          listeners.each(&:shutdown)
+        rescue StandardError => e
+          # If anything wrong happened during shutdown, we also want to record it
+          Karafka.monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: e,
+            type: 'app.forceful_stopping.error'
+          )
+        end.join(FORCEFUL_SHUTDOWN_WAIT)
 
         # We also do not forcefully terminate everything when running in the embedded mode,
         # otherwise we would overwrite the shutdown process of the process that started Karafka
@@ -151,5 +182,10 @@ module Karafka
         config.internal.process
       end
     end
+
+    # Always start with standalone so there always is a value for the execution mode.
+    # This is overwritten quickly during boot, but just in case someone would reach it prior to
+    # booting, we want to have the default value.
+    self.execution_mode = :standalone
   end
 end

data/lib/karafka/setup/attributes_map.rb

@@ -38,6 +38,7 @@ module Karafka
         default_topic_conf
         enable.auto.commit
         enable.auto.offset.store
+        enable.metrics.push
         enable.partition.eof
         enable.random.seed
         enable.sasl.oauthbearer.unsecure.jwt
@@ -89,6 +90,8 @@ module Karafka
         reconnect.backoff.max.ms
         reconnect.backoff.ms
         resolve_cb
+        retry.backoff.max.ms
+        retry.backoff.ms
         sasl.kerberos.keytab
         sasl.kerberos.kinit.cmd
         sasl.kerberos.min.time.before.relogin
@@ -153,6 +156,7 @@ module Karafka
       # List of rdkafka producer accepted attributes
       PRODUCER = %i[
         acks
+        allow.auto.create.topics
         api.version.fallback.ms
         api.version.request
         api.version.request.timeout.ms
@@ -181,6 +185,7 @@ module Karafka
         dr_msg_cb
         enable.gapless.guarantee
         enable.idempotence
+        enable.metrics.push
         enable.random.seed
         enable.sasl.oauthbearer.unsecure.jwt
         enable.ssl.certificate.verification
@@ -289,7 +294,12 @@ module Karafka
       ].freeze
 
       # Location of the file with rdkafka settings list
-      SOURCE = 'https://raw.githubusercontent.com/edenhill/librdkafka/master/CONFIGURATION.md'
+      SOURCE = <<~SOURCE.delete("\n").gsub(/\s+/, '/')
+        https://raw.githubusercontent.com
+          confluentinc/librdkafka
+          v#{Rdkafka::LIBRDKAFKA_VERSION}
+          CONFIGURATION.md
+      SOURCE
 
       private_constant :SOURCE
 
@@ -337,9 +347,6 @@ module Karafka
             end
           end
 
-          # This can be removed when 0.13 librdkafka is released
-          attributes[:producer].delete_if { |val| val == 'allow.auto.create.topics' }
-
           attributes.transform_values!(&:sort)
           attributes.each_value { |vals| vals.map!(&:to_sym) }
           attributes

data/lib/karafka/setup/config.rb

@@ -14,34 +14,6 @@ module Karafka
     class Config
       extend ::Karafka::Core::Configurable
 
-      # Defaults for kafka settings, that will be overwritten only if not present already
-      KAFKA_DEFAULTS = {
-        # We emit the statistics by default, so all the instrumentation and web-ui work out of
-        # the box, without requiring users to take any extra actions aside from enabling.
-        'statistics.interval.ms': 5_000,
-        'client.software.name': 'karafka',
-        'client.software.version': [
-          "v#{Karafka::VERSION}",
-          "rdkafka-ruby-v#{Rdkafka::VERSION}",
-          "librdkafka-v#{Rdkafka::LIBRDKAFKA_VERSION}"
-        ].join('-')
-      }.freeze
-
-      # Contains settings that should not be used in production but make life easier in dev
-      KAFKA_DEV_DEFAULTS = {
-        # Will create non-existing topics automatically.
-        # Note that the broker needs to be configured with `auto.create.topics.enable=true`
-        # While it is not recommended in prod, it simplifies work in dev
-        'allow.auto.create.topics': 'true',
-        # We refresh the cluster state often as newly created topics in dev may not be detected
-        # fast enough. Fast enough means within reasonable time to provide decent user experience
-        # While it's only a one time thing for new topics, it can still be irritating to have to
-        # restart the process.
-        'topic.metadata.refresh.interval.ms': 5_000
-      }.freeze
-
-      private_constant :KAFKA_DEFAULTS, :KAFKA_DEV_DEFAULTS
-
       # Available settings
 
       # Namespace for Pro version related license management. If you use LGPL, no need to worry
@@ -96,6 +68,11 @@ module Karafka
       setting :strict_topics_namespacing, default: true
       # option [String] default consumer group name for implicit routing
       setting :group_id, default: 'app'
+      # option [Boolean] when set to true, it will validate as part of the routing validation, that
+      # all topics and DLQ topics (even not active) have the declarative topics definitions.
+      # Really useful when you want to ensure that all topics in routing are managed via
+      # declaratives.
+      setting :strict_declarative_topics, default: false
 
       setting :oauth do
         # option [false, #call] Listener for using oauth bearer. This listener will be able to
@@ -231,6 +208,9 @@ module Karafka
           setting :manager, default: Connection::Manager.new
           # Controls frequency of connections management checks
           setting :conductor, default: Connection::Conductor.new
+          # How long should we wait before a critical listener recovery
+          # Too short may cause endless rebalance loops
+          setting :reset_backoff, default: 60_000
 
           # Settings that are altered by our client proxy layer
           setting :proxy do
@@ -310,6 +290,9 @@ module Karafka
           setting :expansions_selector, default: Processing::ExpansionsSelector.new
           # option [Class] executor class
           setting :executor_class, default: Processing::Executor
+          # option worker_job_call_wrapper [Proc, false] callable object that will be used to wrap
+          #   the worker execution of a job or false if no wrapper needed
+          setting :worker_job_call_wrapper, default: false
         end
 
         # Things related to operating on messages
@@ -350,7 +333,6 @@ module Karafka
           Pro::Loader.pre_setup_all(config) if Karafka.pro?
 
           configure(&block)
-          merge_kafka_defaults!(config)
 
           Contracts::Config.new.validate!(config.to_h)
 
@@ -374,29 +356,6 @@ module Karafka
 
         private
 
-        # Propagates the kafka setting defaults unless they are already present
-        # This makes it easier to set some values that users usually don't change but still allows
-        # them to overwrite the whole hash if they want to
-        # @param config [Karafka::Core::Configurable::Node] config of this producer
-        def merge_kafka_defaults!(config)
-          KAFKA_DEFAULTS.each do |key, value|
-            next if config.kafka.key?(key)
-
-            config.kafka[key] = value
-          end
-
-          # Use Karafka client_id as kafka client id if not set
-          config.kafka[:'client.id'] ||= config.client_id
-
-          return if Karafka::App.env.production?
-
-          KAFKA_DEV_DEFAULTS.each do |key, value|
-            next if config.kafka.key?(key)
-
-            config.kafka[key] = value
-          end
-        end
-
         # Sets up all the components that are based on the user configuration
         # @note At the moment it is only WaterDrop
         def configure_components

data/lib/karafka/setup/defaults_injector.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Setup
+    # Injector that enriches each Kafka cluster with needed defaults. User may use more than one
+    # cluster and define them on a per-topic basis. We use this when we build the final config
+    # per subscription group.
+    module DefaultsInjector
+      # Defaults for kafka settings, that will be overwritten only if not present already
+      CONSUMER_KAFKA_DEFAULTS = {
+        # We emit the statistics by default, so all the instrumentation and web-ui work out of
+        # the box, without requiring users to take any extra actions aside from enabling.
+        'statistics.interval.ms': 5_000,
+        'client.software.name': 'karafka',
+        # Same as librdkafka default, we inject it nonetheless to have it always available as
+        # some features may use this value for computation and it is better to ensure, we do
+        # always have it
+        'max.poll.interval.ms': 300_000,
+        'client.software.version': [
+          "v#{Karafka::VERSION}",
+          "rdkafka-ruby-v#{Rdkafka::VERSION}",
+          "librdkafka-v#{Rdkafka::LIBRDKAFKA_VERSION}"
+        ].join('-')
+      }.freeze
+
+      # Contains settings that should not be used in production but make life easier in dev
+      CONSUMER_KAFKA_DEV_DEFAULTS = {
+        # Will create non-existing topics automatically.
+        # Note that the broker needs to be configured with `auto.create.topics.enable=true`
+        # While it is not recommended in prod, it simplifies work in dev
+        'allow.auto.create.topics': 'true',
+        # We refresh the cluster state often as newly created topics in dev may not be detected
+        # fast enough. Fast enough means within reasonable time to provide decent user experience
+        # While it's only a one time thing for new topics, it can still be irritating to have to
+        # restart the process.
+        'topic.metadata.refresh.interval.ms': 5_000
+      }.freeze
+
+      private_constant :CONSUMER_KAFKA_DEFAULTS, :CONSUMER_KAFKA_DEV_DEFAULTS
+
+      class << self
+        # Propagates the kafka setting defaults unless they are already present for consumer config
+        # This makes it easier to set some values that users usually don't change but still allows
+        # them to overwrite the whole hash if they want to
+        # @param kafka_config [Hash] kafka scoped config
+        def consumer(kafka_config)
+          CONSUMER_KAFKA_DEFAULTS.each do |key, value|
+            next if kafka_config.key?(key)
+
+            kafka_config[key] = value
+          end
+
+          return if Karafka::App.env.production?
+
+          CONSUMER_KAFKA_DEV_DEFAULTS.each do |key, value|
+            next if kafka_config.key?(key)
+
+            kafka_config[key] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/swarm/node.rb

@@ -68,6 +68,8 @@ module Karafka
           monitor.subscribe(liveness_listener)
           monitor.instrument('swarm.node.after_fork', caller: self)
 
+          Karafka::Process.tags.add(:execution_mode, 'mode:swarm')
+          Server.execution_mode = :swarm
           Server.run
 
           @writer.close

data/lib/karafka/swarm/supervisor.rb

@@ -20,7 +20,9 @@ module Karafka
         shutdown_timeout: %i[shutdown_timeout],
         supervision_sleep: %i[internal supervision_sleep],
         forceful_exit_code: %i[internal forceful_exit_code],
-        process: %i[internal process]
+        process: %i[internal process],
+        cli_contract: %i[internal cli contract],
+        activity_manager: %i[internal routing activity_manager]
       )
 
       # How long extra should we wait on shutdown before forceful termination
@@ -39,6 +41,9 @@ module Karafka
 
       # Creates needed number of forks, installs signals and starts supervision
       def run
+        # Validate the CLI provided options the same way as we do for the regular server
+        cli_contract.validate!(activity_manager.to_h)
+
         # Close producer just in case. While it should not be used, we do not want even a
         # theoretical case since librdkafka is not thread-safe.
         # We close it prior to forking just to make sure, there is no issue with initialized
@@ -67,7 +72,11 @@ module Karafka
           lock
           control
         end
-      # If anything went wrong, signal this and die
+
+      # If the cli contract validation failed reraise immediately and stop the process
+      rescue Karafka::Errors::InvalidConfigurationError => e
+        raise e
+      # If anything went wrong during supervision, signal this and die
       # Supervisor is meant to be thin and not cause any issues. If you encounter this case
       # please report it as it should be considered critical
       rescue StandardError => e

data/lib/karafka/templates/karafka.rb.erb

@@ -59,7 +59,7 @@ class KarafkaApp < Karafka::App
     )
   )
 
-  # You can subscribe to all consumer related errors and record/track then that way
+  # You can subscribe to all consumer related errors and record/track them that way
   #
   # Karafka.monitor.subscribe 'error.occurred' do |event|
   #   type = event[:type]
@@ -68,7 +68,7 @@ class KarafkaApp < Karafka::App
   #   ErrorTracker.send_error(error, type, details)
   # end
 
-  # You can subscribe to all producer related errors and record/track then that way
+  # You can subscribe to all producer related errors and record/track them that way
   # Please note, that producer and consumer have their own notifications pipeline so you need to
   # setup error tracking independently for each of them
   #

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.4.0'
+  VERSION = '2.4.18'
 end

data/lib/karafka.rb

@@ -49,17 +49,35 @@ module Karafka
       @monitor ||= App.config.monitor
     end
 
-    # @return [String] root path of this gem
+    # @return [Pathname] root path of this gem
     def gem_root
       Pathname.new(File.expand_path('..', __dir__))
     end
 
-    # @return [String] Karafka app root path (user application path)
+    # @return [Pathname] Karafka app root path (user application path)
     def root
-      Pathname.new(ENV['KARAFKA_ROOT_DIR'] || File.dirname(ENV['BUNDLE_GEMFILE']))
+      return @root if @root
+
+      # If user points to a different root explicitly, use it
+      if ENV['KARAFKA_ROOT_DIR']
+        @root = Pathname.new(ENV['KARAFKA_ROOT_DIR'])
+
+        return @root
+      end
+
+      # By default we infer the project root from bundler.
+      # We cannot use the BUNDLE_GEMFILE env directly because it may be altered by things like
+      # ruby-lsp. Instead we always fallback to the most outer Gemfile. In most of the cases, it
+      # won't matter but in case of some automatic setup alterations like ruby-lsp, the location
+      # from which the project starts may not match the original Gemfile.
+      @root = Pathname.new(
+        File.dirname(
+          Bundler.with_unbundled_env { Bundler.default_gemfile }
+        )
+      )
     end
 
-    # @return [String] path to Karafka gem root core
+    # @return [Pathname] path to Karafka gem root core
     def core_root
       Pathname.new(File.expand_path('karafka', __dir__))
     end
@@ -71,11 +89,23 @@ module Karafka
 
     # @return [Boolean] Do we run within/with Rails. We use this to initialize Railtie and proxy
     #   the console invocation to Rails
+    #
+    # @note We allow users to disable Rails require because having Rails in the Gemfile does not
+    #   always mean user wants to have it required. User may want to run Karafka without Rails
+    #   even when having both in the same Gemfile.
     def rails?
       return @rails if instance_variable_defined?('@rails')
 
-      # Do not load Rails again if already loaded
-      Object.const_defined?('Rails::Railtie') || require('rails')
+      @rails = Object.const_defined?('Rails::Railtie')
+
+      # If Rails exists we set it immediately based on its presence and return
+      return @rails if @rails
+
+      # If rails is not present and user wants us not to force-load it, we return
+      return @rails if ENV['KARAFKA_REQUIRE_RAILS'] == 'false'
+
+      # If we should try to require it, we try and if no error, it means its there
+      require('rails')
 
       @rails = true
     rescue LoadError
@@ -92,7 +122,17 @@ module Karafka
     #   KARAFKA_BOOT_FILE='/home/app_path/app.rb'
     #   Karafka.boot_file #=> '/home/app_path/app.rb'
     def boot_file
-      Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
+      boot_file = Pathname.new(ENV['KARAFKA_BOOT_FILE'] || File.join(Karafka.root, 'karafka.rb'))
+
+      return boot_file if boot_file.absolute?
+      return boot_file if boot_file.to_s == 'false'
+
+      Pathname.new(
+        File.expand_path(
+          boot_file,
+          Karafka.root
+        )
+      )
     end
 
     # We need to be able to overwrite both monitor and logger after the configuration in case they