karafka 2.3.2 → 2.4.0.beta1

data/lib/karafka/routing/proxy.rb

@@ -18,22 +18,17 @@ module Karafka
         instance_eval(&defaults) if defaults
       end
 
-      # Ruby 2.7.0 to 2.7.2 do not have arg forwarding, so we fallback to the old way
-      arg_forwarding = RUBY_VERSION < '3.0' ? '*args, &block' : '...'
-
-      class_eval <<~RUBY, __FILE__, __LINE__ + 1
-        # Translates the no "=" DSL of routing into elements assignments on target
-        # @param method_name [Symbol] name of the missing method
-        def method_missing(method_name, #{arg_forwarding})
-          return super unless respond_to_missing?(method_name)
+      # Translates the no "=" DSL of routing into elements assignments on target
+      # @param method_name [Symbol] name of the missing method
+      def method_missing(method_name, ...)
+        return super unless respond_to_missing?(method_name)
 
-          if @target.respond_to?(:"\#{method_name}=")
-            @target.public_send(:"\#{method_name}=", #{arg_forwarding})
-          else
-            @target.public_send(method_name, #{arg_forwarding})
-          end
+        if @target.respond_to?(:"#{method_name}=")
+          @target.public_send(:"#{method_name}=", ...)
+        else
+          @target.public_send(method_name, ...)
         end
-      RUBY
+      end
 
       # Tells whether or not a given element exists on the target
       # @param method_name [Symbol] name of the missing method

data/lib/karafka/routing/router.rb

@@ -23,7 +23,7 @@ module Karafka
       end
 
       # Finds the topic by name (in any consumer group) and if not present, will built a new
-      # representation of the topic with the defaults and default deserializer.
+      # representation of the topic with the defaults and default deserializers.
       #
       # This is used in places where we may operate on topics that are not part of the routing
       # but we want to do something on them (display data, iterate over, etc)
@@ -33,7 +33,16 @@ module Karafka
       # @note Please note, that in case of a new topic, it will have a newly built consumer group
       #   as well, that is not part of the routing.
       def find_or_initialize_by_name(name)
-        find_by(name: name) || Topic.new(name, ConsumerGroup.new(name))
+        existing_topic = find_by(name: name)
+
+        return existing_topic if existing_topic
+
+        virtual_topic = Topic.new(name, ConsumerGroup.new(name))
+
+        Karafka::Routing::Proxy.new(
+          virtual_topic,
+          Karafka::App.config.internal.routing.builder.defaults
+        ).target
       end
 
       module_function :find_by

data/lib/karafka/routing/subscription_group.rb

@@ -76,7 +76,8 @@ module Karafka
         activity_manager.active?(:subscription_groups, name)
       end
 
-      # @return [Array<String>] names of topics to which we should subscribe.
+      # @return [false, Array<String>] names of topics to which we should subscribe or false when
+      #   operating only on direct assignments
       #
       # @note Most of the time it should not include inactive topics but in case of pattern
       #   matching the matcher topics become inactive down the road, hence we filter out so
@@ -85,12 +86,32 @@ module Karafka
         topics.select(&:active?).map(&:subscription_name)
       end
 
+      # @param _consumer [Karafka::Connection::Proxy]
+      # @return [false, Rdkafka::Consumer::TopicPartitionList] List of tpls for direct assignments
+      #   or false for the normal mode
+      def assignments(_consumer)
+        false
+      end
+
       # @return [String] id of the subscription group
       # @note This is an alias for displaying in places where we print the stringified version.
       def to_s
         id
       end
 
+      # Refreshes the configuration of this subscription group if needed based on the execution
+      # context.
+      #
+      # Since the initial routing setup happens in the supervisor, it is inherited by the children.
+      # This causes incomplete assignment of `group.instance.id` which is not expanded with proper
+      # node identifier. This refreshes this if needed when in swarm.
+      def refresh
+        return unless node
+        return unless kafka.key?(:'group.instance.id')
+
+        @kafka = build_kafka
+      end
+
       private
 
       # @return [Hash] kafka settings are a bit special. They are exactly the same for all of the

data/lib/karafka/routing/topic.rb

@@ -18,7 +18,6 @@ module Karafka
       # Attributes we can inherit from the root unless they were defined on this level
       INHERITABLE_ATTRIBUTES = %i[
         kafka
-        deserializer
         max_messages
         max_wait_time
         initial_offset

data/lib/karafka/runner.rb

@@ -25,7 +25,7 @@ module Karafka
       # Register all the listeners so they can be started and managed
       @manager.register(listeners)
 
-      workers.each(&:async_call)
+      workers.each_with_index { |worker, i| worker.async_call("karafka.worker##{i}") }
 
       # We aggregate threads here for a supervised shutdown process
       Karafka::Server.workers = workers

data/lib/karafka/setup/config.rb

@@ -64,15 +64,9 @@ module Karafka
       setting :logger, default: ::Karafka::Instrumentation::Logger.new
       # option monitor [Instance] monitor that we will to use (defaults to Karafka::Monitor)
       setting :monitor, default: ::Karafka::Instrumentation::Monitor.new
-      # Mapper used to remap consumer groups ids, so in case users migrate from other tools
-      # or they need to maintain their own internal consumer group naming conventions, they
-      # can easily do it, replacing the default client_id + consumer name pattern concept
-      setting :consumer_mapper, default: Routing::ConsumerMapper.new
       # option [Boolean] should we reload consumers with each incoming batch thus effectively
       # supporting code reload (if someone reloads code) or should we keep the persistence
       setting :consumer_persistence, default: true
-      # Default deserializer for converting incoming data into ruby objects
-      setting :deserializer, default: Karafka::Serialization::Json::Deserializer.new
       # option [String] should we start with the earliest possible offset or latest
       # This will set the `auto.offset.reset` value unless present in the kafka scope
       setting :initial_offset, default: 'earliest'
@@ -100,6 +94,15 @@ module Karafka
       # Disabling this may be needed in scenarios where we do not have control over topics names
       # and/or we work with existing systems where we cannot change topics names.
       setting :strict_topics_namespacing, default: true
+      # option [String] default consumer group name for implicit routing
+      setting :group_id, default: 'app'
+
+      setting :oauth do
+        # option [false, #call] Listener for using oauth bearer. This listener will be able to
+        #   get the client name to decide whether to use a single multi-client token refreshing
+        #   or have separate tokens per instance.
+        setting :token_provider_listener, default: false
+      end
 
       # rdkafka default options
       # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
@@ -137,11 +140,12 @@ module Karafka
           # involving a consumer instance
           'enable.auto.commit': false,
           # Make sure that topic metadata lookups do not create topics accidentally
-          'allow.auto.create.topics': false
+          'allow.auto.create.topics': false,
+          # Do not store offsets automatically in admin in any way
+          'enable.auto.offset.store': false
         }
 
-        # option [String] default name for the admin consumer group. Please note, that this is a
-        # subject to be remapped by the consumer mapper as any other consumer group in the routes
+        # option [String] default name for the admin consumer group.
         setting :group_id, default: 'karafka_admin'
 
         # option max_wait_time [Integer] We wait only for this amount of time before raising error
@@ -196,7 +200,7 @@ module Karafka
           setting :liveness_listener, default: Swarm::LivenessListener.new
           # How long should we wait for any info from the node before we consider it hanging at
           # stop it
-          setting :node_report_timeout, default: 30_000
+          setting :node_report_timeout, default: 60_000
           # How long should we wait before restarting a node. This can prevent us from having a
           # case where for some external reason our spawned process would die immediately and we
           # would immediately try to start it back in an endless loop
@@ -230,6 +234,14 @@ module Karafka
 
           # Settings that are altered by our client proxy layer
           setting :proxy do
+            # commit offsets request
+            setting :commit do
+              # How many times should we try to run this call before raising an error
+              setting :max_attempts, default: 3
+              # How long should we wait before next attempt in case of a failure
+              setting :wait_time, default: 1_000
+            end
+
             # Committed offsets for given CG query
             setting :committed do
               # timeout for this request. For busy or remote clusters, this should be high enough
@@ -259,6 +271,26 @@ module Karafka
               # How long should we wait before next attempt in case of a failure
               setting :wait_time, default: 1_000
             end
+
+            # Settings for lag request
+            setting :lag do
+              # timeout for this request. For busy or remote clusters, this should be high enough
+              setting :timeout, default: 10_000
+              # How many times should we try to run this call before raising an error
+              setting :max_attempts, default: 3
+              # How long should we wait before next attempt in case of a failure
+              setting :wait_time, default: 1_000
+            end
+
+            # Settings for metadata request
+            setting :metadata do
+              # timeout for this request. For busy or remote clusters, this should be high enough
+              setting :timeout, default: 10_000
+              # How many times should we try to run this call before raising an error
+              setting :max_attempts, default: 3
+              # How long should we wait before next attempt in case of a failure
+              setting :wait_time, default: 1_000
+            end
           end
         end
 
@@ -368,10 +400,19 @@ module Karafka
         # Sets up all the components that are based on the user configuration
         # @note At the moment it is only WaterDrop
         def configure_components
+          oauth_listener = config.oauth.token_provider_listener
+          # We need to subscribe the oauth listener here because we want it to be ready before
+          # any consumer/admin runs
+          Karafka::App.monitor.subscribe(oauth_listener) if oauth_listener
+
           config.producer ||= ::WaterDrop::Producer.new do |producer_config|
             # In some cases WaterDrop updates the config and we don't want our consumer config to
             # be polluted by those updates, that's why we copy
             producer_config.kafka = AttributesMap.producer(config.kafka.dup)
+            # We also propagate same listener to the default producer to make sure, that the
+            # listener for oauth is also automatically used by the producer. That way we don't
+            # have to configure it manually for the default producer
+            producer_config.oauth.token_provider_listener = oauth_listener
             producer_config.logger = config.logger
           end
         end

data/lib/karafka/status.rb

@@ -3,6 +3,11 @@
 module Karafka
   # App status monitor
   class Status
+    include Helpers::ConfigImporter.new(
+      monitor: %i[monitor],
+      conductor: %i[internal connection conductor]
+    )
+
     # Available states and their transitions.
     STATES = {
       initializing: :initialize!,
@@ -60,14 +65,8 @@ module Karafka
             # We skip as during this state we do not have yet a monitor
             return if initializing?
 
-            # We do not set conductor in the initializer because this status object is created
-            # before the configuration kicks in
-            # We need to signal conductor on each state change as those may be relevant to
-            # listeners operations
-            @conductor ||= Karafka::App.config.internal.connection.conductor
-            @conductor.signal
-
-            Karafka.monitor.instrument("app.#{state}")
+            conductor.signal
+            monitor.instrument("app.#{state}", caller: self)
           end
         end
       RUBY

data/lib/karafka/swarm/manager.rb

@@ -19,6 +19,13 @@ module Karafka
         node_restart_timeout: %i[internal swarm node_restart_timeout]
       )
 
+      # Status we issue when we decide to shutdown unresponsive node
+      # We use -1 because nodes are expected to report 0+ statuses and we can use negative numbers
+      # for non-node based statuses
+      NOT_RESPONDING_SHUTDOWN_STATUS = -1
+
+      private_constant :NOT_RESPONDING_SHUTDOWN_STATUS
+
       # @return [Array<Node>] All nodes that manager manages
       attr_reader :nodes
 
@@ -29,10 +36,10 @@ module Karafka
 
       # Starts all the expected nodes for the first time
       def start
-        pidfd = Pidfd.new(::Process.pid)
+        parent_pid = ::Process.pid
 
         @nodes = Array.new(nodes_count) do |i|
-          start_one Node.new(i, pidfd)
+          start_one Node.new(i, parent_pid)
         end
       end
 
@@ -148,7 +155,12 @@ module Karafka
         return true unless over?(statuses[:control], node_report_timeout)
 
         # Start the stopping procedure if the node stopped reporting frequently enough
-        monitor.instrument('swarm.manager.stopping', caller: self, node: node) do
+        monitor.instrument(
+          'swarm.manager.stopping',
+          caller: self,
+          node: node,
+          status: NOT_RESPONDING_SHUTDOWN_STATUS
+        ) do
           node.stop
           statuses[:stop] = monotonic_now
         end

data/lib/karafka/swarm/node.rb

@@ -30,10 +30,10 @@ module Karafka
       # @param id [Integer] number of the fork. Used for uniqueness setup for group client ids and
       #   other stuff where we need to know a unique reference of the fork in regards to the rest
       #   of them.
-      # @param parent_pidfd [Pidfd] parent pidfd for zombie fencing
-      def initialize(id, parent_pidfd)
+      # @param parent_pid [Integer] parent pid for zombie fencing
+      def initialize(id, parent_pid)
         @id = id
-        @parent_pidfd = parent_pidfd
+        @parent_pidfd = Pidfd.new(parent_pid)
       end
 
       # Starts a new fork and:

data/lib/karafka/swarm/pidfd.rb

@@ -72,17 +72,33 @@ module Karafka
       def alive?
         @pidfd_select ||= [@pidfd_io]
 
-        IO.select(@pidfd_select, nil, nil, 0).nil?
+        if @mutex.owned?
+          return false if @cleaned
+
+          IO.select(@pidfd_select, nil, nil, 0).nil?
+        else
+          @mutex.synchronize do
+            return false if @cleaned
+
+            IO.select(@pidfd_select, nil, nil, 0).nil?
+          end
+        end
       end
 
       # Cleans the zombie process
       # @note This should run **only** on processes that exited, otherwise will wait
       def cleanup
-        return if @cleaned
+        @mutex.synchronize do
+          return if @cleaned
 
-        waitid(P_PIDFD, @pidfd, nil, WEXITED)
+          waitid(P_PIDFD, @pidfd, nil, WEXITED)
 
-        @cleaned = true
+          @pidfd_io.close
+          @pidfd_select = nil
+          @pidfd_io = nil
+          @pidfd = nil
+          @cleaned = true
+        end
       end
 
       # Sends given signal to the process using its pidfd

data/lib/karafka/swarm/supervisor.rb

@@ -23,6 +23,15 @@ module Karafka
         process: %i[internal process]
       )
 
+      # How long extra should we wait on shutdown before forceful termination
+      # We add this time because we send signals and it always can take a bit of time for them
+      # to reach out nodes and be processed to start the shutdown flow. Because of that and
+      # because we always want to give all nodes all the time of `shutdown_timeout` they are
+      # expected to have, we add this just to compensate.
+      SHUTDOWN_GRACE_PERIOD = 1_000
+
+      private_constant :SHUTDOWN_GRACE_PERIOD
+
       def initialize
         @mutex = Mutex.new
         @queue = Processing::TimedQueue.new
@@ -30,14 +39,16 @@ module Karafka
 
       # Creates needed number of forks, installs signals and starts supervision
       def run
-        Karafka::App.warmup
-
-        manager.start
-
         # Close producer just in case. While it should not be used, we do not want even a
         # theoretical case since librdkafka is not thread-safe.
+        # We close it prior to forking just to make sure, there is no issue with initialized
+        # producer (should not be initialized but just in case)
         Karafka.producer.close
 
+        Karafka::App.warmup
+
+        manager.start
+
         process.on_sigint { stop }
         process.on_sigquit { stop }
         process.on_sigterm { stop }
@@ -68,7 +79,10 @@ module Karafka
           type: 'swarm.supervisor.error'
         )
 
-        @nodes.terminate
+        manager.terminate
+        manager.cleanup
+
+        raise e
       end
 
       private
@@ -100,10 +114,12 @@ module Karafka
 
         manager.stop
 
+        total_shutdown_timeout = shutdown_timeout + SHUTDOWN_GRACE_PERIOD
+
         # We check from time to time (for the timeout period) if all the threads finished
         # their work and if so, we can just return and normal shutdown process will take place
         # We divide it by 1000 because we use time in ms.
-        ((shutdown_timeout / 1_000) * (1 / supervision_sleep)).to_i.times do
+        ((total_shutdown_timeout / 1_000) * (1 / supervision_sleep)).to_i.times do
           if manager.stopped?
             manager.cleanup
             return
@@ -132,8 +148,9 @@ module Karafka
         # Cleanup the process table
         manager.cleanup
 
-        # exit! is not within the instrumentation as it would not trigger due to exit
-        Kernel.exit!(forceful_exit_code)
+        # We do not use `exit!` here similar to regular server because we do not have to worry
+        # about any librdkafka related hanging connections, etc
+        Kernel.exit(forceful_exit_code)
       ensure
         if initialized
           Karafka::App.stopped!

data/lib/karafka/templates/karafka.rb.erb

@@ -37,7 +37,14 @@ class KarafkaApp < Karafka::App
   # interested in logging events for certain environments. Since instrumentation
   # notifications add extra boilerplate, if you want to achieve max performance,
   # listen to only what you really need for given environment.
-  Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
+  Karafka.monitor.subscribe(
+    Karafka::Instrumentation::LoggerListener.new(
+      # Karafka, when the logger is set to info producers logs each time it polls data from an
+      # internal messages wueue. This can be extensive, so you can turn it off by setting below
+      # to false.
+      log_polling: true
+    )
+  )
   # Karafka.monitor.subscribe(Karafka::Instrumentation::ProctitleListener.new)
 
   # This logger prints the producer development info using the Karafka logger.
@@ -52,6 +59,26 @@ class KarafkaApp < Karafka::App
     )
   )
 
+  # You can subscribe to all consumer related errors and record/track then that way
+  #
+  # Karafka.monitor.subscribe 'error.occurred' do |event|
+  #   type = event[:type]
+  #   error = event[:error]
+  #   details = (error.backtrace || []).join("\n")
+  #   ErrorTracker.send_error(error, type, details)
+  # end
+
+  # You can subscribe to all producer related errors and record/track then that way
+  # Please note, that producer and consumer have their own notifications pipeline so you need to
+  # setup error tracking independently for each of them
+  #
+  # Karafka.producer.monitor.subscribe('error.occurred') do |event|
+  #   type = event[:type]
+  #   error = event[:error]
+  #   details = (error.backtrace || []).join("\n")
+  #   ErrorTracker.send_error(error, type, details)
+  # end
+
   routes.draw do
 <% if rails? -%>
     # Uncomment this if you use Karafka with ActiveJob

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.3.2'
+  VERSION = '2.4.0.beta1'
 end