karafka 2.3.1 → 2.3.3

data/lib/karafka/status.rb

@@ -7,6 +7,7 @@ module Karafka
     STATES = {
       initializing: :initialize!,
       initialized: :initialized!,
+      supervising: :supervise!,
       running: :run!,
       # will no longer pickup any work, but current work will be finished
       quieting: :quiet!,
@@ -49,8 +50,8 @@ module Karafka
 
         def #{transition}
           MUTEX.synchronize do
-            # Do not allow reverse state transitions (we always go one way) or transition to the same
-            # state as currently
+            # Do not allow reverse state transitions (we always go one way) or transition to the
+            # same state as currently
             return if @status && STATES.keys.index(:#{state}) <= STATES.keys.index(@status)
 
             @status = :#{state}
@@ -78,6 +79,7 @@ module Karafka
     def done?
       # Short-track for the most common case not to invoke all others on normal execution
       return false if running?
+      return false if supervising?
 
       stopping? || stopped? || quieting? || quiet? || terminated?
     end

data/lib/karafka/swarm/liveness_listener.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Swarm
+    # Simple listener for swarm nodes that:
+    #   - reports once in a while to make sure that supervisor is aware we do not hang
+    #   - makes sure we did not become an orphan and if so, exits
+    class LivenessListener
+      include Karafka::Core::Helpers::Time
+      include Helpers::ConfigImporter.new(
+        node: %i[swarm node],
+        liveness_interval: %i[internal swarm liveness_interval],
+        orphaned_exit_code: %i[internal swarm orphaned_exit_code]
+      )
+
+      def initialize
+        @last_checked_at = 0
+        @mutex = Mutex.new
+      end
+
+      # Since there may be many statistics emitted from multiple listeners, we do not want to write
+      # statuses that often. Instead we do it only once in a while which should be enough
+      #
+      # While this may provide a small lag in the orphaned detection, it does not really matter
+      # as it will be picked up fast enough.
+      # @param _event [Karafka::Core::Monitoring::Event]
+      def on_statistics_emitted(_event)
+        periodically do
+          Kernel.exit!(orphaned_exit_code) if node.orphaned?
+
+          node.healthy
+        end
+      end
+
+      private
+
+      # Wraps the logic with a mutex
+      # @param block [Proc] code we want to run in mutex
+      def synchronize(&block)
+        @mutex.synchronize(&block)
+      end
+
+      # Runs requested code once in a while
+      def periodically
+        return if monotonic_now - @last_checked_at < liveness_interval
+
+        synchronize do
+          @last_checked_at = monotonic_now
+
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/karafka/swarm/manager.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Swarm
+    # Manager similar to the one for threads but managing processing nodes
+    # It starts nodes and keeps an eye on them.
+    #
+    # In any of the nodes is misbehaving (based on liveness listener) it will be restarted.
+    # Initially gracefully but if won't stop itself, it will be forced to.
+    #
+    # @note This is intended to run in the supervisor under mutexes (when needed)
+    class Manager
+      include Karafka::Core::Helpers::Time
+      include Helpers::ConfigImporter.new(
+        monitor: %i[monitor],
+        nodes_count: %i[swarm nodes],
+        shutdown_timeout: %i[shutdown_timeout],
+        node_report_timeout: %i[internal swarm node_report_timeout],
+        node_restart_timeout: %i[internal swarm node_restart_timeout]
+      )
+
+      # Status we issue when we decide to shutdown unresponsive node
+      # We use -1 because nodes are expected to report 0+ statuses and we can use negative numbers
+      # for non-node based statuses
+      NOT_RESPONDING_SHUTDOWN_STATUS = -1
+
+      private_constant :NOT_RESPONDING_SHUTDOWN_STATUS
+
+      # @return [Array<Node>] All nodes that manager manages
+      attr_reader :nodes
+
+      def initialize
+        @nodes = []
+        @statuses = Hash.new { |h, k| h[k] = {} }
+      end
+
+      # Starts all the expected nodes for the first time
+      def start
+        parent_pid = ::Process.pid
+
+        @nodes = Array.new(nodes_count) do |i|
+          start_one Node.new(i, parent_pid)
+        end
+      end
+
+      # Attempts to quiet all the nodes
+      def quiet
+        @nodes.each(&:quiet)
+      end
+
+      # Attempts to stop all the nodes
+      def stop
+        @nodes.each(&:stop)
+      end
+
+      # Terminates all the nodes
+      def terminate
+        @nodes.each(&:terminate)
+      end
+
+      # Collects all processes statuses
+      def cleanup
+        @nodes.each(&:cleanup)
+      end
+
+      # Sends given signal to all nodes
+      # @param signal [String] signal name
+      def signal(signal)
+        @nodes.each { |node| node.signal(signal) }
+      end
+
+      # @return [Boolean] true if none of the nodes is running
+      def stopped?
+        @nodes.none?(&:alive?)
+      end
+
+      # Checks on nodes if they are ok one after another
+      def control
+        monitor.instrument('swarm.manager.control', caller: self) do
+          @nodes.each do |node|
+            statuses = @statuses[node]
+
+            if node.alive?
+              next if terminate_if_hanging(statuses, node)
+              next if stop_if_not_healthy(statuses, node)
+              next if stop_if_not_responding(statuses, node)
+            else
+              next if cleanup_one(statuses, node)
+              next if restart_after_timeout(statuses, node)
+            end
+          end
+        end
+      end
+
+      private
+
+      # If we've issued a stop to this process and it does not want to stop in the period, kills it
+      #
+      # @param statuses [Hash] hash with statuses transitions with times
+      # @param [Swarm::Node] node we're checking
+      # @return [Boolean] should it be the last action taken on this node in this run
+      def terminate_if_hanging(statuses, node)
+        return false unless statuses.key?(:stop)
+        # If we already sent the termination request, we should not do it again
+        return true if statuses.key?(:terminate)
+        # Do not run any other checks on this node if it is during stopping but still has time
+        return true unless over?(statuses[:stop], shutdown_timeout)
+
+        monitor.instrument('swarm.manager.terminating', caller: self, node: node) do
+          node.terminate
+          statuses[:terminate] = monotonic_now
+        end
+
+        true
+      end
+
+      # Checks if there is any new liveness report from given node and if yes, issues stop if it
+      # reported it is not healthy.
+      #
+      # @param statuses [Hash] hash with statuses transitions with times
+      # @param [Swarm::Node] node we're checking
+      # @return [Boolean] should it be the last action taken on this node in this run
+      def stop_if_not_healthy(statuses, node)
+        status = node.status
+
+        case status
+        # If no new state reported, we should just move with other checks
+        when -1
+          false
+        when 0
+          # Exists and reports as healthy, so no other checks should happen on it in this go
+          statuses[:control] = monotonic_now
+          true
+        else
+          # A single invalid report will cause it to stop. We do not support intermediate failures
+          # that would recover. Such states should be implemented in the listener.
+          monitor.instrument('swarm.manager.stopping', caller: self, node: node, status: status) do
+            node.stop
+            statuses[:stop] = monotonic_now
+          end
+
+          true
+        end
+      end
+
+      # If node stopped responding, starts the stopping procedure.
+      #
+      # @param statuses [Hash] hash with statuses transitions with times
+      # @param [Swarm::Node] node we're checking
+      # @return [Boolean] should it be the last action taken on this node in this run
+      def stop_if_not_responding(statuses, node)
+        # Do nothing if already stopping
+        return true if statuses.key?(:stop)
+        # Do nothing if we've received status update recently enough
+        return true unless over?(statuses[:control], node_report_timeout)
+
+        # Start the stopping procedure if the node stopped reporting frequently enough
+        monitor.instrument(
+          'swarm.manager.stopping',
+          caller: self,
+          node: node,
+          status: NOT_RESPONDING_SHUTDOWN_STATUS
+        ) do
+          node.stop
+          statuses[:stop] = monotonic_now
+        end
+
+        true
+      end
+
+      # Cleans up a dead process and remembers time of death for restart after a period.
+      #
+      # @param statuses [Hash] hash with statuses transitions with times
+      # @param [Swarm::Node] node we're checking
+      # @return [Boolean] should it be the last action taken on this node in this run
+      def cleanup_one(statuses, node)
+        return false if statuses.key?(:dead_since)
+
+        node.cleanup
+        statuses[:dead_since] = monotonic_now
+
+        true
+      end
+
+      # Restarts the node if there was enough of a backoff.
+      #
+      # We always wait a bit to make sure, we do not overload the system in case forks would be
+      # killed for some external reason.
+      #
+      # @param statuses [Hash] hash with statuses transitions with times
+      # @param [Swarm::Node] node we're checking
+      # @return [Boolean] should it be the last action taken on this node in this run
+      def restart_after_timeout(statuses, node)
+        return false unless over?(statuses[:dead_since], node_restart_timeout)
+
+        start_one(node)
+
+        true
+      end
+
+      # Starts a new node (or restarts dead)
+      #
+      # @param [Swarm::Node] node we're starting
+      def start_one(node)
+        instr_args = { caller: self, node: node }
+
+        statuses = @statuses[node]
+
+        statuses.clear
+        statuses[:control] = monotonic_now
+
+        monitor.instrument('swarm.manager.before_fork', instr_args)
+        node.start
+        monitor.instrument('swarm.manager.after_fork', instr_args)
+
+        node
+      end
+
+      # Are we over certain time from an event happening
+      #
+      # @param event_time [Float] when something happened
+      # @param delay [Float] how long should we wait
+      # @return [Boolean] true if we're past the delay
+      def over?(event_time, delay)
+        monotonic_now - event_time >= delay
+      end
+    end
+  end
+end

data/lib/karafka/swarm/node.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Swarm
+    # Represents a single forked process node in a swarm
+    # Provides simple API to control forks and check their status
+    #
+    # @note Some of this APIs are for parent process only
+    #
+    # @note Keep in mind this can be used in both forks and supervisor and has a slightly different
+    #   role in each. In case of the supervisor it is used to get information about the child and
+    #   make certain requests to it. In case of child, it is used to provide zombie-fencing and
+    #   report liveness
+    class Node
+      include Helpers::ConfigImporter.new(
+        monitor: %i[monitor],
+        config: %i[itself],
+        kafka: %i[kafka],
+        swarm: %i[swarm],
+        process: %i[process],
+        liveness_listener: %i[internal swarm liveness_listener]
+      )
+
+      # @return [Integer] id of the node. Useful for client.group.id assignment
+      attr_reader :id
+
+      # @return [Integer] pid of the node
+      attr_reader :pid
+
+      # @param id [Integer] number of the fork. Used for uniqueness setup for group client ids and
+      #   other stuff where we need to know a unique reference of the fork in regards to the rest
+      #   of them.
+      # @param parent_pid [Integer] parent pid for zombie fencing
+      def initialize(id, parent_pid)
+        @id = id
+        @parent_pidfd = Pidfd.new(parent_pid)
+      end
+
+      # Starts a new fork and:
+      #   - stores pid and parent reference
+      #   - makes sure reader pipe is closed
+      #   - sets up liveness listener
+      #   - recreates producer and web producer
+      # @note Parent API
+      def start
+        @reader, @writer = IO.pipe
+
+        # :nocov:
+        @pid = ::Process.fork do
+          # Close the old producer so it is not a subject to GC
+          # While it was not opened in the parent, without explicit closing, there still could be
+          # an attempt to close it when finalized, meaning it would be kept in memory.
+          config.producer.close
+
+          # Supervisor producer is closed, hence we need a new one here
+          config.producer = ::WaterDrop::Producer.new do |p_config|
+            p_config.kafka = Setup::AttributesMap.producer(kafka.dup)
+            p_config.logger = config.logger
+          end
+
+          @pid = ::Process.pid
+          @reader.close
+
+          # Indicate we are alive right after start
+          healthy
+
+          swarm.node = self
+          monitor.subscribe(liveness_listener)
+          monitor.instrument('swarm.node.after_fork', caller: self)
+
+          Server.run
+
+          @writer.close
+        end
+        # :nocov:
+
+        @writer.close
+        @pidfd = Pidfd.new(@pid)
+      end
+
+      # Indicates that this node is doing well
+      # @note Child API
+      def healthy
+        write('0')
+      end
+
+      # Indicates, that this node has failed
+      # @param reason_code [Integer, String] numeric code we want to use to indicate that we are
+      #   not healthy. Anything bigger than 0 will be considered not healthy. Useful it we want to
+      #   have complex health-checking with reporting.
+      # @note Child API
+      # @note We convert this to string to normalize the API
+      def unhealthy(reason_code = '1')
+        write(reason_code.to_s)
+      end
+
+      # @return [Integer] This returns following status code depending on the data:
+      #   - -1 if node did not report anything new
+      #   - 0 if all good,
+      #   - positive number if there was a problem (indicates error code)
+      #
+      # @note Parent API
+      # @note If there were few issues reported, it will pick the one with highest number
+      def status
+        result = read
+
+        return -1 if result.nil?
+        return -1 if result == false
+
+        result.split("\n").map(&:to_i).max
+      end
+
+      # @return [Boolean] true if node is alive or false if died
+      # @note Parent API
+      # @note Keep in mind that the fact that process is alive does not mean it is healthy
+      def alive?
+        @pidfd.alive?
+      end
+
+      # @return [Boolean] true if node is orphaned or false otherwise. Used for orphans detection.
+      # @note Child API
+      def orphaned?
+        !@parent_pidfd.alive?
+      end
+
+      # Sends sigterm to the node
+      # @note Parent API
+      def stop
+        signal('TERM')
+      end
+
+      # Sends sigtstp to the node
+      # @note Parent API
+      def quiet
+        signal('TSTP')
+      end
+
+      # Terminates node
+      # @note Parent API
+      def terminate
+        signal('KILL')
+      end
+
+      # Sends provided signal to the node
+      # @param signal [String]
+      def signal(signal)
+        @pidfd.signal(signal)
+      end
+
+      # Removes the dead process from the processes table
+      def cleanup
+        @pidfd.cleanup
+      end
+
+      private
+
+      # Reads in a non-blocking way provided content
+      # @return [String, false] Content from the pipe or false if nothing or something went wrong
+      # @note Parent API
+      def read
+        @reader.read_nonblock(1024)
+      rescue IO::WaitReadable, Errno::EPIPE, IOError
+        false
+      end
+
+      # Writes in a non-blocking way provided content into the pipe
+      # @param content [Integer, String] anything we want to write to the parent
+      # @return [Boolean] true if ok, otherwise false
+      # @note Child API
+      def write(content)
+        @writer.write_nonblock "#{content}\n"
+
+        true
+      rescue IO::WaitWritable, Errno::EPIPE, IOError
+        false
+      end
+    end
+  end
+end

data/lib/karafka/swarm/pidfd.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Swarm
+    # Pidfd Linux representation wrapped with Ruby for communication within Swarm
+    # It is more stable than using `#pid` and `#ppid` + signals and cheaper
+    class Pidfd
+      include Helpers::ConfigImporter.new(
+        pidfd_open_syscall: %i[internal swarm pidfd_open_syscall],
+        pidfd_signal_syscall: %i[internal swarm pidfd_signal_syscall],
+        waitid_syscall: %i[internal swarm waitid_syscall]
+      )
+
+      extend FFI::Library
+
+      begin
+        ffi_lib FFI::Library::LIBC
+
+        # direct usage of this is only available since glibc 2.36, hence we use bindings and call
+        # it directly via syscalls
+        attach_function :fdpid_open, :syscall, %i[long int uint], :int
+        attach_function :fdpid_signal, :syscall, %i[long int int pointer uint], :int
+        attach_function :waitid, %i[int int pointer uint], :int
+
+        API_SUPPORTED = true
+      # LoadError is a parent to FFI::NotFoundError
+      rescue LoadError
+        API_SUPPORTED = false
+      ensure
+        private_constant :API_SUPPORTED
+      end
+
+      # https://github.com/torvalds/linux/blob/7e90b5c295/include/uapi/linux/wait.h#L20
+      P_PIDFD = 3
+
+      # Wait for child processes that have exited
+      WEXITED = 4
+
+      private_constant :P_PIDFD, :WEXITED
+
+      class << self
+        # @return [Boolean] true if syscall is supported via FFI
+        def supported?
+          # If we were not even able to load the FFI C lib, it won't be supported
+          return false unless API_SUPPORTED
+          # Won't work on macOS because it does not support pidfd
+          return false if RUBY_DESCRIPTION.include?('darwin')
+          # Won't work on Windows for the same reason as on macOS
+          return false if RUBY_DESCRIPTION.match?(/mswin|ming|cygwin/)
+
+          # There are some OSes like BSD that will have C lib for FFI bindings but will not support
+          # the needed syscalls. In such cases, we can just try and fail, which will indicate it
+          # won't work. The same applies to using new glibc on an old kernel.
+          new(::Process.pid)
+
+          true
+        rescue Errors::PidfdOpenFailedError
+          false
+        end
+      end
+
+      # @param pid [Integer] pid of the node we want to work with
+      def initialize(pid)
+        @mutex = Mutex.new
+
+        @pid = pid
+        @pidfd = open(pid)
+        @pidfd_io = IO.new(@pidfd)
+      end
+
+      # @return [Boolean] true if given process is alive, false if no longer
+      def alive?
+        @pidfd_select ||= [@pidfd_io]
+
+        if @mutex.owned?
+          return false if @cleaned
+
+          IO.select(@pidfd_select, nil, nil, 0).nil?
+        else
+          @mutex.synchronize do
+            return false if @cleaned
+
+            IO.select(@pidfd_select, nil, nil, 0).nil?
+          end
+        end
+      end
+
+      # Cleans the zombie process
+      # @note This should run **only** on processes that exited, otherwise will wait
+      def cleanup
+        @mutex.synchronize do
+          return if @cleaned
+
+          waitid(P_PIDFD, @pidfd, nil, WEXITED)
+
+          @pidfd_io.close
+          @pidfd_select = nil
+          @pidfd_io = nil
+          @pidfd = nil
+          @cleaned = true
+        end
+      end
+
+      # Sends given signal to the process using its pidfd
+      # @param sig_name [String] signal name
+      # @return [Boolean] true if signal was sent, otherwise false or error raised. `false`
+      #   returned when we attempt to send a signal to a dead process
+      # @note It will not send signals to dead processes
+      def signal(sig_name)
+        @mutex.synchronize do
+          return false if @cleaned
+          # Never signal processes that are dead
+          return false unless alive?
+
+          result = fdpid_signal(
+            pidfd_signal_syscall,
+            @pidfd,
+            Signal.list.fetch(sig_name),
+            nil,
+            0
+          )
+
+          return true if result.zero?
+
+          raise Errors::PidfdSignalFailedError, result
+        end
+      end
+
+      private
+
+      # Opens a pidfd for the provided pid
+      # @param pid [Integer]
+      # @return [Integer] pidfd
+      def open(pid)
+        pidfd = fdpid_open(
+          pidfd_open_syscall,
+          pid,
+          0
+        )
+
+        return pidfd if pidfd != -1
+
+        raise Errors::PidfdOpenFailedError, pidfd
+      end
+    end
+  end
+end