karafka 2.3.0 → 2.3.2

data/lib/karafka/swarm/manager.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Swarm
+    # Manager similar to the one for threads but managing processing nodes
+    # It starts nodes and keeps an eye on them.
+    #
+    # In any of the nodes is misbehaving (based on liveness listener) it will be restarted.
+    # Initially gracefully but if won't stop itself, it will be forced to.
+    #
+    # @note This is intended to run in the supervisor under mutexes (when needed)
+    class Manager
+      include Karafka::Core::Helpers::Time
+      include Helpers::ConfigImporter.new(
+        monitor: %i[monitor],
+        nodes_count: %i[swarm nodes],
+        shutdown_timeout: %i[shutdown_timeout],
+        node_report_timeout: %i[internal swarm node_report_timeout],
+        node_restart_timeout: %i[internal swarm node_restart_timeout]
+      )
+
+      # @return [Array<Node>] All nodes that manager manages
+      attr_reader :nodes
+
+      def initialize
+        @nodes = []
+        @statuses = Hash.new { |h, k| h[k] = {} }
+      end
+
+      # Starts all the expected nodes for the first time
+      def start
+        pidfd = Pidfd.new(::Process.pid)
+
+        @nodes = Array.new(nodes_count) do |i|
+          start_one Node.new(i, pidfd)
+        end
+      end
+
+      # Attempts to quiet all the nodes
+      def quiet
+        @nodes.each(&:quiet)
+      end
+
+      # Attempts to stop all the nodes
+      def stop
+        @nodes.each(&:stop)
+      end
+
+      # Terminates all the nodes
+      def terminate
+        @nodes.each(&:terminate)
+      end
+
+      # Collects all processes statuses
+      def cleanup
+        @nodes.each(&:cleanup)
+      end
+
+      # Sends given signal to all nodes
+      # @param signal [String] signal name
+      def signal(signal)
+        @nodes.each { |node| node.signal(signal) }
+      end
+
+      # @return [Boolean] true if none of the nodes is running
+      def stopped?
+        @nodes.none?(&:alive?)
+      end
+
+      # Checks on nodes if they are ok one after another
+      def control
+        monitor.instrument('swarm.manager.control', caller: self) do
+          @nodes.each do |node|
+            statuses = @statuses[node]
+
+            if node.alive?
+              next if terminate_if_hanging(statuses, node)
+              next if stop_if_not_healthy(statuses, node)
+              next if stop_if_not_responding(statuses, node)
+            else
+              next if cleanup_one(statuses, node)
+              next if restart_after_timeout(statuses, node)
+            end
+          end
+        end
+      end
+
+      private
+
+      # If we've issued a stop to this process and it does not want to stop in the period, kills it
+      #
+      # @param statuses [Hash] hash with statuses transitions with times
+      # @param [Swarm::Node] node we're checking
+      # @return [Boolean] should it be the last action taken on this node in this run
+      def terminate_if_hanging(statuses, node)
+        return false unless statuses.key?(:stop)
+        # If we already sent the termination request, we should not do it again
+        return true if statuses.key?(:terminate)
+        # Do not run any other checks on this node if it is during stopping but still has time
+        return true unless over?(statuses[:stop], shutdown_timeout)
+
+        monitor.instrument('swarm.manager.terminating', caller: self, node: node) do
+          node.terminate
+          statuses[:terminate] = monotonic_now
+        end
+
+        true
+      end
+
+      # Checks if there is any new liveness report from given node and if yes, issues stop if it
+      # reported it is not healthy.
+      #
+      # @param statuses [Hash] hash with statuses transitions with times
+      # @param [Swarm::Node] node we're checking
+      # @return [Boolean] should it be the last action taken on this node in this run
+      def stop_if_not_healthy(statuses, node)
+        status = node.status
+
+        case status
+        # If no new state reported, we should just move with other checks
+        when -1
+          false
+        when 0
+          # Exists and reports as healthy, so no other checks should happen on it in this go
+          statuses[:control] = monotonic_now
+          true
+        else
+          # A single invalid report will cause it to stop. We do not support intermediate failures
+          # that would recover. Such states should be implemented in the listener.
+          monitor.instrument('swarm.manager.stopping', caller: self, node: node, status: status) do
+            node.stop
+            statuses[:stop] = monotonic_now
+          end
+
+          true
+        end
+      end
+
+      # If node stopped responding, starts the stopping procedure.
+      #
+      # @param statuses [Hash] hash with statuses transitions with times
+      # @param [Swarm::Node] node we're checking
+      # @return [Boolean] should it be the last action taken on this node in this run
+      def stop_if_not_responding(statuses, node)
+        # Do nothing if already stopping
+        return true if statuses.key?(:stop)
+        # Do nothing if we've received status update recently enough
+        return true unless over?(statuses[:control], node_report_timeout)
+
+        # Start the stopping procedure if the node stopped reporting frequently enough
+        monitor.instrument('swarm.manager.stopping', caller: self, node: node) do
+          node.stop
+          statuses[:stop] = monotonic_now
+        end
+
+        true
+      end
+
+      # Cleans up a dead process and remembers time of death for restart after a period.
+      #
+      # @param statuses [Hash] hash with statuses transitions with times
+      # @param [Swarm::Node] node we're checking
+      # @return [Boolean] should it be the last action taken on this node in this run
+      def cleanup_one(statuses, node)
+        return false if statuses.key?(:dead_since)
+
+        node.cleanup
+        statuses[:dead_since] = monotonic_now
+
+        true
+      end
+
+      # Restarts the node if there was enough of a backoff.
+      #
+      # We always wait a bit to make sure, we do not overload the system in case forks would be
+      # killed for some external reason.
+      #
+      # @param statuses [Hash] hash with statuses transitions with times
+      # @param [Swarm::Node] node we're checking
+      # @return [Boolean] should it be the last action taken on this node in this run
+      def restart_after_timeout(statuses, node)
+        return false unless over?(statuses[:dead_since], node_restart_timeout)
+
+        start_one(node)
+
+        true
+      end
+
+      # Starts a new node (or restarts dead)
+      #
+      # @param [Swarm::Node] node we're starting
+      def start_one(node)
+        instr_args = { caller: self, node: node }
+
+        statuses = @statuses[node]
+
+        statuses.clear
+        statuses[:control] = monotonic_now
+
+        monitor.instrument('swarm.manager.before_fork', instr_args)
+        node.start
+        monitor.instrument('swarm.manager.after_fork', instr_args)
+
+        node
+      end
+
+      # Are we over certain time from an event happening
+      #
+      # @param event_time [Float] when something happened
+      # @param delay [Float] how long should we wait
+      # @return [Boolean] true if we're past the delay
+      def over?(event_time, delay)
+        monotonic_now - event_time >= delay
+      end
+    end
+  end
+end

data/lib/karafka/swarm/node.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Swarm
+    # Represents a single forked process node in a swarm
+    # Provides simple API to control forks and check their status
+    #
+    # @note Some of this APIs are for parent process only
+    #
+    # @note Keep in mind this can be used in both forks and supervisor and has a slightly different
+    #   role in each. In case of the supervisor it is used to get information about the child and
+    #   make certain requests to it. In case of child, it is used to provide zombie-fencing and
+    #   report liveness
+    class Node
+      include Helpers::ConfigImporter.new(
+        monitor: %i[monitor],
+        config: %i[itself],
+        kafka: %i[kafka],
+        swarm: %i[swarm],
+        process: %i[process],
+        liveness_listener: %i[internal swarm liveness_listener]
+      )
+
+      # @return [Integer] id of the node. Useful for client.group.id assignment
+      attr_reader :id
+
+      # @return [Integer] pid of the node
+      attr_reader :pid
+
+      # @param id [Integer] number of the fork. Used for uniqueness setup for group client ids and
+      #   other stuff where we need to know a unique reference of the fork in regards to the rest
+      #   of them.
+      # @param parent_pidfd [Pidfd] parent pidfd for zombie fencing
+      def initialize(id, parent_pidfd)
+        @id = id
+        @parent_pidfd = parent_pidfd
+      end
+
+      # Starts a new fork and:
+      #   - stores pid and parent reference
+      #   - makes sure reader pipe is closed
+      #   - sets up liveness listener
+      #   - recreates producer and web producer
+      # @note Parent API
+      def start
+        @reader, @writer = IO.pipe
+
+        # :nocov:
+        @pid = ::Process.fork do
+          # Close the old producer so it is not a subject to GC
+          # While it was not opened in the parent, without explicit closing, there still could be
+          # an attempt to close it when finalized, meaning it would be kept in memory.
+          config.producer.close
+
+          # Supervisor producer is closed, hence we need a new one here
+          config.producer = ::WaterDrop::Producer.new do |p_config|
+            p_config.kafka = Setup::AttributesMap.producer(kafka.dup)
+            p_config.logger = config.logger
+          end
+
+          @pid = ::Process.pid
+          @reader.close
+
+          # Indicate we are alive right after start
+          healthy
+
+          swarm.node = self
+          monitor.subscribe(liveness_listener)
+          monitor.instrument('swarm.node.after_fork', caller: self)
+
+          Server.run
+
+          @writer.close
+        end
+        # :nocov:
+
+        @writer.close
+        @pidfd = Pidfd.new(@pid)
+      end
+
+      # Indicates that this node is doing well
+      # @note Child API
+      def healthy
+        write('0')
+      end
+
+      # Indicates, that this node has failed
+      # @param reason_code [Integer, String] numeric code we want to use to indicate that we are
+      #   not healthy. Anything bigger than 0 will be considered not healthy. Useful it we want to
+      #   have complex health-checking with reporting.
+      # @note Child API
+      # @note We convert this to string to normalize the API
+      def unhealthy(reason_code = '1')
+        write(reason_code.to_s)
+      end
+
+      # @return [Integer] This returns following status code depending on the data:
+      #   - -1 if node did not report anything new
+      #   - 0 if all good,
+      #   - positive number if there was a problem (indicates error code)
+      #
+      # @note Parent API
+      # @note If there were few issues reported, it will pick the one with highest number
+      def status
+        result = read
+
+        return -1 if result.nil?
+        return -1 if result == false
+
+        result.split("\n").map(&:to_i).max
+      end
+
+      # @return [Boolean] true if node is alive or false if died
+      # @note Parent API
+      # @note Keep in mind that the fact that process is alive does not mean it is healthy
+      def alive?
+        @pidfd.alive?
+      end
+
+      # @return [Boolean] true if node is orphaned or false otherwise. Used for orphans detection.
+      # @note Child API
+      def orphaned?
+        !@parent_pidfd.alive?
+      end
+
+      # Sends sigterm to the node
+      # @note Parent API
+      def stop
+        signal('TERM')
+      end
+
+      # Sends sigtstp to the node
+      # @note Parent API
+      def quiet
+        signal('TSTP')
+      end
+
+      # Terminates node
+      # @note Parent API
+      def terminate
+        signal('KILL')
+      end
+
+      # Sends provided signal to the node
+      # @param signal [String]
+      def signal(signal)
+        @pidfd.signal(signal)
+      end
+
+      # Removes the dead process from the processes table
+      def cleanup
+        @pidfd.cleanup
+      end
+
+      private
+
+      # Reads in a non-blocking way provided content
+      # @return [String, false] Content from the pipe or false if nothing or something went wrong
+      # @note Parent API
+      def read
+        @reader.read_nonblock(1024)
+      rescue IO::WaitReadable, Errno::EPIPE, IOError
+        false
+      end
+
+      # Writes in a non-blocking way provided content into the pipe
+      # @param content [Integer, String] anything we want to write to the parent
+      # @return [Boolean] true if ok, otherwise false
+      # @note Child API
+      def write(content)
+        @writer.write_nonblock "#{content}\n"
+
+        true
+      rescue IO::WaitWritable, Errno::EPIPE, IOError
+        false
+      end
+    end
+  end
+end

data/lib/karafka/swarm/pidfd.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Swarm
+    # Pidfd Linux representation wrapped with Ruby for communication within Swarm
+    # It is more stable than using `#pid` and `#ppid` + signals and cheaper
+    class Pidfd
+      include Helpers::ConfigImporter.new(
+        pidfd_open_syscall: %i[internal swarm pidfd_open_syscall],
+        pidfd_signal_syscall: %i[internal swarm pidfd_signal_syscall],
+        waitid_syscall: %i[internal swarm waitid_syscall]
+      )
+
+      extend FFI::Library
+
+      begin
+        ffi_lib FFI::Library::LIBC
+
+        # direct usage of this is only available since glibc 2.36, hence we use bindings and call
+        # it directly via syscalls
+        attach_function :fdpid_open, :syscall, %i[long int uint], :int
+        attach_function :fdpid_signal, :syscall, %i[long int int pointer uint], :int
+        attach_function :waitid, %i[int int pointer uint], :int
+
+        API_SUPPORTED = true
+      # LoadError is a parent to FFI::NotFoundError
+      rescue LoadError
+        API_SUPPORTED = false
+      ensure
+        private_constant :API_SUPPORTED
+      end
+
+      # https://github.com/torvalds/linux/blob/7e90b5c295/include/uapi/linux/wait.h#L20
+      P_PIDFD = 3
+
+      # Wait for child processes that have exited
+      WEXITED = 4
+
+      private_constant :P_PIDFD, :WEXITED
+
+      class << self
+        # @return [Boolean] true if syscall is supported via FFI
+        def supported?
+          # If we were not even able to load the FFI C lib, it won't be supported
+          return false unless API_SUPPORTED
+          # Won't work on macOS because it does not support pidfd
+          return false if RUBY_DESCRIPTION.include?('darwin')
+          # Won't work on Windows for the same reason as on macOS
+          return false if RUBY_DESCRIPTION.match?(/mswin|ming|cygwin/)
+
+          # There are some OSes like BSD that will have C lib for FFI bindings but will not support
+          # the needed syscalls. In such cases, we can just try and fail, which will indicate it
+          # won't work. The same applies to using new glibc on an old kernel.
+          new(::Process.pid)
+
+          true
+        rescue Errors::PidfdOpenFailedError
+          false
+        end
+      end
+
+      # @param pid [Integer] pid of the node we want to work with
+      def initialize(pid)
+        @mutex = Mutex.new
+
+        @pid = pid
+        @pidfd = open(pid)
+        @pidfd_io = IO.new(@pidfd)
+      end
+
+      # @return [Boolean] true if given process is alive, false if no longer
+      def alive?
+        @pidfd_select ||= [@pidfd_io]
+
+        IO.select(@pidfd_select, nil, nil, 0).nil?
+      end
+
+      # Cleans the zombie process
+      # @note This should run **only** on processes that exited, otherwise will wait
+      def cleanup
+        return if @cleaned
+
+        waitid(P_PIDFD, @pidfd, nil, WEXITED)
+
+        @cleaned = true
+      end
+
+      # Sends given signal to the process using its pidfd
+      # @param sig_name [String] signal name
+      # @return [Boolean] true if signal was sent, otherwise false or error raised. `false`
+      #   returned when we attempt to send a signal to a dead process
+      # @note It will not send signals to dead processes
+      def signal(sig_name)
+        @mutex.synchronize do
+          return false if @cleaned
+          # Never signal processes that are dead
+          return false unless alive?
+
+          result = fdpid_signal(
+            pidfd_signal_syscall,
+            @pidfd,
+            Signal.list.fetch(sig_name),
+            nil,
+            0
+          )
+
+          return true if result.zero?
+
+          raise Errors::PidfdSignalFailedError, result
+        end
+      end
+
+      private
+
+      # Opens a pidfd for the provided pid
+      # @param pid [Integer]
+      # @return [Integer] pidfd
+      def open(pid)
+        pidfd = fdpid_open(
+          pidfd_open_syscall,
+          pid,
+          0
+        )
+
+        return pidfd if pidfd != -1
+
+        raise Errors::PidfdOpenFailedError, pidfd
+      end
+    end
+  end
+end

data/lib/karafka/swarm/supervisor.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Swarm
+    # Supervisor that starts forks and uses monitor to monitor them. Also handles shutdown of
+    # all the processes including itself.
+    #
+    # In case any node dies, it will be restarted.
+    #
+    # @note Technically speaking supervisor is never in the running state because we do not want
+    #   to have any sockets or anything else on it that could break under forking.
+    #   It has its own "supervising" state from which it can go to the final shutdown.
+    class Supervisor
+      include Karafka::Core::Helpers::Time
+      include Helpers::ConfigImporter.new(
+        monitor: %i[monitor],
+        swarm: %i[internal swarm],
+        manager: %i[internal swarm manager],
+        supervision_interval: %i[internal swarm supervision_interval],
+        shutdown_timeout: %i[shutdown_timeout],
+        supervision_sleep: %i[internal supervision_sleep],
+        forceful_exit_code: %i[internal forceful_exit_code],
+        process: %i[internal process]
+      )
+
+      def initialize
+        @mutex = Mutex.new
+        @queue = Processing::TimedQueue.new
+      end
+
+      # Creates needed number of forks, installs signals and starts supervision
+      def run
+        Karafka::App.warmup
+
+        manager.start
+
+        # Close producer just in case. While it should not be used, we do not want even a
+        # theoretical case since librdkafka is not thread-safe.
+        Karafka.producer.close
+
+        process.on_sigint { stop }
+        process.on_sigquit { stop }
+        process.on_sigterm { stop }
+        process.on_sigtstp { quiet }
+        process.on_sigttin { signal('TTIN') }
+        # Needed to be registered as we want to unlock on child changes
+        process.on_sigchld {}
+        process.on_any_active { unlock }
+        process.supervise
+
+        Karafka::App.supervise!
+
+        loop do
+          return if Karafka::App.terminated?
+
+          lock
+          control
+        end
+      # If anything went wrong, signal this and die
+      # Supervisor is meant to be thin and not cause any issues. If you encounter this case
+      # please report it as it should be considered critical
+      rescue StandardError => e
+        monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          manager: manager,
+          type: 'swarm.supervisor.error'
+        )
+
+        @nodes.terminate
+      end
+
+      private
+
+      # Keeps the lock on the queue so we control nodes only when it is needed
+      # @note We convert to seconds since the queue timeout requires seconds
+      def lock
+        @queue.pop(timeout: supervision_interval / 1_000.0)
+      end
+
+      # Frees the lock on events that could require nodes control
+      def unlock
+        @queue << true
+      end
+
+      # Stops all the nodes and supervisor once all nodes are dead.
+      # It will forcefully stop all nodes if they exit the shutdown timeout. While in theory each
+      # of the nodes anyhow has its own supervisor, this is a last resort to stop everything.
+      def stop
+        # Ensure that the stopping procedure is initialized only once
+        @mutex.synchronize do
+          return if @stopping
+
+          @stopping = true
+        end
+
+        initialized = true
+        Karafka::App.stop!
+
+        manager.stop
+
+        # We check from time to time (for the timeout period) if all the threads finished
+        # their work and if so, we can just return and normal shutdown process will take place
+        # We divide it by 1000 because we use time in ms.
+        ((shutdown_timeout / 1_000) * (1 / supervision_sleep)).to_i.times do
+          if manager.stopped?
+            manager.cleanup
+            return
+          end
+
+          sleep(supervision_sleep)
+        end
+
+        raise Errors::ForcefulShutdownError
+      rescue Errors::ForcefulShutdownError => e
+        monitor.instrument(
+          'error.occurred',
+          caller: self,
+          error: e,
+          manager: manager,
+          type: 'app.stopping.error'
+        )
+
+        # Run forceful kill
+        manager.terminate
+        # And wait until linux kills them
+        # This prevents us from existing forcefully with any dead child process still existing
+        # Since we have sent the `KILL` signal, it must die, so we can wait until all dead
+        sleep(supervision_sleep) until manager.stopped?
+
+        # Cleanup the process table
+        manager.cleanup
+
+        # exit! is not within the instrumentation as it would not trigger due to exit
+        Kernel.exit!(forceful_exit_code)
+      ensure
+        if initialized
+          Karafka::App.stopped!
+          Karafka::App.terminate!
+        end
+      end
+
+      # Moves all the nodes and itself to the quiet state
+      def quiet
+        @mutex.synchronize do
+          return if @quieting
+
+          @quieting = true
+
+          Karafka::App.quiet!
+          manager.quiet
+          Karafka::App.quieted!
+        end
+      end
+
+      # Checks on the children nodes and takes appropriate actions.
+      # - If node is dead, will cleanup
+      # - If node is no longer reporting as healthy will start a graceful shutdown
+      # - If node does not want to close itself gracefully, will kill it
+      # - If node was dead, new node will be started as a recovery means
+      def control
+        @mutex.synchronize do
+          # If we are in quieting or stopping we should no longer control children
+          # Those states aim to finally shutdown nodes and we should not forcefully do anything
+          # to them. This especially applies to the quieting mode where any complex lifecycle
+          # reporting listeners may no longer report correctly
+          return if @quieting
+          return if @stopping
+
+          manager.control
+        end
+      end
+
+      # Sends desired signal to each node
+      # @param signal [String]
+      def signal(signal)
+        @mutex.synchronize do
+          manager.signal(signal)
+        end
+      end
+    end
+  end
+end