cable_room 0.6.2.beta1 → 0.7.0.beta2

data/lib/cable_room/host/supervisor.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+
+module CableRoom
+  class Host
+    # The parent process behind `cable_room server --workers N`. It forks N children, runs one
+    # block in each, replaces a child that dies, and passes SIGTERM and SIGINT on to every child
+    # before exiting itself. It never hosts a room, never opens Redis, and never starts a Bus:
+    # each child builds its own after the fork (`CableRoom.after_fork!`), so nothing with a
+    # socket or a thread behind it is ever shared between processes.
+    #
+    #   Supervisor.new(count: 2, logger: logger) { |index| run_a_host(index) }.run
+    #
+    # The block is the whole child: it runs right after the fork and its return value (an Integer,
+    # or nil for 0) is the child's exit status. The child leaves with `exit!`, so `at_exit` hooks
+    # the parent registered before forking don't run a second time in every child (Resque does
+    # the same for its forked workers). A block that raises is logged and exits 1, which makes
+    # the supervisor replace it.
+    #
+    # Restarts: a child that dies is replaced. One that ran for at least `stable_after` seconds
+    # comes back at once; one that died sooner is treated as failing to boot and comes back
+    # after a delay that doubles from `first_backoff` up to `max_backoff`, so a broken app forks
+    # a few times a minute rather than thousands of times a second.
+    #
+    # Signals: SIGTERM and SIGINT to the parent are relayed to every live child and the parent
+    # then waits, with no time limit of its own, until they've all exited (a child migrating rooms
+    # may take a while; the container's `timeout -k` is the outer bound). A signal sent to one
+    # child touches only that child: it exits, and the parent replaces it. The handlers do
+    # nothing but write a byte to a pipe; the main loop reads the pipe, so no real work runs in
+    # signal context. SIGCHLD writes the same pipe to wake the loop when a child exits, and the
+    # loop also polls every `POLL_INTERVAL` seconds in case a wake-up is ever missed.
+    #
+    # If the parent itself dies without relaying anything (SIGKILL, a crash), each child notices
+    # through a second pipe (`watch_parent`) and stops as if it had been sent SIGTERM, so a dead
+    # supervisor never leaves orphaned workers hosting rooms.
+    class Supervisor
+      STOP_SIGNALS = %w[TERM INT].freeze
+      POLL_INTERVAL = 1
+
+      Worker = Struct.new(:index, :pid, :started_at)
+
+      attr_reader :count, :logger
+
+      def initialize(count:, logger:, stable_after: 5, first_backoff: 1, max_backoff: 30, &body)
+        raise ArgumentError, "count must be at least 1 (got #{count.inspect})" unless count.is_a?(Integer) && count >= 1
+        raise ArgumentError, "Supervisor needs a block to run in each worker" unless body
+
+        @count = count
+        @logger = logger
+        @stable_after = stable_after
+        @first_backoff = first_backoff
+        @max_backoff = max_backoff
+        @body = body
+
+        @workers = {}      # index => Worker, for every live child
+        @restart_at = {}   # index => monotonic time to fork a replacement
+        @backoff = {}      # index => the delay used for that slot's last boot failure
+        @stopping = false
+      end
+
+      # Fork the workers and supervise them until a stop signal has arrived and every child has
+      # exited. Blocks. Returns the process exit status (0).
+      def run
+        @signal_reader, @signal_writer = IO.pipe
+        @lifeline_reader, @lifeline_writer = IO.pipe
+        previous_traps = install_traps
+
+        count.times { |index| start_worker(index) }
+
+        loop do
+          wait_for_wakeup
+          reap_exited_workers
+          break if @stopping && @workers.empty?
+          start_due_restarts unless @stopping
+        end
+
+        logger.info "all workers exited"
+        0
+      ensure
+        # If we're leaving for any reason other than "every child is gone" (a bug, say), don't
+        # orphan the children: tell them to stop too. Then hand the signals back.
+        relay("TERM") if @workers.any?
+        previous_traps&.each { |sig, handler| trap(sig, handler) }
+        [@signal_reader, @signal_writer, @lifeline_reader, @lifeline_writer].each { |io| io&.close }
+      end
+
+      # Ask the supervisor to shut down as if `signal` had arrived: relay it to every worker and
+      # exit once they're gone. Safe from any thread and from a trap handler; the work happens on
+      # the thread running `run`.
+      def stop!(signal = "TERM")
+        wake(signal)
+      end
+
+      def stopping?
+        @stopping
+      end
+
+      # Pids of the children alive right now.
+      def worker_pids
+        @workers.values.map(&:pid)
+      end
+
+      private
+
+      # ---- The parent ----------------------------------------------------------------------
+
+      def install_traps
+        (STOP_SIGNALS + %w[CHLD]).to_h do |sig|
+          [sig, trap(sig) { wake(sig) }]
+        end
+      end
+
+      # Runs in signal context, so it does the one thing that's safe there: a non-blocking write
+      # to the pipe. A full pipe means plenty of wake-ups are already queued, so dropping is fine.
+      def wake(signal)
+        @signal_writer&.write_nonblock("#{signal}\n")
+      rescue IO::WaitWritable, IOError, Errno::EPIPE
+        nil
+      end
+
+      # Sleep until a signal or child exit wakes us, a restart is due, or the poll interval
+      # passes. Any stop signal on the pipe starts the shutdown.
+      def wait_for_wakeup
+        timeout = [POLL_INTERVAL, seconds_until_next_restart].compact.min
+        ready, = IO.select([@signal_reader], nil, nil, timeout)
+        return unless ready
+
+        pending = begin
+          @signal_reader.read_nonblock(4096)
+        rescue IO::WaitReadable, EOFError
+          ""
+        end
+        pending.split("\n").each { |signal| begin_stopping(signal) if STOP_SIGNALS.include?(signal) }
+      end
+
+      def begin_stopping(signal)
+        if @stopping
+          # A second Ctrl-C or TERM: pass it along again, in case a child missed the first
+          relay(signal)
+          return
+        end
+
+        @stopping = true
+        @restart_at.clear
+        logger.info "got SIG#{signal}, relaying to #{@workers.size} worker(s) and waiting for them"
+        relay(signal)
+      end
+
+      def relay(signal)
+        @workers.each_value do |worker|
+          Process.kill(signal, worker.pid)
+        rescue Errno::ESRCH
+          nil # already gone; the next reap logs it
+        end
+      end
+
+      def reap_exited_workers
+        @workers.values.each do |worker|
+          status = begin
+            _, status = Process.wait2(worker.pid, Process::WNOHANG)
+            status
+          rescue Errno::ECHILD
+            :unknown # someone else reaped it; treat it as gone
+          end
+          next if status.nil?
+
+          @workers.delete(worker.index)
+          uptime = now - worker.started_at
+          reason = describe_exit(status)
+
+          if @stopping
+            logger.info "worker #{worker.index} (pid #{worker.pid}) #{reason} after #{uptime.round}s"
+          else
+            delay = restart_delay(worker.index, uptime)
+            @restart_at[worker.index] = now + delay
+            logger.warn "worker #{worker.index} (pid #{worker.pid}) #{reason} after #{uptime.round}s; " \
+                        "restarting #{delay.zero? ? 'now' : "in #{delay}s"}"
+          end
+        end
+      end
+
+      def describe_exit(status)
+        return "exited (status unknown)" if status == :unknown
+        return "killed by SIG#{Signal.signame(status.termsig)}" if status.signaled?
+
+        "exited with status #{status.exitstatus}"
+      end
+
+      # A worker that lived a while and then died gets replaced right away. One that died
+      # almost immediately most likely can't boot, so wait, and wait longer each time it happens.
+      def restart_delay(index, uptime)
+        if uptime >= @stable_after
+          @backoff.delete(index)
+          0
+        else
+          @backoff[index] = @backoff[index] ? [@backoff[index] * 2, @max_backoff].min : @first_backoff
+        end
+      end
+
+      def start_due_restarts
+        due = @restart_at.select { |_, at| at <= now }.keys
+        due.each do |index|
+          @restart_at.delete(index)
+          start_worker(index)
+        end
+      end
+
+      def seconds_until_next_restart
+        next_at = @restart_at.values.min
+        next_at && [next_at - now, 0].max
+      end
+
+      def start_worker(index)
+        pid = fork_worker(index)
+        @workers[index] = Worker.new(index, pid, now)
+        logger.info "forked worker #{index} (pid #{pid})"
+      end
+
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      # ---- The child -----------------------------------------------------------------------
+
+      def fork_worker(index)
+        Process.fork do
+          status = 1
+          begin
+            # The child inherits the parent's traps and both ends of its wake-up pipe. Neither
+            # belongs here: a relayed SIGTERM must reach the body's own handler (or kill the
+            # child by default) instead of writing to a pipe nobody in this process reads.
+            (STOP_SIGNALS + %w[CHLD]).each { |sig| trap(sig, "DEFAULT") }
+            @signal_reader.close
+            @signal_writer.close
+            @workers = {}
+
+            # Give up our copy of the lifeline's write end first, so a sibling can never keep the
+            # pipe open after the parent is gone; then watch the read end for the parent's death
+            @lifeline_writer.close
+            watch_parent(@lifeline_reader)
+
+            CableRoom.after_fork!
+
+            result = @body.call(index)
+            status = result.is_a?(Integer) ? result : 0
+          rescue SystemExit => e
+            status = e.status
+          rescue SignalException
+            raise # let Ruby end the process with the signal, so the parent sees which one
+          rescue Exception => e # rubocop:disable Lint/RescueException -- the child must not outlive a broken body
+            logger.error "worker #{index} (pid #{Process.pid}) crashed: #{e.class}: #{e.message}\n  #{e.backtrace&.first(10)&.join("\n  ")}"
+            status = 1
+          end
+
+          $stdout.flush
+          $stderr.flush
+          exit!(status)
+        end
+      end
+
+      # Once every child has closed its copy, the parent holds the only write end of the lifeline
+      # pipe, so a read here returns EOF exactly when the parent is gone: crashed, or SIGKILLed by
+      # the container's `timeout -k`. A worker must not carry on hosting rooms with nobody
+      # supervising it, so it stops itself the same way a relayed SIGTERM would stop it.
+      def watch_parent(reader)
+        thread = Thread.new do
+          reader.read(1)
+          Process.kill("TERM", Process.pid)
+        end
+        thread.name = "cable_room-supervisor-lifeline"
+        thread.report_on_exception = false
+        thread
+      end
+    end
+  end
+end

data/lib/cable_room/host/worker_pool.rb

@@ -0,0 +1,37 @@
+module CableRoom
+  class Host
+    # The thread pool every room's work runs on. It's an ActionCable Worker so the `:work`
+    # callbacks Rails installs (the executor wrap, ActiveRecord log tagging) still apply to room
+    # work exactly as they did when rooms were channels.
+    #
+    # The "connection" passed around here is the room's Host::Runner. ActionCable's Worker was
+    # written for connections; rooms don't have one, but the runner fills the same role: it's
+    # the thing with a logger and an error reporter.
+    class WorkerPool < ActionCable::Server::Worker
+      set_callback :work, :around do |_, blk|
+        pconn = ActionCable::Server::Worker.connection
+        ActionCable::Server::Worker.connection = connection
+        blk.call
+      ensure
+        ActionCable::Server::Worker.connection = pconn
+      end
+
+      # ActionCable's Worker#invoke reduces every exception to a log line and a no-argument
+      # `handle_exception` call, which discards the error itself. Rooms run all of their work
+      # through here, so report it properly instead.
+      def invoke(receiver, method, *args, connection:, &block)
+        work(connection) do
+          receiver.send method, *args, &block
+        rescue Exception => e
+          if connection.respond_to?(:report_work_error)
+            connection.report_work_error(e)
+          else
+            logger.error "There was an exception - #{e.class}(#{e.message})"
+            logger.error Array(e.backtrace).join("\n")
+            CableRoom.report_error(e, connection: connection)
+          end
+        end
+      end
+    end
+  end
+end