cable_room 0.7.0.beta3 → 0.8.0.beta2

data/lib/cable_room/host/supervisor.rb

@@ -4,9 +4,9 @@ module CableRoom
   class Host
     # The parent process behind `cable_room server --workers N`. It forks N children, runs one
     # block in each, replaces a child that dies, and passes SIGTERM and SIGINT on to every child
-    # before exiting itself. It never hosts a room, never opens Redis, and never starts a Bus:
-    # each child builds its own after the fork (`CableRoom.after_fork!`), so nothing with a
-    # socket or a thread behind it is ever shared between processes.
+    # before exiting itself. It never hosts a room, never subscribes the Bus, and never starts a
+    # Host thread: each child builds its own after the fork (`CableRoom.after_fork!`), so nothing
+    # with a socket or a thread behind it is ever shared between processes.
     #
     #   Supervisor.new(count: 2, logger: logger) { |index| run_a_host(index) }.run
     #
@@ -18,16 +18,21 @@ module CableRoom
     #
     # Restarts: a child that dies is replaced. One that ran for at least `stable_after` seconds
     # comes back at once; one that died sooner is treated as failing to boot and comes back
-    # after a delay that doubles from `first_backoff` up to `max_backoff`, so a broken app forks
-    # a few times a minute rather than thousands of times a second.
+    # after a delay that doubles from `first_backoff` up to `max_backoff`. The delay only resets
+    # once a replacement stays up `stable_after` seconds, so the fastest any slot can cycle is
+    # once per `stable_after` seconds, and a broken app settles at one fork per slot every
+    # `max_backoff` seconds rather than thousands a second.
     #
-    # Signals: SIGTERM and SIGINT to the parent are relayed to every live child and the parent
-    # then waits, with no time limit of its own, until they've all exited (a child migrating rooms
-    # may take a while; the container's `timeout -k` is the outer bound). A signal sent to one
-    # child touches only that child: it exits, and the parent replaces it. The handlers do
-    # nothing but write a byte to a pipe; the main loop reads the pipe, so no real work runs in
-    # signal context. SIGCHLD writes the same pipe to wake the loop when a child exits, and the
-    # loop also polls every `POLL_INTERVAL` seconds in case a wake-up is ever missed.
+    # Signals: SIGTERM and SIGINT to the parent are relayed to every live child, and the parent
+    # waits up to `shutdown_timeout` seconds for them to exit, then SIGKILLs whatever is left.
+    # A second SIGTERM or SIGINT while waiting doesn't wait any longer: it SIGKILLs the children
+    # at once. Either way the parent exits 0 once they're all gone. A signal sent to one child
+    # touches only that child: it exits, and the parent replaces it.
+    #
+    # The handlers do nothing but write a line to a pipe; the main loop reads the pipe, so no
+    # real work runs in signal context. SIGCHLD writes the same pipe to wake the loop when a
+    # child exits, and the loop also polls every `POLL_INTERVAL` seconds in case a wake-up is
+    # ever missed.
     #
     # If the parent itself dies without relaying anything (SIGKILL, a crash), each child notices
     # through a second pipe (`watch_parent`) and stops as if it had been sent SIGTERM, so a dead
@@ -40,7 +45,7 @@ module CableRoom
 
       attr_reader :count, :logger
 
-      def initialize(count:, logger:, stable_after: 5, first_backoff: 1, max_backoff: 30, &body)
+      def initialize(count:, logger:, stable_after: 5, first_backoff: 1, max_backoff: 30, shutdown_timeout: 25, &body)
         raise ArgumentError, "count must be at least 1 (got #{count.inspect})" unless count.is_a?(Integer) && count >= 1
         raise ArgumentError, "Supervisor needs a block to run in each worker" unless body
 
@@ -49,12 +54,14 @@ module CableRoom
         @stable_after = stable_after
         @first_backoff = first_backoff
         @max_backoff = max_backoff
+        @shutdown_timeout = shutdown_timeout
         @body = body
 
         @workers = {}      # index => Worker, for every live child
         @restart_at = {}   # index => monotonic time to fork a replacement
         @backoff = {}      # index => the delay used for that slot's last boot failure
         @stopping = false
+        @kill_at = nil     # monotonic time to SIGKILL children that haven't exited
       end
 
       # Fork the workers and supervise them until a stop signal has arrived and every child has
@@ -70,7 +77,12 @@ module CableRoom
           wait_for_wakeup
           reap_exited_workers
           break if @stopping && @workers.empty?
-          start_due_restarts unless @stopping
+
+          if @stopping
+            kill_stragglers if now >= @kill_at
+          else
+            start_due_restarts
+          end
         end
 
         logger.info "all workers exited"
@@ -117,10 +129,10 @@ module CableRoom
         nil
       end
 
-      # Sleep until a signal or child exit wakes us, a restart is due, or the poll interval
-      # passes. Any stop signal on the pipe starts the shutdown.
+      # Sleep until a signal or child exit wakes us, a restart or the kill deadline is due, or
+      # the poll interval passes. Any stop signal on the pipe starts (or escalates) the shutdown.
       def wait_for_wakeup
-        timeout = [POLL_INTERVAL, seconds_until_next_restart].compact.min
+        timeout = [POLL_INTERVAL, seconds_until_next_restart, seconds_until_kill].compact.min
         ready, = IO.select([@signal_reader], nil, nil, timeout)
         return unless ready
 
@@ -134,17 +146,27 @@ module CableRoom
 
       def begin_stopping(signal)
         if @stopping
-          # A second Ctrl-C or TERM: pass it along again, in case a child missed the first
-          relay(signal)
+          # A second Ctrl-C or TERM: whoever sent it doesn't want to wait for a graceful stop
+          logger.warn "got SIG#{signal} again, killing #{@workers.size} worker(s) now"
+          relay("KILL")
+          @kill_at = Float::INFINITY
           return
         end
 
         @stopping = true
         @restart_at.clear
-        logger.info "got SIG#{signal}, relaying to #{@workers.size} worker(s) and waiting for them"
+        @kill_at = now + @shutdown_timeout
+        logger.info "got SIG#{signal}, relaying to #{@workers.size} worker(s) and waiting up to #{@shutdown_timeout}s for them"
         relay(signal)
       end
 
+      # Past the shutdown deadline: kill what's left, once (the next reap collects them)
+      def kill_stragglers
+        logger.warn "#{@workers.size} worker(s) still running after #{@shutdown_timeout}s, killing them"
+        relay("KILL")
+        @kill_at = Float::INFINITY
+      end
+
       def relay(signal)
         @workers.each_value do |worker|
           Process.kill(signal, worker.pid)
@@ -209,6 +231,10 @@ module CableRoom
         next_at && [next_at - now, 0].max
       end
 
+      def seconds_until_kill
+        @kill_at && [@kill_at - now, 0].max
+      end
+
       def start_worker(index)
         pid = fork_worker(index)
         @workers[index] = Worker.new(index, pid, now)
@@ -259,8 +285,8 @@ module CableRoom
 
       # Once every child has closed its copy, the parent holds the only write end of the lifeline
       # pipe, so a read here returns EOF exactly when the parent is gone: crashed, or SIGKILLed by
-      # the container's `timeout -k`. A worker must not carry on hosting rooms with nobody
-      # supervising it, so it stops itself the same way a relayed SIGTERM would stop it.
+      # the container. A worker must not carry on hosting rooms with nobody supervising it, so it
+      # stops itself the same way a relayed SIGTERM would stop it.
       def watch_parent(reader)
         thread = Thread.new do
           reader.read(1)

data/lib/cable_room/host/worker_pool.rb

@@ -1,50 +1,37 @@
-require 'concurrent'
-
 module CableRoom
   class Host
-    # The thread pool every room's work runs on. Deliberately *not* an ActionCable::Server::Worker
-    # subclass: that would put Room work on the same shared `:work` callback chain as ordinary
-    # ActionCable connections, and that chain is leaky by construction -- `ActiveSupport::Callbacks`
-    # re-injects a callback added to the base class into every existing descendant regardless of
-    # load order, so an app's own tenant-switching hook (PandaPal's, say, registered from a Rails
-    # initializer well after this class is defined) ends up wrapping Room work too, whether we want
-    # it to or not. Owning a distinct namespace here avoids that category of problem entirely:
-    # nothing outside cable_room can attach to Room work by surprise. The one seam an app gets is
-    # a Room's own `before_work`/`after_work`/`around_work` (see Room::Callbacks and
-    # Host::Runner#with_room_context).
-    class WorkerPool
-      attr_reader :executor
-
-      def initialize(max_size: 5)
-        @executor = Concurrent::ThreadPoolExecutor.new(
-          name: "CableRoom",
-          min_threads: 1,
-          max_threads: max_size,
-          max_queue: 0,
-        )
+    # The thread pool every room's work runs on. It's an ActionCable Worker so the `:work`
+    # callbacks Rails installs (the executor wrap, ActiveRecord log tagging) still apply to room
+    # work exactly as they did when rooms were channels.
+    #
+    # The "connection" passed around here is the room's Host::Runner. ActionCable's Worker was
+    # written for connections; rooms don't have one, but the runner fills the same role: it's
+    # the thing with a logger and an error reporter.
+    class WorkerPool < ActionCable::Server::Worker
+      set_callback :work, :around do |_, blk|
+        pconn = ActionCable::Server::Worker.connection
+        ActionCable::Server::Worker.connection = connection
+        blk.call
+      ensure
+        ActionCable::Server::Worker.connection = pconn
       end
 
-      # Reduces every exception to a proper report instead of ActionCable's Worker#invoke, which
-      # logs a line and calls a no-argument `handle_exception`, discarding the error itself. Rooms
-      # run all of their work through here, so report it properly. `connection:` is always a
-      # Host::Runner in practice (see Runner#post_work); the else branch is a defensive fallback.
+      # ActionCable's Worker#invoke reduces every exception to a log line and a no-argument
+      # `handle_exception` call, which discards the error itself. Rooms run all of their work
+      # through here, so report it properly instead.
       def invoke(receiver, method, *args, connection:, &block)
-        receiver.send method, *args, &block
-      rescue Exception => e
-        if connection.respond_to?(:report_work_error)
-          connection.report_work_error(e)
-        else
-          logger.error "There was an exception - #{e.class}(#{e.message})"
-          logger.error Array(e.backtrace).join("\n")
-          CableRoom.report_error(e, connection: connection)
+        work(connection) do
+          receiver.send method, *args, &block
+        rescue Exception => e
+          if connection.respond_to?(:report_work_error)
+            connection.report_work_error(e)
+          else
+            logger.error "There was an exception - #{e.class}(#{e.message})"
+            logger.error Array(e.backtrace).join("\n")
+            CableRoom.report_error(e, connection: connection)
+          end
         end
       end
-
-      private
-
-      def logger
-        ActionCable.server.logger
-      end
     end
   end
 end