cable_room 0.7.0.beta3 → 0.8.0.beta2

data/lib/cable_room/host.rb

@@ -1,8 +1,8 @@
-require 'socket'
 require_relative 'host/worker_pool'
 require_relative 'host/bus_inbound'
 require_relative 'host/runner'
 require_relative 'host/supervisor'
+require_relative 'host/placement'
 
 module CableRoom
   # One Host per process. It owns every Room running here: the worker pool their work runs on,
@@ -20,8 +20,8 @@ module CableRoom
     class NotHosting < StandardError; end
 
     NOT_HOSTING_MESSAGE =
-      "This process doesn't host rooms: CableRoom.config.room_host is :remote, so rooms run " \
-      "under `cable_room server`. Start rooms there (or call CableRoom::Host.start! first if this " \
+      "This process doesn't host rooms: CableRoom.room_host is :remote, so rooms run under " \
+      "`cable_room server`. Start rooms there (or call CableRoom::Host.start! first if this " \
       "process really should host them).".freeze
 
     @start_mutex = Mutex.new
@@ -39,88 +39,53 @@ module CableRoom
       # (`cable_room server`) has one; anyone else gets NotHosting, so a room can never quietly
       # start in the web tier because some code path reached for the Host.
       def instance
-        current || (CableRoom.config.inline? ? start! : raise(NotHosting, NOT_HOSTING_MESSAGE))
-      end
-
-      # Make this process a rooms host. Builds the Host once and starts its provisioning listener,
-      # so this process hears `create: true` requests from then on; later calls return the same
-      # Host. `cable_room server` calls this explicitly; in :inline `instance` calls it for you.
-      def start!
-        @start_mutex.synchronize { @current ||= new.tap(&:start_placement) }
+        current || (CableRoom.inline? ? start! : raise(NotHosting, NOT_HOSTING_MESSAGE))
+      end
+
+      # Make this process a rooms host. Builds the Host once, with `options` (see `new`); later
+      # calls return the same one and ignore them. `cable_room server` calls this explicitly; in
+      # :inline `instance` calls it for you.
+      #
+      # In :remote it also starts the Host's Placement, so from then on this process hears
+      # `create: true` members' provision requests and races its peers for their rooms. In :inline
+      # nobody sends those (a member starts its room itself), so there's nothing to listen for.
+      def start!(**options)
+        @start_mutex.synchronize do
+          @current ||= begin
+            host = new(**options)
+            host.start_placement if CableRoom.remote?
+            host
+          rescue StandardError
+            host&.shutdown!
+            raise
+          end
+        end
       end
 
       # Swap this process's Host for `host` (nil allowed) and return the previous one. Only specs
-      # that play two process roles in one Ruby process should need this (see RoomHostHarness):
-      # the "web" side has to see no Host while another Host runs the rooms.
+      # that play both process roles in one Ruby process need this: the web side has to see no
+      # Host while another Host runs the rooms.
       def replace_current(host)
         @start_mutex.synchronize { @current.tap { @current = host } }
       end
     end
 
-    # How many rooms `drain!` hands off at once before `drain_timeout`. Each handoff costs the
-    # adopting peer a full room restore (its startup chain, often database reads), so a host with
-    # hundreds of rooms must not offer them all in the same second; and each one holds a thread
-    # here for up to `handoff_timeout`. Four keeps a big host draining in well under the default
-    # ten minutes (a handoff normally takes well under a second) without swamping the fleet.
-    DRAIN_CONCURRENCY = 4
-
-    # What `drain!` returns: how the rooms ended up, and how long it took. `migrated` and
-    # `closed` are Migration objects (see their `state` and `room`).
-    DrainResult = Struct.new(:reason, :migrated, :closed, :duration, keyword_init: true) do
-      def rooms
-        migrated.size + closed.size
-      end
-    end
-
     attr_reader :scheduler, :inbound, :placement
 
-    # True once this host has started shutting down for good and is handing its rooms off (see
-    # `drain!`). A draining host never claims a provision request, so it can't take on a room it
-    # would only have to hand off again.
-    attr_writer :draining
-
     delegate :logger, to: :cable_server
 
-    class << self
-      # Register a block to run when a `drain!` finishes, with the Host and its DrainResult. An
-      # app that scales in through an ASG termination lifecycle hook completes the hook here
-      # (`complete-lifecycle-action`), once every room has moved or closed. Blocks run on the
-      # draining thread before `drain!` returns; one that raises is reported and doesn't stop
-      # the others.
-      def after_drain(&blk)
-        raise ArgumentError, "after_drain needs a block" unless blk
-        after_drain_callbacks << blk
-        blk
-      end
-
-      def after_drain_callbacks
-        @after_drain_callbacks ||= []
-      end
-    end
-
-    # `inbound` is how rooms receive messages: by default the process-wide Bus, so a room hears
-    # its members the same way whether it runs inside the web process or on a remote host.
-    def initialize(inbound: BusInbound.new)
+    # `inbound` is how rooms receive messages: by default the process-wide Bus, waiting up to
+    # `subscribe_timeout` seconds for Redis to confirm each room's subscriptions (a room's startup
+    # blocks on that). Anything with BusInbound's two methods works.
+    def initialize(inbound: nil, subscribe_timeout: Bus::DEFAULT_TIMEOUT)
       @runners = Set.new
       @monitor = Monitor.new
-      @inbound = inbound
-      @draining = false
-      @drain_result = nil
-      @drain_thread = nil
-      @migrations = []
+      @inbound = inbound || BusInbound.new(timeout: subscribe_timeout)
       @scheduler = Rufus::Scheduler.new
 
-      # A rooms host that leaves with rooms still open (a crash on the main thread, say; on a
-      # SIGTERM `cable_room server` has already drained by now, so this is a no-op) gives them
-      # the same chance to move as a planned shutdown would. In :inline there is no fleet of
-      # peers by design (see `drain!`), so exit closes rooms the way it always has.
       at_exit do
         logger.info "Shutting down CableRoom"
-        if CableRoom.config.remote? && !shutdown? && runners.any?
-          drain!(reason: "process exit")
-        else
-          shutdown!
-        end
+        shutdown!
       end
 
       # Every few seconds each room renews its Redlock and checks its watchdog
@@ -130,118 +95,56 @@ module CableRoom
     end
 
     def worker_pool
+      # TODO Pin Rooms to a specific thread so that they never have to worry about thread safety?
       @worker_pool || @monitor.synchronize do
         @worker_pool ||= WorkerPool.new(max_size: cable_server.config.worker_pool_size)
       end
     end
 
     # Start `room_class`'s room for `key` on this Host if nobody else is running it. Takes the
-    # room's Redlock first, so exactly one process cluster-wide wins; returns false when the lock
-    # is already held. This is the host-side way to bring a room up: `Room::Base.ensure` calls it
-    # on the process Host, Placement calls it for a provision request, and `cable_room server`
-    # (or a spec harness) can call it on a Host it owns without any member asking.
+    # room's Redlock first, so exactly one process wins; returns false when the lock is held.
+    # `Room::Base.ensure` calls this on the process Host; `cable_room server` (or a spec playing
+    # one) can call it on a Host it owns.
     #
-    # If a snapshot of the room is on offer (another host is handing it off, see
-    # CableRoom::Migration), winning the lock means adopting it: the room comes back here with
-    # its members and state rather than starting fresh underneath the old host. The check is
-    # made after the lock is won, so it can't be raced. `handoff_only: true` (a handoff request)
-    # starts nothing when there is no snapshot to adopt — it has expired, or the old host took
-    # the room back — and lets the lock go.
-    def ensure_room(room_class, key = nil, handoff_only: false, tenant: nil)
-      lock_key = room_class.room_port_key(key)
-      lock_info = CableRoom.lock_manager.lock(lock_key, room_class::LOCK_DURATION.in_milliseconds)
-      return false unless lock_info
-
-      snapshot = bus.get(Bus.snapshot_key(lock_key))
-      return Migration.adopt(self, room_class, key, lock_info: lock_info, snapshot: snapshot) if snapshot
-
-      if handoff_only
-        logger.info "No snapshot to adopt for #{lock_key}; letting the lock go"
-        CableRoom.lock_manager.unlock(lock_info)
-        return false
-      end
+    # `tenant` is the Apartment tenant the room runs in. It defaults to the caller's, which is
+    # right for a member's own request thread (:inline). A rooms process has no request tenant,
+    # so whatever starts a room there must pass it. The lock key, the room's stream names, and its
+    # startup all depend on it, so the whole start runs switched into it.
+    def ensure_room(room_class, key = nil, tenant: CableRoom.current_tenant)
+      CableRoom.with_tenant(tenant) do
+        lock_key = room_class.room_port_key(key)
+        # One attempt: a held lock means the room is already running, and retrying (Redlock's
+        # default is 3 retries, 200-250 ms apart) would park the caller's ActionCable worker for
+        # ~0.7 s on every `create: true` member's ping.
+        lock_info = CableRoom.lock_manager.lock(lock_key, room_class::LOCK_DURATION.in_milliseconds, retry_count: 0)
+        next false unless lock_info
+
+        begin
+          start_room(
+            room_class,
+            key,
+            lock_info,
+            watchdog_interval: room_class::WATCH_DOG_INTERVAL,
+            lock_duration: room_class::LOCK_DURATION,
+            tenant: tenant,
+          )
+        rescue => e
+          CableRoom.lock_manager.unlock(lock_info)
+          raise e
+        end
 
-      start_room(
-        room_class,
-        key,
-        lock_info,
-        watchdog_interval: room_class::WATCH_DOG_INTERVAL,
-        lock_duration: room_class::LOCK_DURATION,
-        tenant: tenant,
-      )
-
-      true
-    rescue => e
-      CableRoom.lock_manager.unlock(lock_info) if lock_info
-      raise e
+        true
+      end
     end
 
     # Start running a room here. The caller (`ensure_room`) has already taken the room's Redlock;
-    # the runner renews it from now on and releases it when the room dies. `tenant:` comes from the
-    # member's provision request (see RoomMembership#initialize / #maybe_provision_room and
-    # Placement#claim), already defaulted there to the ambient tenant if the app didn't pass one
-    # explicitly -- nil past that point just means no Apartment multi-tenancy is in play.
-    def start_room(room_class, key, lock_info, watchdog_interval:, lock_duration:, tenant: nil)
+    # the runner renews it from now on and releases it when the room dies.
+    def start_room(room_class, key, lock_info, watchdog_interval:, lock_duration:, tenant: CableRoom.current_tenant)
       runner = Runner.new(self, room_class, key, lock_info, watchdog_interval:, lock_duration:, tenant:)
       runner.start!
       runner
     end
 
-    # Bring a room back from a CableRoom::Snapshot on this Host, the way a migration's adopter
-    # does. With no `lock_info`, takes the room's Redlock first and returns false if someone else
-    # holds it — the same rule as `ensure_room`. With one, the caller has already claimed the lock.
-    # Raises Snapshot::UnknownVersion or Snapshot::UnknownRoomClass before anything is built if
-    # the snapshot can't be used here. `hold_inbound: true` brings the room up frozen, holding
-    # its inbound until `Runner#thaw!` (see Runner#restore!).
-    #
-    # If the room fails to come up, the lock is released either way — the caller's too. A room
-    # that can't start here must not keep its key locked: the migration that handed it over
-    # takes the lock back and restarts it as soon as the lock is free.
-    def restore_room(snapshot, lock_info = nil, hold_inbound: false)
-      snapshot = Snapshot.validate!(snapshot)
-      room_class = Snapshot.room_class_for(snapshot)
-      key = Snapshot.key_for(snapshot)
-      lock_key = room_class.room_port_key(key)
-
-      if lock_info
-        unless lock_info[:resource] == lock_key
-          raise ArgumentError, "lock_info is for #{lock_info[:resource].inspect}, but the snapshot is #{lock_key}"
-        end
-      else
-        lock_info = CableRoom.lock_manager.lock(lock_key, room_class::LOCK_DURATION.in_milliseconds)
-        return false unless lock_info
-      end
-
-      runner = Runner.new(
-        self,
-        room_class,
-        key,
-        lock_info,
-        watchdog_interval: room_class::WATCH_DOG_INTERVAL,
-        lock_duration: room_class::LOCK_DURATION,
-        tenant: snapshot["tenant"],
-      )
-      runner.restore!(snapshot, hold_inbound: hold_inbound)
-      runner
-    rescue => e
-      # `restore!` releases the lock itself when the room fails to come up; this covers a failure
-      # before the runner exists
-      CableRoom.lock_manager.unlock(lock_info) if lock_info && runner.nil?
-      raise e
-    end
-
-    # A name for this host in migration traffic and instrumentation (`from_host`, `to_host`):
-    # machine, pid, and a few random characters, so two hosts in one process (a spec) differ too.
-    def id
-      @id ||= "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.hex(3)}"
-    end
-
-    # The Bus this host's rooms listen on, for the migration's list and key operations. Falls
-    # back to the process-wide Bus when `inbound` isn't Bus-backed.
-    def bus
-      inbound.respond_to?(:bus) ? inbound.bus : CableRoom.bus
-    end
-
     def track(runner)
       @monitor.synchronize do
         raise "Cannot add Room after shutdown" if @shutdown
@@ -262,128 +165,37 @@ module CableRoom
       runners.map(&:room)
     end
 
-    # How many member ports are attached to rooms on this Host right now. This is the load
-    # number the design scales the rooms pool on, so an app can publish it as a gauge. It reads
-    # each room's port table from outside the room's thread; a count that's a moment stale is
-    # fine for a metric.
-    def open_ports
-      rooms.sum(&:open_port_count)
-    end
-
-    # How many rooms this Host runs right now. This is the load number provisioning weights its
-    # delay by: a host with more rooms waits longer before racing for a new one.
-    def open_rooms_count
-      @monitor.synchronize { @runners.size }
-    end
-
     def each_runner(&blk)
       runners.each(&blk)
     end
 
-    def draining?
-      @draining
-    end
-
-    # True once `drain!` has finished (every room moved or closed, and the host is shut down).
-    def drained?
-      !@drain_result.nil?
-    end
-
-    # The result of the drain, or nil until `drain!` has finished.
-    def drain_result
-      @drain_result
-    end
-
-    # Every CableRoom::Migration this host's drain has started, oldest room first. Each one's
-    # `state` says where it is; the list is complete once `drained?`.
-    def migrations
-      @monitor.synchronize { @migrations.dup }
+    # How many rooms this Host runs right now. Cheap enough to publish as a gauge.
+    def open_rooms_count
+      @monitor.synchronize { @runners.size }
     end
 
-    # Planned shutdown: move every room here to a peer host, then shut down. This is what SIGTERM
-    # means to `cable_room server` (see CableRoom::Migration for the per-room protocol):
-    #
-    #   1. Mark the host draining, so its Placement claims nothing more (a room it took now would
-    #      only have to move again), and instrument `host_draining.cable_room`.
-    #   2. Hand rooms off oldest first, DRAIN_CONCURRENCY at a time. A room nobody adopts within
-    #      `handoff_timeout` closes with `room_closed`; so does one that can't be snapshotted.
-    #   3. When `drain_timeout` passes with rooms still waiting their turn, offer all of them at
-    #      once — the same path, just no longer throttled — so the drain ends within about one
-    #      more `handoff_timeout` whatever the fleet does.
-    #   4. `shutdown!`, run the `Host.after_drain` callbacks, and return a DrainResult.
-    #
-    # Blocks until done. Runs once: a second call (from another thread, or `at_exit` after
-    # `cable_room server` has already drained) waits for the first and returns its result.
-    # `drain_timeout` and `handoff_timeout` default to the config's; specs pass short ones.
-    #
-    # In :inline the peers are the other web processes hosting rooms, if any. The design gives
-    # :inline no fleet of its own, so the gem doesn't drain there on exit (see `at_exit`); an app
-    # running several inline processes may still call this from its own signal handling.
-    def drain!(reason:, drain_timeout: nil, handoff_timeout: nil, concurrency: DRAIN_CONCURRENCY)
-      other = @monitor.synchronize do
-        return @drain_result if @drain_result
-        @drain_thread.tap { @drain_thread ||= Thread.current }
-      end
-      if other
-        other.join unless other == Thread.current
-        return @drain_result
-      end
-
-      drain_timeout ||= CableRoom.config.drain_timeout
-      handoff_timeout ||= CableRoom.config.handoff_timeout
-      started = monotonic_now
-      self.draining = true
-
-      queued = runners.sort_by(&:started_at)
-      migrations = queued.map { |runner| Migration.new(self, runner, reason: reason, handoff_timeout: handoff_timeout) }
-      @monitor.synchronize { @migrations = migrations }
-      logger.info "Draining #{migrations.size} room(s): #{reason}"
-
-      payload = { host: self, rooms: queued.map(&:room), reason: reason }
-      ActiveSupport::Notifications.instrument("host_draining.cable_room", payload) do |event|
-        run_migrations(migrations, concurrency: concurrency, deadline: started + drain_timeout.to_f)
-        event[:migrated] = migrations.count(&:migrated?)
-        event[:closed] = migrations.count { |m| !m.migrated? }
-      end
-
-      shutdown!
-
-      result = DrainResult.new(
-        reason: reason,
-        migrated: migrations.select(&:migrated?),
-        closed: migrations.reject(&:migrated?),
-        duration: monotonic_now - started,
-      )
-      logger.info "Drained: #{result.migrated.size} room(s) migrated, #{result.closed.size} closed, in #{result.duration.round(2)}s"
-      @monitor.synchronize { @drain_result = result }
-
-      self.class.after_drain_callbacks.each do |callback|
-        callback.call(self, result)
-      rescue => e
-        CableRoom.report_error(e, host: self, drain_result: result)
-      end
-
-      result
+    # How many member ports are attached across every room here. It reads each room's port table
+    # from outside the room's thread; a count that's a moment stale is fine for a metric.
+    def open_ports
+      rooms.sum(&:open_port_count)
     end
 
     def shutdown?
       @shutdown
     end
 
-    # Start listening for provision requests (see CableRoom::Placement). `Host.start!` does this
-    # for the process Host; a Host built with `new` (a spec harness) calls it itself. Pass a
-    # Placement to use one with non-default settings; calling it again re-arms the current one.
-    def start_placement(placement = nil)
-      previous = @monitor.synchronize do
+    # Start hearing provision requests (see Host::Placement). `Host.start!` does this for a :remote
+    # rooms process; a Host built with `new` (a spec playing one) calls it itself. `options` go to
+    # Placement.new the first time; later calls restart the same Placement.
+    def start_placement(**options)
+      placement = @monitor.synchronize do
         raise "Cannot start placement after shutdown" if @shutdown
-        @placement.tap { @placement = placement || @placement || Placement.new(self) }
+        @placement ||= Placement.new(self, **options)
       end
-      previous.stop if previous && previous != @placement
-      @placement.start
-      @placement
+      placement.start
     end
 
-    # Stop listening and drop any claim still waiting. Nothing starts a room here after this.
+    # Stop hearing requests and drop any claim still waiting. Nothing starts a room here after this.
     def stop_placement
       @placement&.stop
     end
@@ -405,6 +217,15 @@ module CableRoom
       worker_pool.executor.kill
     end
 
+    # Give this Host up without touching anything it holds. For a copy of a Host that a forked
+    # child inherited from its parent (see `CableRoom.after_fork!`): its rooms, locks, and threads
+    # belong to the parent, so the child must neither close those rooms nor let the `at_exit`
+    # hook above do it when the child exits. Marks it shut down so both are no-ops.
+    def abandon!
+      @shutdown = true
+      self
+    end
+
     # Ask every room to finish what it has queued and shut down, then give them up to
     # `wait` seconds to do it
     def shutdown_rooms!(wait: 15)
@@ -424,57 +245,6 @@ module CableRoom
 
     private
 
-    # Run the migrations `concurrency` at a time until `deadline`, then everything still waiting
-    # at once. Each runs on a thread of its own rather than the worker pool: a handoff blocks for
-    # up to `handoff_timeout` waiting on a peer, and the rooms still need the pool to run.
-    def run_migrations(migrations, concurrency:, deadline:)
-      queue = Queue.new
-      migrations.each { |migration| queue << migration }
-      next_migration = -> { queue.pop(true) rescue nil } # non-blocking: nil once the queue is empty
-
-      workers = Array.new([concurrency, migrations.size].min) do |i|
-        migration_thread("cable_room-drain-#{i}") do
-          while (migration = next_migration.call)
-            migration.run
-          end
-        end
-      end
-
-      workers.each do |thread|
-        remaining = deadline - monotonic_now
-        break if remaining <= 0
-        thread.join(remaining)
-      end
-
-      forced = []
-      while (migration = next_migration.call)
-        forced << migration
-      end
-      if forced.any?
-        logger.warn "drain_timeout passed with #{forced.size} room(s) still waiting; handing them all off now"
-      end
-      forced_threads = forced.map { |migration| migration_thread("cable_room-drain-forced") { migration.run } }
-
-      (workers + forced_threads).each(&:join)
-    end
-
-    def migration_thread(name)
-      thread = Thread.new do
-        yield
-      rescue => e
-        # Migration#run handles its own failures; this is for a bug in the loop itself, which
-        # must not take the whole drain down with it
-        CableRoom.report_error(e, host: self)
-      end
-      thread.name = name
-      thread.report_on_exception = false
-      thread
-    end
-
-    def monotonic_now
-      Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    end
-
     def cable_server
       ActionCable.server
     end

data/lib/cable_room/ports.rb

@@ -4,14 +4,14 @@ module CableRoom
   # listens on it. Which transport that touches depends on which side you're on, and each side
   # implements the three methods below:
   #
-  #   * Member side (RoomMembership): sending publishes on the Bus channel for the room's port
-  #     (member→room), listening is an ActionCable stream (room→member, fed by the broadcaster).
-  #   * Room side (Room::Base and Room::HostAdapter): sending goes through the configured
-  #     CableRoom::Broadcaster, listening subscribes on the Bus through the room's Host.
+  #   * Member side (RoomMembership): sending publishes on the room's Bus channel for the port
+  #     (member→room); listening is an ActionCable stream (room→member).
+  #   * Room side (Room::Base and Room::HostAdapter): sending is an ActionCable broadcast on the
+  #     port's stream; listening subscribes on the Bus through the room's Host.
   #
   # So a member's `ports[:custom] << msg` and a room's `ports[:custom].stream` meet on the Bus,
   # and a room's `ports[:custom] << msg` and a member's `ports[:custom].stream` meet on the
-  # broadcaster's streams. Neither side needs to know that.
+  # ActionCable stream named by `room_port_key`. Neither side needs to know that.
   module Ports
     extend ActiveSupport::Concern
 

data/lib/cable_room/railtie.rb

@@ -5,6 +5,20 @@ require "active_model/railtie"
 
 module CableRoom
   class Railtie < Rails::Railtie # :nodoc:
+    rake_tasks do
+    end
+
+    console do |app|
+    end
+
+    runner do
+    end
+
+    # Read CABLE_ROOM_HOST now so a bad value fails the boot, not the first room
+    initializer "cable_room.room_host" do
+      CableRoom.room_host
+    end
+
     initializer "cable_room.hook_action_cable_restart" do
       module ActionCableServerExtensions
         # ActionCable restarts when the app reloads; stop every room with it. A process that