cable_room 0.6.2.beta1 → 0.7.0.beta2

data/lib/cable_room/host.rb

@@ -0,0 +1,482 @@
+require 'socket'
+require_relative 'host/worker_pool'
+require_relative 'host/bus_inbound'
+require_relative 'host/runner'
+require_relative 'host/supervisor'
+
+module CableRoom
+  # One Host per process. It owns every Room running here: the worker pool their work runs on,
+  # the scheduler behind `periodically` timers and the watchdog beat, the inbound transport they
+  # receive on, the list of open rooms, and shutting all of them down when the process exits.
+  #
+  # Each room gets a Host::Runner, which keeps that room's own ordered work queue. A room is only
+  # ever processed by one thread at a time, in arrival order. The Host is what lets many rooms
+  # share one pool of threads without breaking that promise.
+  class Host
+    BEAT_INTERVAL = 5.seconds
+
+    # Raised when a process that doesn't host rooms asks for its Host. With `room_host = :remote`
+    # only `cable_room server` hosts rooms; a web process must never start one.
+    class NotHosting < StandardError; end
+
+    NOT_HOSTING_MESSAGE =
+      "This process doesn't host rooms: CableRoom.config.room_host is :remote, so rooms run " \
+      "under `cable_room server`. Start rooms there (or call CableRoom::Host.start! first if this " \
+      "process really should host them).".freeze
+
+    @start_mutex = Mutex.new
+
+    class << self
+      # The Host this process runs, or nil when it runs none (a web process in :remote, or an
+      # :inline process that hasn't started a room yet). Use this for introspection and cleanup,
+      # where "no rooms here" is a normal answer.
+      def current
+        @current
+      end
+
+      # The Host this process runs. In :inline every process hosts rooms, so the first caller
+      # builds it. In :remote only a process that declared itself a rooms host with `start!`
+      # (`cable_room server`) has one; anyone else gets NotHosting, so a room can never quietly
+      # start in the web tier because some code path reached for the Host.
+      def instance
+        current || (CableRoom.config.inline? ? start! : raise(NotHosting, NOT_HOSTING_MESSAGE))
+      end
+
+      # Make this process a rooms host. Builds the Host once and starts its provisioning listener,
+      # so this process hears `create: true` requests from then on; later calls return the same
+      # Host. `cable_room server` calls this explicitly; in :inline `instance` calls it for you.
+      def start!
+        @start_mutex.synchronize { @current ||= new.tap(&:start_placement) }
+      end
+
+      # Swap this process's Host for `host` (nil allowed) and return the previous one. Only specs
+      # that play two process roles in one Ruby process should need this (see RoomHostHarness):
+      # the "web" side has to see no Host while another Host runs the rooms.
+      def replace_current(host)
+        @start_mutex.synchronize { @current.tap { @current = host } }
+      end
+    end
+
+    # How many rooms `drain!` hands off at once before `drain_timeout`. Each handoff costs the
+    # adopting peer a full room restore (its startup chain, often database reads), so a host with
+    # hundreds of rooms must not offer them all in the same second; and each one holds a thread
+    # here for up to `handoff_timeout`. Four keeps a big host draining in well under the default
+    # ten minutes (a handoff normally takes well under a second) without swamping the fleet.
+    DRAIN_CONCURRENCY = 4
+
+    # What `drain!` returns: how the rooms ended up, and how long it took. `migrated` and
+    # `closed` are Migration objects (see their `state` and `room`).
+    DrainResult = Struct.new(:reason, :migrated, :closed, :duration, keyword_init: true) do
+      def rooms
+        migrated.size + closed.size
+      end
+    end
+
+    attr_reader :scheduler, :inbound, :placement
+
+    # True once this host has started shutting down for good and is handing its rooms off (see
+    # `drain!`). A draining host never claims a provision request, so it can't take on a room it
+    # would only have to hand off again.
+    attr_writer :draining
+
+    delegate :logger, to: :cable_server
+
+    class << self
+      # Register a block to run when a `drain!` finishes, with the Host and its DrainResult. An
+      # app that scales in through an ASG termination lifecycle hook completes the hook here
+      # (`complete-lifecycle-action`), once every room has moved or closed. Blocks run on the
+      # draining thread before `drain!` returns; one that raises is reported and doesn't stop
+      # the others.
+      def after_drain(&blk)
+        raise ArgumentError, "after_drain needs a block" unless blk
+        after_drain_callbacks << blk
+        blk
+      end
+
+      def after_drain_callbacks
+        @after_drain_callbacks ||= []
+      end
+    end
+
+    # `inbound` is how rooms receive messages: by default the process-wide Bus, so a room hears
+    # its members the same way whether it runs inside the web process or on a remote host.
+    def initialize(inbound: BusInbound.new)
+      @runners = Set.new
+      @monitor = Monitor.new
+      @inbound = inbound
+      @draining = false
+      @drain_result = nil
+      @drain_thread = nil
+      @migrations = []
+      @scheduler = Rufus::Scheduler.new
+
+      # A rooms host that leaves with rooms still open (a crash on the main thread, say; on a
+      # SIGTERM `cable_room server` has already drained by now, so this is a no-op) gives them
+      # the same chance to move as a planned shutdown would. In :inline there is no fleet of
+      # peers by design (see `drain!`), so exit closes rooms the way it always has.
+      at_exit do
+        logger.info "Shutting down CableRoom"
+        if CableRoom.config.remote? && !shutdown? && runners.any?
+          drain!(reason: "process exit")
+        else
+          shutdown!
+        end
+      end
+
+      # Every few seconds each room renews its Redlock and checks its watchdog
+      scheduler.every(BEAT_INTERVAL) do
+        each_runner(&:beat)
+      end
+    end
+
+    def worker_pool
+      @worker_pool || @monitor.synchronize do
+        @worker_pool ||= WorkerPool.new(max_size: cable_server.config.worker_pool_size)
+      end
+    end
+
+    # Start `room_class`'s room for `key` on this Host if nobody else is running it. Takes the
+    # room's Redlock first, so exactly one process cluster-wide wins; returns false when the lock
+    # is already held. This is the host-side way to bring a room up: `Room::Base.ensure` calls it
+    # on the process Host, Placement calls it for a provision request, and `cable_room server`
+    # (or a spec harness) can call it on a Host it owns without any member asking.
+    #
+    # If a snapshot of the room is on offer (another host is handing it off, see
+    # CableRoom::Migration), winning the lock means adopting it: the room comes back here with
+    # its members and state rather than starting fresh underneath the old host. The check is
+    # made after the lock is won, so it can't be raced. `handoff_only: true` (a handoff request)
+    # starts nothing when there is no snapshot to adopt — it has expired, or the old host took
+    # the room back — and lets the lock go.
+    def ensure_room(room_class, key = nil, handoff_only: false, tenant: nil)
+      lock_key = room_class.room_port_key(key)
+      lock_info = CableRoom.lock_manager.lock(lock_key, room_class::LOCK_DURATION.in_milliseconds)
+      return false unless lock_info
+
+      snapshot = bus.get(Bus.snapshot_key(lock_key))
+      return Migration.adopt(self, room_class, key, lock_info: lock_info, snapshot: snapshot) if snapshot
+
+      if handoff_only
+        logger.info "No snapshot to adopt for #{lock_key}; letting the lock go"
+        CableRoom.lock_manager.unlock(lock_info)
+        return false
+      end
+
+      start_room(
+        room_class,
+        key,
+        lock_info,
+        watchdog_interval: room_class::WATCH_DOG_INTERVAL,
+        lock_duration: room_class::LOCK_DURATION,
+        tenant: tenant,
+      )
+
+      true
+    rescue => e
+      CableRoom.lock_manager.unlock(lock_info) if lock_info
+      raise e
+    end
+
+    # Start running a room here. The caller (`ensure_room`) has already taken the room's Redlock;
+    # the runner renews it from now on and releases it when the room dies. `tenant:` comes from the
+    # member's provision request (see RoomMembership#initialize / #maybe_provision_room and
+    # Placement#claim), already defaulted there to the ambient tenant if the app didn't pass one
+    # explicitly -- nil past that point just means no Apartment multi-tenancy is in play.
+    def start_room(room_class, key, lock_info, watchdog_interval:, lock_duration:, tenant: nil)
+      runner = Runner.new(self, room_class, key, lock_info, watchdog_interval:, lock_duration:, tenant:)
+      runner.start!
+      runner
+    end
+
+    # Bring a room back from a CableRoom::Snapshot on this Host, the way a migration's adopter
+    # does. With no `lock_info`, takes the room's Redlock first and returns false if someone else
+    # holds it — the same rule as `ensure_room`. With one, the caller has already claimed the lock.
+    # Raises Snapshot::UnknownVersion or Snapshot::UnknownRoomClass before anything is built if
+    # the snapshot can't be used here. `hold_inbound: true` brings the room up frozen, holding
+    # its inbound until `Runner#thaw!` (see Runner#restore!).
+    #
+    # If the room fails to come up, the lock is released either way — the caller's too. A room
+    # that can't start here must not keep its key locked: the migration that handed it over
+    # takes the lock back and restarts it as soon as the lock is free.
+    def restore_room(snapshot, lock_info = nil, hold_inbound: false)
+      snapshot = Snapshot.validate!(snapshot)
+      room_class = Snapshot.room_class_for(snapshot)
+      key = Snapshot.key_for(snapshot)
+      lock_key = room_class.room_port_key(key)
+
+      if lock_info
+        unless lock_info[:resource] == lock_key
+          raise ArgumentError, "lock_info is for #{lock_info[:resource].inspect}, but the snapshot is #{lock_key}"
+        end
+      else
+        lock_info = CableRoom.lock_manager.lock(lock_key, room_class::LOCK_DURATION.in_milliseconds)
+        return false unless lock_info
+      end
+
+      runner = Runner.new(
+        self,
+        room_class,
+        key,
+        lock_info,
+        watchdog_interval: room_class::WATCH_DOG_INTERVAL,
+        lock_duration: room_class::LOCK_DURATION,
+        tenant: snapshot["tenant"],
+      )
+      runner.restore!(snapshot, hold_inbound: hold_inbound)
+      runner
+    rescue => e
+      # `restore!` releases the lock itself when the room fails to come up; this covers a failure
+      # before the runner exists
+      CableRoom.lock_manager.unlock(lock_info) if lock_info && runner.nil?
+      raise e
+    end
+
+    # A name for this host in migration traffic and instrumentation (`from_host`, `to_host`):
+    # machine, pid, and a few random characters, so two hosts in one process (a spec) differ too.
+    def id
+      @id ||= "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.hex(3)}"
+    end
+
+    # The Bus this host's rooms listen on, for the migration's list and key operations. Falls
+    # back to the process-wide Bus when `inbound` isn't Bus-backed.
+    def bus
+      inbound.respond_to?(:bus) ? inbound.bus : CableRoom.bus
+    end
+
+    def track(runner)
+      @monitor.synchronize do
+        raise "Cannot add Room after shutdown" if @shutdown
+        @runners << runner
+      end
+    end
+
+    def untrack(runner)
+      @monitor.synchronize { @runners.delete(runner) }
+    end
+
+    # A snapshot, so callers can iterate while rooms come and go
+    def runners
+      @monitor.synchronize { @runners.to_a }
+    end
+
+    def rooms
+      runners.map(&:room)
+    end
+
+    # How many member ports are attached to rooms on this Host right now. This is the load
+    # number the design scales the rooms pool on, so an app can publish it as a gauge. It reads
+    # each room's port table from outside the room's thread; a count that's a moment stale is
+    # fine for a metric.
+    def open_ports
+      rooms.sum(&:open_port_count)
+    end
+
+    # How many rooms this Host runs right now. This is the load number provisioning weights its
+    # delay by: a host with more rooms waits longer before racing for a new one.
+    def open_rooms_count
+      @monitor.synchronize { @runners.size }
+    end
+
+    def each_runner(&blk)
+      runners.each(&blk)
+    end
+
+    def draining?
+      @draining
+    end
+
+    # True once `drain!` has finished (every room moved or closed, and the host is shut down).
+    def drained?
+      !@drain_result.nil?
+    end
+
+    # The result of the drain, or nil until `drain!` has finished.
+    def drain_result
+      @drain_result
+    end
+
+    # Every CableRoom::Migration this host's drain has started, oldest room first. Each one's
+    # `state` says where it is; the list is complete once `drained?`.
+    def migrations
+      @monitor.synchronize { @migrations.dup }
+    end
+
+    # Planned shutdown: move every room here to a peer host, then shut down. This is what SIGTERM
+    # means to `cable_room server` (see CableRoom::Migration for the per-room protocol):
+    #
+    #   1. Mark the host draining, so its Placement claims nothing more (a room it took now would
+    #      only have to move again), and instrument `host_draining.cable_room`.
+    #   2. Hand rooms off oldest first, DRAIN_CONCURRENCY at a time. A room nobody adopts within
+    #      `handoff_timeout` closes with `room_closed`; so does one that can't be snapshotted.
+    #   3. When `drain_timeout` passes with rooms still waiting their turn, offer all of them at
+    #      once — the same path, just no longer throttled — so the drain ends within about one
+    #      more `handoff_timeout` whatever the fleet does.
+    #   4. `shutdown!`, run the `Host.after_drain` callbacks, and return a DrainResult.
+    #
+    # Blocks until done. Runs once: a second call (from another thread, or `at_exit` after
+    # `cable_room server` has already drained) waits for the first and returns its result.
+    # `drain_timeout` and `handoff_timeout` default to the config's; specs pass short ones.
+    #
+    # In :inline the peers are the other web processes hosting rooms, if any. The design gives
+    # :inline no fleet of its own, so the gem doesn't drain there on exit (see `at_exit`); an app
+    # running several inline processes may still call this from its own signal handling.
+    def drain!(reason:, drain_timeout: nil, handoff_timeout: nil, concurrency: DRAIN_CONCURRENCY)
+      other = @monitor.synchronize do
+        return @drain_result if @drain_result
+        @drain_thread.tap { @drain_thread ||= Thread.current }
+      end
+      if other
+        other.join unless other == Thread.current
+        return @drain_result
+      end
+
+      drain_timeout ||= CableRoom.config.drain_timeout
+      handoff_timeout ||= CableRoom.config.handoff_timeout
+      started = monotonic_now
+      self.draining = true
+
+      queued = runners.sort_by(&:started_at)
+      migrations = queued.map { |runner| Migration.new(self, runner, reason: reason, handoff_timeout: handoff_timeout) }
+      @monitor.synchronize { @migrations = migrations }
+      logger.info "Draining #{migrations.size} room(s): #{reason}"
+
+      payload = { host: self, rooms: queued.map(&:room), reason: reason }
+      ActiveSupport::Notifications.instrument("host_draining.cable_room", payload) do |event|
+        run_migrations(migrations, concurrency: concurrency, deadline: started + drain_timeout.to_f)
+        event[:migrated] = migrations.count(&:migrated?)
+        event[:closed] = migrations.count { |m| !m.migrated? }
+      end
+
+      shutdown!
+
+      result = DrainResult.new(
+        reason: reason,
+        migrated: migrations.select(&:migrated?),
+        closed: migrations.reject(&:migrated?),
+        duration: monotonic_now - started,
+      )
+      logger.info "Drained: #{result.migrated.size} room(s) migrated, #{result.closed.size} closed, in #{result.duration.round(2)}s"
+      @monitor.synchronize { @drain_result = result }
+
+      self.class.after_drain_callbacks.each do |callback|
+        callback.call(self, result)
+      rescue => e
+        CableRoom.report_error(e, host: self, drain_result: result)
+      end
+
+      result
+    end
+
+    def shutdown?
+      @shutdown
+    end
+
+    # Start listening for provision requests (see CableRoom::Placement). `Host.start!` does this
+    # for the process Host; a Host built with `new` (a spec harness) calls it itself. Pass a
+    # Placement to use one with non-default settings; calling it again re-arms the current one.
+    def start_placement(placement = nil)
+      previous = @monitor.synchronize do
+        raise "Cannot start placement after shutdown" if @shutdown
+        @placement.tap { @placement = placement || @placement || Placement.new(self) }
+      end
+      previous.stop if previous && previous != @placement
+      @placement.start
+      @placement
+    end
+
+    # Stop listening and drop any claim still waiting. Nothing starts a room here after this.
+    def stop_placement
+      @placement&.stop
+    end
+
+    # Stop everything: the beat, every room (gracefully, with a bounded wait), then the pool.
+    # Runs once; later calls do nothing.
+    def shutdown!
+      @monitor.synchronize do
+        return if @shutdown
+        @shutdown = true
+      end
+
+      # Stop claiming first: a room started after this point would only be shut down again
+      stop_placement
+      scheduler.shutdown
+      shutdown_rooms!
+      worker_pool.executor.shutdown
+      worker_pool.executor.wait_for_termination(5)
+      worker_pool.executor.kill
+    end
+
+    # Ask every room to finish what it has queued and shut down, then give them up to
+    # `wait` seconds to do it
+    def shutdown_rooms!(wait: 15)
+      each_runner { |runner| runner.initiate_shutdown("Server shutting down") }
+
+      wait.times do
+        break if runners.empty?
+        sleep 1
+      end
+    end
+
+    # Stop every room right now, dropping whatever work is still queued. ActionCable's `restart`
+    # calls this when the app reloads.
+    def stop_all_rooms!
+      each_runner(&:stop!)
+    end
+
+    private
+
+    # Run the migrations `concurrency` at a time until `deadline`, then everything still waiting
+    # at once. Each runs on a thread of its own rather than the worker pool: a handoff blocks for
+    # up to `handoff_timeout` waiting on a peer, and the rooms still need the pool to run.
+    def run_migrations(migrations, concurrency:, deadline:)
+      queue = Queue.new
+      migrations.each { |migration| queue << migration }
+      next_migration = -> { queue.pop(true) rescue nil } # non-blocking: nil once the queue is empty
+
+      workers = Array.new([concurrency, migrations.size].min) do |i|
+        migration_thread("cable_room-drain-#{i}") do
+          while (migration = next_migration.call)
+            migration.run
+          end
+        end
+      end
+
+      workers.each do |thread|
+        remaining = deadline - monotonic_now
+        break if remaining <= 0
+        thread.join(remaining)
+      end
+
+      forced = []
+      while (migration = next_migration.call)
+        forced << migration
+      end
+      if forced.any?
+        logger.warn "drain_timeout passed with #{forced.size} room(s) still waiting; handing them all off now"
+      end
+      forced_threads = forced.map { |migration| migration_thread("cable_room-drain-forced") { migration.run } }
+
+      (workers + forced_threads).each(&:join)
+    end
+
+    def migration_thread(name)
+      thread = Thread.new do
+        yield
+      rescue => e
+        # Migration#run handles its own failures; this is for a bug in the loop itself, which
+        # must not take the whole drain down with it
+        CableRoom.report_error(e, host: self)
+      end
+      thread.name = name
+      thread.report_on_exception = false
+      thread
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    def cable_server
+      ActionCable.server
+    end
+  end
+end

data/lib/cable_room/membership_store.rb

@@ -0,0 +1,105 @@
+module CableRoom
+  # Where a channel keeps its RoomMemberships between calls. `RoomMember#_room_memberships` is one
+  # of these, and RoomMembership adds and removes itself from it.
+  #
+  # Under ActionCable the channel object lives as long as the socket, so an in-memory set is the
+  # whole story: this base class. Under anycable-rails every RPC call (subscribe, message,
+  # unsubscribe, disconnect) builds a fresh channel instance and never re-runs `subscribed`, so an
+  # in-memory set is empty by the time `hello` or `receive` arrive. The AnyCable subclass writes
+  # each membership's identity into the channel's AnyCable state, which anycable-go hands back on
+  # every call, and rebuilds the memberships from it on first use.
+  class MembershipStore
+    include Enumerable
+
+    def self.for(channel)
+      channel.anycable_channel? ? AnyCable.new(channel) : new(channel)
+    end
+
+    def initialize(channel)
+      @channel = channel
+      @memberships = Set.new
+    end
+
+    def <<(membership)
+      memberships << membership
+      persist!
+      self
+    end
+
+    def delete(membership)
+      memberships.delete(membership)
+      persist!
+    end
+
+    def clear
+      memberships.clear
+      persist!
+    end
+
+    def each(&blk)
+      memberships.each(&blk)
+    end
+
+    def to_a
+      memberships.to_a
+    end
+
+    def empty?
+      memberships.empty?
+    end
+
+    def size
+      memberships.size
+    end
+
+    # Write the memberships back to wherever they live. Call it after changing something about a
+    # membership that has to survive the call (RoomMembership#hello! does). In memory there is
+    # nothing to do.
+    def persist!; end
+
+    protected
+
+    def memberships
+      @memberships
+    end
+
+    # The anycable-rails flavor: the memberships also live in the channel's AnyCable state
+    # (`state_attr_accessor`), as one JSON string of RoomMembership#persisted_identity records.
+    #
+    # One string rather than an array of hashes because anycable-rails runs every value in the
+    # state through its own serializer, which locates any GlobalID-looking string through the
+    # database. The `extra` a member joined with is ActiveJob-serialized and may well contain one,
+    # and the room is the only party that should deserialize it. Encoding the records ourselves
+    # keeps that serializer out of them.
+    class AnyCable < MembershipStore
+      STATE_ATTRIBUTE = :_cable_room_memberships
+
+      def persist!
+        # The accessor is private on the channel (see RoomMember) so it can't be performed as a
+        # channel action; `send` is deliberate.
+        @channel.send(:"#{STATE_ATTRIBUTE}=", ActiveSupport::JSON.encode(memberships.map(&:persisted_identity)))
+      end
+
+      protected
+
+      # Rebuild on first use, whatever the use is. The subscribe call that creates the first
+      # membership finds nothing to rebuild; every later call finds what subscribe (and hello)
+      # wrote. Restored memberships are added straight to the set: they exist already as far as
+      # anycable-go is concerned, so there is nothing to persist and no stream to open.
+      def memberships
+        restore! unless @restored
+        super
+      end
+
+      def restore!
+        @restored = true
+        raw = @channel.send(STATE_ATTRIBUTE)
+        return if raw.blank?
+
+        ActiveSupport::JSON.decode(raw).each do |record|
+          @memberships << RoomMembership.restore(@channel, record)
+        end
+      end
+    end
+  end
+end