cable_room 0.7.0.beta3 → 0.8.0.beta2

data/lib/cable_room/config.rb

@@ -1,112 +1,78 @@
 # frozen_string_literal: true
 
 module CableRoom
-  # Holds the app-wide settings for CableRoom. Get the current one with `CableRoom.config`
-  # and change it with `CableRoom.configure`:
-  #
-  #   CableRoom.configure do |c|
-  #     c.room_host   = :remote       # :inline | :remote
-  #     c.broadcaster = :anycable     # :action_cable | :anycable
-  #     c.provision_delay_ms = 20
-  #     c.drain_timeout   = 10.minutes
-  #     c.handoff_timeout = 10.seconds
-  #   end
-  #
-  # The env vars CABLE_ROOM_HOST and CABLE_ROOM_BROADCASTER override `room_host` and
-  # `broadcaster`, even when the configure block sets them. That lets one build run in
-  # both modes without a code change. With no configuration at all, every default keeps
-  # today's behavior: rooms run inline and broadcast through ActionCable.
-  class Config
-    class InvalidOption < ArgumentError; end
-
-    ROOM_HOSTS = %i[inline remote].freeze
-    BROADCASTERS = %i[action_cable anycable].freeze
-
-    ENV_OVERRIDES = {
-      room_host: "CABLE_ROOM_HOST",
-      broadcaster: "CABLE_ROOM_BROADCASTER",
-    }.freeze
-
-    DEFAULTS = {
-      room_host: :inline,
-      broadcaster: :action_cable,
-      provision_delay_ms: 20,
-      drain_timeout: 10.minutes,
-      handoff_timeout: 10.seconds,
-    }.freeze
-
-    attr_reader :room_host, :broadcaster
-    attr_accessor :provision_delay_ms, :drain_timeout, :handoff_timeout
-
-    def initialize
-      DEFAULTS.each { |name, value| instance_variable_set("@#{name}", value) }
+  class ConfigError < ArgumentError; end
+
+  ROOM_HOSTS = %i[inline remote].freeze
+  ROOM_HOST_ENV = "CABLE_ROOM_HOST"
+  DEFAULT_PROVISION_DELAY_MS = 20
+
+  class << self
+    # Where rooms run. `:inline` (the default) keeps them in the web process; `:remote` moves
+    # them to a dedicated host pool. Set it in an initializer:
+    #
+    #   CableRoom.room_host = :remote
+    #
+    # The CABLE_ROOM_HOST env var (`inline` or `remote`, any case) wins over the setter, so one
+    # build can run in either mode without a code change. The env var is read once, on first
+    # use; the Railtie reads it at boot so a bad value fails there instead of mid-request.
+    def room_host
+      @room_host_from_env = room_host_from_env unless defined?(@room_host_from_env)
+      @room_host_from_env || @room_host || :inline
     end
 
-    # Where rooms run. `:inline` keeps them in the web process; `:remote` moves them to a
-    # dedicated host pool. Accepts a Symbol or String; unknown values raise right away.
+    # Accepts a Symbol or String; unknown values raise CableRoom::ConfigError
     def room_host=(value)
-      @room_host = validate_choice!(:room_host, value, ROOM_HOSTS)
-    end
-
-    # How a room pushes messages to members. Accepts a Symbol or String; unknown values raise.
-    def broadcaster=(value)
-      @broadcaster = validate_choice!(:broadcaster, value, BROADCASTERS)
+      @room_host = cast_room_host(value) do
+        "CableRoom.room_host must be one of #{ROOM_HOSTS.map(&:inspect).join(', ')} (got #{value.inspect})"
+      end
     end
 
-    # Rooms run inside every process that joins them (the web process in a Rails app).
     def inline?
       room_host == :inline
     end
 
-    # Rooms run only in `cable_room server` processes; a web process never hosts one.
     def remote?
       room_host == :remote
     end
 
-    # Copies CABLE_ROOM_HOST and CABLE_ROOM_BROADCASTER onto this config. Blank or missing
-    # variables leave the current value alone. `env` is injectable for specs.
-    def apply_env_overrides(env = ENV)
-      ENV_OVERRIDES.each do |name, var|
-        value = env[var]
-        next if value.nil? || value.strip.empty?
+    # How long a rooms Host waits, per room it already runs, before racing for a room a member
+    # asked for (see Host::Placement). The least-loaded Host wakes first and usually wins, so
+    # this is what spreads rooms across a pool. Milliseconds; defaults to 20. `nil` restores it.
+    def provision_delay_ms
+      @provision_delay_ms || DEFAULT_PROVISION_DELAY_MS
+    end
 
-        public_send("#{name}=", value)
+    def provision_delay_ms=(value)
+      unless value.nil? || (value.is_a?(Numeric) && value >= 0)
+        raise ConfigError, "CableRoom.provision_delay_ms must be a number >= 0 (got #{value.inspect})"
       end
-      self
-    end
 
-    # Checks every knob at once. The choice knobs are already checked by their setters, so
-    # this mostly guards the timing knobs, which are plain accessors.
-    def validate!
-      validate_choice!(:room_host, room_host, ROOM_HOSTS)
-      validate_choice!(:broadcaster, broadcaster, BROADCASTERS)
-      validate_number!(:provision_delay_ms, provision_delay_ms, min: 0)
-      validate_number!(:drain_timeout, drain_timeout, min: 0, exclusive: true)
-      validate_number!(:handoff_timeout, handoff_timeout, min: 0, exclusive: true)
-      self
+      @provision_delay_ms = value
     end
 
-    def to_h
-      DEFAULTS.keys.to_h { |name| [name, public_send(name)] }
+    # Forgets both the setter and the env var, so the next read starts over. For specs.
+    def reset_room_host!
+      remove_instance_variable(:@room_host) if defined?(@room_host)
+      remove_instance_variable(:@room_host_from_env) if defined?(@room_host_from_env)
     end
 
     private
 
-    def validate_choice!(name, value, allowed)
-      symbol = value.to_s.to_sym unless value.nil?
-      return symbol if allowed.include?(symbol)
+    def room_host_from_env
+      value = ENV[ROOM_HOST_ENV]
+      return nil if value.nil? || value.strip.empty?
 
-      raise InvalidOption,
-            "CableRoom config: #{name} must be one of #{allowed.map(&:inspect).join(', ')} " \
-            "(got #{value.inspect})"
+      cast_room_host(value.strip.downcase) do
+        "#{ROOM_HOST_ENV} must be one of #{ROOM_HOSTS.join(', ')} (got #{value.inspect})"
+      end
     end
 
-    def validate_number!(name, value, min:, exclusive: false)
-      ok = value.is_a?(Numeric) && (exclusive ? value > min : value >= min)
-      return value if ok
+    def cast_room_host(value)
+      symbol = value.to_s.to_sym unless value.nil?
+      return symbol if ROOM_HOSTS.include?(symbol)
 
-      bound = exclusive ? "greater than #{min}" : "at least #{min}"
-      raise InvalidOption, "CableRoom config: #{name} must be a number #{bound} (got #{value.inspect})"
+      raise ConfigError, yield
     end
   end
 end

data/lib/cable_room/host/bus_inbound.rb

@@ -1,10 +1,8 @@
 module CableRoom
   class Host
     # How rooms receive messages: off the process-wide CableRoom::Bus (see `CableRoom.bus`), on
-    # the channels `Room::Base.inbound_channel` names. This is the only room-side code that
-    # touches the inbound transport, and it's the same in `:inline` and `:remote` mode — the only
-    # difference is which process the Host lives in. Anything with these two methods can be handed
-    # to `Host.new(inbound:)`.
+    # the channels `Bus.inbound_channel` names. This is the only room-side code that touches the
+    # inbound transport; anything with these two methods can be handed to `Host.new(inbound:)`.
     #
     # `subscribe` returns a handle that `unsubscribe` needs back. Handlers get the decoded message
     # on the Bus's single subscriber thread, in publish order, so they must be quick and hand the
@@ -12,22 +10,29 @@ module CableRoom
     # Redis confirms the subscription, so `on_live` runs on the caller's thread right before it
     # returns, and anything published after that is delivered.
     class BusInbound
-      # `bus` defaults to `CableRoom.bus` (looked up lazily so specs can swap it)
-      def initialize(bus: nil)
+      # `bus` defaults to `CableRoom.bus` (looked up lazily so specs can swap it). `timeout` is how
+      # long `subscribe` and `unsubscribe` wait for Redis to confirm.
+      def initialize(bus: nil, timeout: Bus::DEFAULT_TIMEOUT)
         @bus = bus
+        @timeout = timeout
       end
 
       def subscribe(channel, on_live: nil, &on_message)
-        bus.subscribe(channel) { |message, _channel| on_message.call(message) }
+        handle = ->(message, _channel) { on_message.call(message) }
+        bus.subscribe(channel, timeout: @timeout, &handle)
         on_live&.call
-        on_message
+        handle
       end
 
-      def unsubscribe(channel, _handle)
-        bus.unsubscribe(channel)
+      # The Bus keeps one handler per channel, so only let go of the channel while it's still
+      # ours. A runner undoing a late subscribe (see Runner#subscribe) must not drop the handler
+      # of the next runner for the same room.
+      def unsubscribe(channel, handle)
+        bus.unsubscribe(channel, handler: handle, timeout: @timeout)
       end
 
-      # The Bus behind this transport (the process-wide one unless a spec handed us another)
+      private
+
       def bus
         @bus || CableRoom.bus
       end

data/lib/cable_room/host/placement.rb

@@ -0,0 +1,261 @@
+require 'active_support/inflector'
+
+module CableRoom
+  class Host
+    # The rooms side of provisioning. In :remote a `create: true` member can't start its room, so
+    # it publishes a provision request on `Bus.provision_channel` (see
+    # RoomMembership#request_remote_room). Every rooms Host runs one Placement, which hears every
+    # request and decides whether this Host should be the one to start the room.
+    #
+    # The decision is a race weighted by load. Each Host waits `CableRoom.provision_delay_ms` for
+    # every room it already runs, plus up to one room's worth of jitter to break ties, then tries
+    # the room's Redlock once through `Host#ensure_room`. The least-loaded Host wakes first and
+    # usually wins; the rest find the lock held and do nothing. Losing is normal and quiet.
+    #
+    # Requests arrive on the Bus subscriber thread, which must stay quick, and a Bus subscribe made
+    # from that thread doesn't wait for Redis to confirm it, so a room started there would take
+    # traffic before it's listening. So `handle` only validates and schedules; the wait and the
+    # claim run later on a small executor of Placement's own. It's bounded twice: CLAIM_THREADS
+    # claims run at once, and at most MAX_PENDING wait at a time (more are dropped; the member
+    # asks again on its next ping).
+    #
+    # The request carries the member's tenant, and the claim runs switched into it, so the lock
+    # key, the inbound channel, and the stream names come out the same as on the web side under a
+    # tenant-prefixing `broadcasting_for` (PandaPal's).
+    #
+    # Specs make the race deterministic with `jitter: -> { 0 }` (any callable returning 0..1).
+    class Placement
+      # A request this Host can't act on. Logged and dropped, never raised to the Bus.
+      class InvalidRequest < ArgumentError; end
+
+      REQUEST_TYPE = "provision".freeze
+
+      # How many claims run at once. A claim is mostly a Redlock call and, for the winner, room
+      # startup (a few Bus subscribes), so a couple of threads keep up with a burst.
+      CLAIM_THREADS = 2
+
+      # How many claims may wait on their timer at once. Past this, requests are dropped.
+      MAX_PENDING = 100
+
+      # Only something shaped like a constant name gets near `safe_constantize`
+      CLASS_NAME = /\A(::)?[A-Z]\w*(::[A-Z]\w*)*\z/
+
+      attr_reader :host, :channel
+
+      # `jitter` returns a Float in 0..1 (defaults to `rand`).
+      def initialize(host, jitter: nil)
+        @host = host
+        @jitter = jitter || -> { Kernel.rand }
+        @channel = Bus.provision_channel
+
+        # Everything below is guarded by @mutex, which is never held across a call into the Bus
+        @mutex = Mutex.new
+        @idle = ConditionVariable.new
+        @pending = {} # request identity => its scheduled claim, so one room never queues twice here
+        @stopped = true
+        @handle = nil
+        @executor = nil
+      end
+
+      # Subscribe to the provision channel on the Host's inbound transport. Blocks until Redis
+      # confirms, so a request published after this returns is heard. Calling it while running
+      # does nothing.
+      def start
+        @mutex.synchronize do
+          return self unless @stopped
+
+          @stopped = false
+          @executor = Concurrent::ThreadPoolExecutor.new(
+            name: "cable_room-placement",
+            min_threads: 0,
+            max_threads: CLAIM_THREADS,
+            max_queue: 0, # MAX_PENDING bounds what can be queued
+          )
+        end
+
+        handle = host.inbound.subscribe(channel) { |request| self.handle(request) }
+        @mutex.synchronize { @handle = handle }
+        self
+      rescue StandardError
+        stop
+        raise
+      end
+
+      # Stop hearing requests and drop every claim still waiting on its timer, then wait (up to
+      # `wait` seconds) for a claim already mid-attempt to finish, so nothing starts a room here
+      # after this returns.
+      def stop(wait: 5)
+        handle, executor = @mutex.synchronize do
+          return self if @stopped
+
+          @stopped = true
+          [@handle, @executor].tap do
+            @handle = nil
+            @executor = nil
+          end
+        end
+
+        host.inbound.unsubscribe(channel, handle) if handle
+
+        # A cancelled task never runs its block (and so never its `finish`), so forget it here
+        tasks = @mutex.synchronize { @pending.to_a }
+        tasks.each { |id, task| finish(id) if task.cancel }
+        wait_idle(timeout: wait)
+        executor&.shutdown
+        self
+      end
+
+      def running?
+        @mutex.synchronize { !@stopped }
+      end
+
+      # How many claims are waiting on their timer or mid-attempt right now
+      def pending_count
+        @mutex.synchronize { @pending.size }
+      end
+
+      # Block until every scheduled claim has finished (won, lost, or been cancelled). Returns
+      # false if `timeout` passes first.
+      def wait_idle(timeout: 5)
+        deadline = monotonic_now + timeout
+        @mutex.synchronize do
+          until @pending.empty?
+            remaining = deadline - monotonic_now
+            return false if remaining <= 0
+            @idle.wait(@mutex, remaining)
+          end
+        end
+        true
+      end
+
+      # One request off the Bus, on its subscriber thread. Only the cheap checks happen here; the
+      # delayed claim is scheduled on the executor. Returns the scheduled task, or nil when there
+      # is nothing to do.
+      def handle(request)
+        room_class, serialized_key, tenant = parse(request)
+        id = [room_class.name, serialized_key, tenant]
+
+        return skip(id, "host is shutting down") if host.shutdown?
+
+        load = host.open_rooms_count
+        delay = delay_for(load)
+
+        @mutex.synchronize do
+          return skip(id, "placement is stopped") if @stopped
+          return skip(id, "a claim is already pending here") if @pending.key?(id)
+          return skip(id, "#{MAX_PENDING} claims are already pending here") if @pending.size >= MAX_PENDING
+
+          task = Concurrent::ScheduledTask.new(delay, executor: @executor) do
+            claim(room_class, serialized_key, tenant, delay: delay, load: load)
+          ensure
+            finish(id)
+          end
+          @pending[id] = task
+          task.execute
+        end
+      rescue InvalidRequest => e
+        logger.warn "Dropping provision request: #{e.message}"
+        nil
+      end
+
+      # Seconds to wait before racing for a room, given how many rooms this Host already runs:
+      # `provision_delay_ms × (load + jitter)`, jitter in 0..1. A Host with more rooms always waits
+      # longer than one with fewer; the jitter only orders Hosts with the same load.
+      def delay_for(load)
+        CableRoom.provision_delay_ms / 1000.0 * (load + @jitter.call.to_f)
+      end
+
+      private
+
+      # After the delay, on the executor. Re-checks what may have changed while we waited, then
+      # tries the room's lock once. The key is deserialized here, in the tenant, because a record
+      # key is a database lookup.
+      def claim(room_class, serialized_key, tenant, delay:, load:)
+        return if stopped? || host.shutdown?
+
+        CableRoom.with_app_executor do
+          CableRoom.with_tenant(tenant) do
+            key = deserialize_key(serialized_key)
+            next if running_here?(room_class, key, tenant)
+
+            payload = { host: host, room_class: room_class, room_key: key, tenant: tenant, delay: delay, open_rooms: load }
+            claimed = ActiveSupport::Notifications.instrument("provision_claimed.cable_room", payload) do |p|
+              p[:claimed] = host.ensure_room(room_class, key, tenant: tenant)
+            end
+
+            name = "#{room_class.name}[#{key}]"
+            if claimed
+              logger.info "Claimed #{name} after #{(delay * 1000).round}ms with #{load} room(s) open"
+            else
+              logger.debug "Lost the race for #{name}: another host holds its lock"
+            end
+          end
+        end
+      rescue InvalidRequest => e
+        logger.warn "Dropping provision request: #{e.message}"
+      rescue StandardError => e
+        logger.error "Failed to claim #{room_class.name}: #{e.class}: #{e.message}"
+        CableRoom.report_error(e, placement: self, room_class: room_class, room_key: serialized_key, tenant: tenant)
+      end
+
+      def finish(id)
+        @mutex.synchronize do
+          @pending.delete(id)
+          @idle.broadcast if @pending.empty?
+        end
+      end
+
+      def skip(id, why)
+        logger.debug "Ignoring provision request for #{id.inspect}: #{why}"
+        nil
+      end
+
+      def stopped?
+        @mutex.synchronize { @stopped }
+      end
+
+      def running_here?(room_class, key, tenant)
+        host.runners.any? { |r| r.room_class == room_class && r.key == key && r.tenant == tenant }
+      end
+
+      # Check a decoded Bus message's shape and find its Room class. The class name comes off the
+      # wire, so only a CableRoom::Room::Base subclass is accepted: `constantize` on an arbitrary
+      # string is not something a request should be able to do.
+      def parse(request)
+        raise InvalidRequest, "not a Hash: #{request.inspect.truncate(200)}" unless request.is_a?(Hash)
+
+        type = request["type"]
+        raise InvalidRequest, "expected a #{REQUEST_TYPE.inspect} request (got #{type.inspect.truncate(100)})" unless type == REQUEST_TYPE
+
+        name = request["room_class"]
+        klass = ActiveSupport::Inflector.safe_constantize(name) if name.is_a?(String) && name.match?(CLASS_NAME)
+        unless klass.is_a?(Class) && klass < Room::Base
+          raise InvalidRequest, "#{name.inspect.truncate(100)} is not a CableRoom::Room::Base subclass"
+        end
+
+        # The key travels the way `extra` does (ActiveJob's argument serializer): a one-item list
+        key = request["room_key"]
+        raise InvalidRequest, "room_key must be a serialized one-item list (got #{key.inspect.truncate(100)})" unless key.is_a?(Array) && key.size == 1
+
+        tenant = request["tenant"]
+        raise InvalidRequest, "tenant must be a String or null (got #{tenant.inspect.truncate(100)})" unless tenant.nil? || tenant.is_a?(String)
+
+        [klass, key, tenant]
+      end
+
+      def deserialize_key(serialized)
+        ::ActiveJob::Arguments.deserialize(serialized).first
+      rescue StandardError => e
+        raise InvalidRequest, "room_key can't be deserialized: #{e.class}: #{e.message}"
+      end
+
+      def logger
+        host.logger
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end