cable_room 0.8.0.beta1 → 0.8.0.beta2

data/lib/cable_room/config.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module CableRoom
+  class ConfigError < ArgumentError; end
+
+  ROOM_HOSTS = %i[inline remote].freeze
+  ROOM_HOST_ENV = "CABLE_ROOM_HOST"
+  DEFAULT_PROVISION_DELAY_MS = 20
+
+  class << self
+    # Where rooms run. `:inline` (the default) keeps them in the web process; `:remote` moves
+    # them to a dedicated host pool. Set it in an initializer:
+    #
+    #   CableRoom.room_host = :remote
+    #
+    # The CABLE_ROOM_HOST env var (`inline` or `remote`, any case) wins over the setter, so one
+    # build can run in either mode without a code change. The env var is read once, on first
+    # use; the Railtie reads it at boot so a bad value fails there instead of mid-request.
+    def room_host
+      @room_host_from_env = room_host_from_env unless defined?(@room_host_from_env)
+      @room_host_from_env || @room_host || :inline
+    end
+
+    # Accepts a Symbol or String; unknown values raise CableRoom::ConfigError
+    def room_host=(value)
+      @room_host = cast_room_host(value) do
+        "CableRoom.room_host must be one of #{ROOM_HOSTS.map(&:inspect).join(', ')} (got #{value.inspect})"
+      end
+    end
+
+    def inline?
+      room_host == :inline
+    end
+
+    def remote?
+      room_host == :remote
+    end
+
+    # How long a rooms Host waits, per room it already runs, before racing for a room a member
+    # asked for (see Host::Placement). The least-loaded Host wakes first and usually wins, so
+    # this is what spreads rooms across a pool. Milliseconds; defaults to 20. `nil` restores it.
+    def provision_delay_ms
+      @provision_delay_ms || DEFAULT_PROVISION_DELAY_MS
+    end
+
+    def provision_delay_ms=(value)
+      unless value.nil? || (value.is_a?(Numeric) && value >= 0)
+        raise ConfigError, "CableRoom.provision_delay_ms must be a number >= 0 (got #{value.inspect})"
+      end
+
+      @provision_delay_ms = value
+    end
+
+    # Forgets both the setter and the env var, so the next read starts over. For specs.
+    def reset_room_host!
+      remove_instance_variable(:@room_host) if defined?(@room_host)
+      remove_instance_variable(:@room_host_from_env) if defined?(@room_host_from_env)
+    end
+
+    private
+
+    def room_host_from_env
+      value = ENV[ROOM_HOST_ENV]
+      return nil if value.nil? || value.strip.empty?
+
+      cast_room_host(value.strip.downcase) do
+        "#{ROOM_HOST_ENV} must be one of #{ROOM_HOSTS.join(', ')} (got #{value.inspect})"
+      end
+    end
+
+    def cast_room_host(value)
+      symbol = value.to_s.to_sym unless value.nil?
+      return symbol if ROOM_HOSTS.include?(symbol)
+
+      raise ConfigError, yield
+    end
+  end
+end

data/lib/cable_room/host/bus_inbound.rb

@@ -0,0 +1,41 @@
+module CableRoom
+  class Host
+    # How rooms receive messages: off the process-wide CableRoom::Bus (see `CableRoom.bus`), on
+    # the channels `Bus.inbound_channel` names. This is the only room-side code that touches the
+    # inbound transport; anything with these two methods can be handed to `Host.new(inbound:)`.
+    #
+    # `subscribe` returns a handle that `unsubscribe` needs back. Handlers get the decoded message
+    # on the Bus's single subscriber thread, in publish order, so they must be quick and hand the
+    # real work off; Host::Runner does that by queueing it on the room. `subscribe` blocks until
+    # Redis confirms the subscription, so `on_live` runs on the caller's thread right before it
+    # returns, and anything published after that is delivered.
+    class BusInbound
+      # `bus` defaults to `CableRoom.bus` (looked up lazily so specs can swap it). `timeout` is how
+      # long `subscribe` and `unsubscribe` wait for Redis to confirm.
+      def initialize(bus: nil, timeout: Bus::DEFAULT_TIMEOUT)
+        @bus = bus
+        @timeout = timeout
+      end
+
+      def subscribe(channel, on_live: nil, &on_message)
+        handle = ->(message, _channel) { on_message.call(message) }
+        bus.subscribe(channel, timeout: @timeout, &handle)
+        on_live&.call
+        handle
+      end
+
+      # The Bus keeps one handler per channel, so only let go of the channel while it's still
+      # ours. A runner undoing a late subscribe (see Runner#subscribe) must not drop the handler
+      # of the next runner for the same room.
+      def unsubscribe(channel, handle)
+        bus.unsubscribe(channel, handler: handle, timeout: @timeout)
+      end
+
+      private
+
+      def bus
+        @bus || CableRoom.bus
+      end
+    end
+  end
+end

data/lib/cable_room/host/placement.rb

@@ -0,0 +1,261 @@
+require 'active_support/inflector'
+
+module CableRoom
+  class Host
+    # The rooms side of provisioning. In :remote a `create: true` member can't start its room, so
+    # it publishes a provision request on `Bus.provision_channel` (see
+    # RoomMembership#request_remote_room). Every rooms Host runs one Placement, which hears every
+    # request and decides whether this Host should be the one to start the room.
+    #
+    # The decision is a race weighted by load. Each Host waits `CableRoom.provision_delay_ms` for
+    # every room it already runs, plus up to one room's worth of jitter to break ties, then tries
+    # the room's Redlock once through `Host#ensure_room`. The least-loaded Host wakes first and
+    # usually wins; the rest find the lock held and do nothing. Losing is normal and quiet.
+    #
+    # Requests arrive on the Bus subscriber thread, which must stay quick, and a Bus subscribe made
+    # from that thread doesn't wait for Redis to confirm it, so a room started there would take
+    # traffic before it's listening. So `handle` only validates and schedules; the wait and the
+    # claim run later on a small executor of Placement's own. It's bounded twice: CLAIM_THREADS
+    # claims run at once, and at most MAX_PENDING wait at a time (more are dropped; the member
+    # asks again on its next ping).
+    #
+    # The request carries the member's tenant, and the claim runs switched into it, so the lock
+    # key, the inbound channel, and the stream names come out the same as on the web side under a
+    # tenant-prefixing `broadcasting_for` (PandaPal's).
+    #
+    # Specs make the race deterministic with `jitter: -> { 0 }` (any callable returning 0..1).
+    class Placement
+      # A request this Host can't act on. Logged and dropped, never raised to the Bus.
+      class InvalidRequest < ArgumentError; end
+
+      REQUEST_TYPE = "provision".freeze
+
+      # How many claims run at once. A claim is mostly a Redlock call and, for the winner, room
+      # startup (a few Bus subscribes), so a couple of threads keep up with a burst.
+      CLAIM_THREADS = 2
+
+      # How many claims may wait on their timer at once. Past this, requests are dropped.
+      MAX_PENDING = 100
+
+      # Only something shaped like a constant name gets near `safe_constantize`
+      CLASS_NAME = /\A(::)?[A-Z]\w*(::[A-Z]\w*)*\z/
+
+      attr_reader :host, :channel
+
+      # `jitter` returns a Float in 0..1 (defaults to `rand`).
+      def initialize(host, jitter: nil)
+        @host = host
+        @jitter = jitter || -> { Kernel.rand }
+        @channel = Bus.provision_channel
+
+        # Everything below is guarded by @mutex, which is never held across a call into the Bus
+        @mutex = Mutex.new
+        @idle = ConditionVariable.new
+        @pending = {} # request identity => its scheduled claim, so one room never queues twice here
+        @stopped = true
+        @handle = nil
+        @executor = nil
+      end
+
+      # Subscribe to the provision channel on the Host's inbound transport. Blocks until Redis
+      # confirms, so a request published after this returns is heard. Calling it while running
+      # does nothing.
+      def start
+        @mutex.synchronize do
+          return self unless @stopped
+
+          @stopped = false
+          @executor = Concurrent::ThreadPoolExecutor.new(
+            name: "cable_room-placement",
+            min_threads: 0,
+            max_threads: CLAIM_THREADS,
+            max_queue: 0, # MAX_PENDING bounds what can be queued
+          )
+        end
+
+        handle = host.inbound.subscribe(channel) { |request| self.handle(request) }
+        @mutex.synchronize { @handle = handle }
+        self
+      rescue StandardError
+        stop
+        raise
+      end
+
+      # Stop hearing requests and drop every claim still waiting on its timer, then wait (up to
+      # `wait` seconds) for a claim already mid-attempt to finish, so nothing starts a room here
+      # after this returns.
+      def stop(wait: 5)
+        handle, executor = @mutex.synchronize do
+          return self if @stopped
+
+          @stopped = true
+          [@handle, @executor].tap do
+            @handle = nil
+            @executor = nil
+          end
+        end
+
+        host.inbound.unsubscribe(channel, handle) if handle
+
+        # A cancelled task never runs its block (and so never its `finish`), so forget it here
+        tasks = @mutex.synchronize { @pending.to_a }
+        tasks.each { |id, task| finish(id) if task.cancel }
+        wait_idle(timeout: wait)
+        executor&.shutdown
+        self
+      end
+
+      def running?
+        @mutex.synchronize { !@stopped }
+      end
+
+      # How many claims are waiting on their timer or mid-attempt right now
+      def pending_count
+        @mutex.synchronize { @pending.size }
+      end
+
+      # Block until every scheduled claim has finished (won, lost, or been cancelled). Returns
+      # false if `timeout` passes first.
+      def wait_idle(timeout: 5)
+        deadline = monotonic_now + timeout
+        @mutex.synchronize do
+          until @pending.empty?
+            remaining = deadline - monotonic_now
+            return false if remaining <= 0
+            @idle.wait(@mutex, remaining)
+          end
+        end
+        true
+      end
+
+      # One request off the Bus, on its subscriber thread. Only the cheap checks happen here; the
+      # delayed claim is scheduled on the executor. Returns the scheduled task, or nil when there
+      # is nothing to do.
+      def handle(request)
+        room_class, serialized_key, tenant = parse(request)
+        id = [room_class.name, serialized_key, tenant]
+
+        return skip(id, "host is shutting down") if host.shutdown?
+
+        load = host.open_rooms_count
+        delay = delay_for(load)
+
+        @mutex.synchronize do
+          return skip(id, "placement is stopped") if @stopped
+          return skip(id, "a claim is already pending here") if @pending.key?(id)
+          return skip(id, "#{MAX_PENDING} claims are already pending here") if @pending.size >= MAX_PENDING
+
+          task = Concurrent::ScheduledTask.new(delay, executor: @executor) do
+            claim(room_class, serialized_key, tenant, delay: delay, load: load)
+          ensure
+            finish(id)
+          end
+          @pending[id] = task
+          task.execute
+        end
+      rescue InvalidRequest => e
+        logger.warn "Dropping provision request: #{e.message}"
+        nil
+      end
+
+      # Seconds to wait before racing for a room, given how many rooms this Host already runs:
+      # `provision_delay_ms × (load + jitter)`, jitter in 0..1. A Host with more rooms always waits
+      # longer than one with fewer; the jitter only orders Hosts with the same load.
+      def delay_for(load)
+        CableRoom.provision_delay_ms / 1000.0 * (load + @jitter.call.to_f)
+      end
+
+      private
+
+      # After the delay, on the executor. Re-checks what may have changed while we waited, then
+      # tries the room's lock once. The key is deserialized here, in the tenant, because a record
+      # key is a database lookup.
+      def claim(room_class, serialized_key, tenant, delay:, load:)
+        return if stopped? || host.shutdown?
+
+        CableRoom.with_app_executor do
+          CableRoom.with_tenant(tenant) do
+            key = deserialize_key(serialized_key)
+            next if running_here?(room_class, key, tenant)
+
+            payload = { host: host, room_class: room_class, room_key: key, tenant: tenant, delay: delay, open_rooms: load }
+            claimed = ActiveSupport::Notifications.instrument("provision_claimed.cable_room", payload) do |p|
+              p[:claimed] = host.ensure_room(room_class, key, tenant: tenant)
+            end
+
+            name = "#{room_class.name}[#{key}]"
+            if claimed
+              logger.info "Claimed #{name} after #{(delay * 1000).round}ms with #{load} room(s) open"
+            else
+              logger.debug "Lost the race for #{name}: another host holds its lock"
+            end
+          end
+        end
+      rescue InvalidRequest => e
+        logger.warn "Dropping provision request: #{e.message}"
+      rescue StandardError => e
+        logger.error "Failed to claim #{room_class.name}: #{e.class}: #{e.message}"
+        CableRoom.report_error(e, placement: self, room_class: room_class, room_key: serialized_key, tenant: tenant)
+      end
+
+      def finish(id)
+        @mutex.synchronize do
+          @pending.delete(id)
+          @idle.broadcast if @pending.empty?
+        end
+      end
+
+      def skip(id, why)
+        logger.debug "Ignoring provision request for #{id.inspect}: #{why}"
+        nil
+      end
+
+      def stopped?
+        @mutex.synchronize { @stopped }
+      end
+
+      def running_here?(room_class, key, tenant)
+        host.runners.any? { |r| r.room_class == room_class && r.key == key && r.tenant == tenant }
+      end
+
+      # Check a decoded Bus message's shape and find its Room class. The class name comes off the
+      # wire, so only a CableRoom::Room::Base subclass is accepted: `constantize` on an arbitrary
+      # string is not something a request should be able to do.
+      def parse(request)
+        raise InvalidRequest, "not a Hash: #{request.inspect.truncate(200)}" unless request.is_a?(Hash)
+
+        type = request["type"]
+        raise InvalidRequest, "expected a #{REQUEST_TYPE.inspect} request (got #{type.inspect.truncate(100)})" unless type == REQUEST_TYPE
+
+        name = request["room_class"]
+        klass = ActiveSupport::Inflector.safe_constantize(name) if name.is_a?(String) && name.match?(CLASS_NAME)
+        unless klass.is_a?(Class) && klass < Room::Base
+          raise InvalidRequest, "#{name.inspect.truncate(100)} is not a CableRoom::Room::Base subclass"
+        end
+
+        # The key travels the way `extra` does (ActiveJob's argument serializer): a one-item list
+        key = request["room_key"]
+        raise InvalidRequest, "room_key must be a serialized one-item list (got #{key.inspect.truncate(100)})" unless key.is_a?(Array) && key.size == 1
+
+        tenant = request["tenant"]
+        raise InvalidRequest, "tenant must be a String or null (got #{tenant.inspect.truncate(100)})" unless tenant.nil? || tenant.is_a?(String)
+
+        [klass, key, tenant]
+      end
+
+      def deserialize_key(serialized)
+        ::ActiveJob::Arguments.deserialize(serialized).first
+      rescue StandardError => e
+        raise InvalidRequest, "room_key can't be deserialized: #{e.class}: #{e.message}"
+      end
+
+      def logger
+        host.logger
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/cable_room/host/runner.rb

@@ -2,7 +2,7 @@ module CableRoom
   class Host
     # Runs one Room for the Host. It holds everything that is per-room but not the room's own
     # business: the ordered work queue, lifecycle state, the Redlock and watchdog, the inbound
-    # stream subscriptions, and the periodic timers.
+    # Bus subscriptions, and the periodic timers.
     #
     # A Room talks to its runner (`@runner`) for anything that touches threads or the outside
     # world. The Room DSL (`shutdown!`, `stop!`, `async`, `on_room_thread`, `ports[]`,
@@ -20,7 +20,7 @@ module CableRoom
 
       delegate :worker_pool, :inbound, to: :host
 
-      def initialize(host, room_class, key, lock_info, watchdog_interval:, lock_duration:)
+      def initialize(host, room_class, key, lock_info, watchdog_interval:, lock_duration:, tenant: nil)
         @host = host
         @room_class = room_class
         @key = key
@@ -42,8 +42,10 @@ module CableRoom
 
         # Nothing in the gem reads this, but the runner is the "connection" for its work items
         # (see WorkerPool), and PandaPal's Worker :work hook and broadcasting_for read
-        # `connection.tenant` to pick the room's tenant. Don't remove it.
-        @tenant = Apartment::Tenant.current if defined?(Apartment)
+        # `connection.tenant` to pick the room's tenant. Don't remove it. It's whatever the Host
+        # was told (see Host#ensure_room), never read from ambient state here: in a rooms process
+        # the thread building the runner has no request tenant of its own.
+        @tenant = tenant
 
         @logger = ActionCable::Connection::TaggedLoggerProxy.new(
           host.logger,
@@ -212,14 +214,14 @@ module CableRoom
 
       # -- Inbound streams ---------------------------------------------------------------------
 
-      # Deliver every message published on `stream` to `handler`, on the room's own queue.
-      # Decoding happens on the room's thread too, so a bad payload is reported like any other
-      # work error instead of hurting the transport. `on_live` runs once the transport confirms
-      # the subscription; anything published before that may be missed.
+      # Deliver every message published on `stream` to `handler`, on the room's own queue. The
+      # transport hands us decoded messages on its own thread; all we do there is queue, so one
+      # room's handler can't hold up another room's traffic. `on_live` runs once the transport
+      # confirms the subscription; anything published before that may be missed.
       #
       # The subscribe itself happens outside the mutex (see the class comment), so a room that
       # stopped meanwhile undoes it.
-      def subscribe(stream, coder: ActiveSupport::JSON, on_live: nil, &handler)
+      def subscribe(stream, on_live: nil, &handler)
         raise ArgumentError, "Block required" unless handler
 
         stream = String(stream)
@@ -230,10 +232,8 @@ module CableRoom
           on_live&.call
         end
 
-        handle = inbound.subscribe(stream, on_live: confirmed) do |raw|
-          post_work(async: false, silent: true) do
-            handler.call(coder ? coder.decode(raw) : raw)
-          end
+        handle = inbound.subscribe(stream, on_live: confirmed) do |message|
+          post_work(async: false, silent: true) { handler.call(message) }
         end
 
         stopped = @mutex.synchronize do
@@ -329,15 +329,9 @@ module CableRoom
       end
 
       # Room startup and shutdown used to run inside ActionCable's subscribe/unsubscribe
-      # callbacks, which Rails wraps in its executor (database connections, reloading, and so
-      # on). Keep that behavior. Nesting is fine: the executor yields straight through when it's
-      # already active on this thread.
+      # callbacks, which Rails wraps in its executor. Keep that behavior.
       def with_executor(&blk)
-        if defined?(Rails) && Rails.application
-          Rails.application.executor.wrap(&blk)
-        else
-          yield
-        end
+        CableRoom.with_app_executor(&blk)
       end
     end
   end