cable_room 0.7.0.beta2 → 0.8.0.beta1

data/cable_room.gemspec

@@ -18,17 +18,14 @@ Gem::Specification.new do |spec|
   spec.summary       = "Build live Rooms on top of ActionCable"
   spec.homepage      = "https://instructure.com"
 
-  spec.files = Dir["{app,config,db,exe,lib}/**/*", "README.md", "CHANGELOG.md", "*.gemspec"]
-  spec.bindir = "exe"
-  spec.executables = ["cable_room"]
+  spec.files = Dir["{app,config,db,lib}/**/*", "README.md", "*.gemspec"]
   spec.require_paths = ['lib']
 
   spec.add_dependency "rails", ">= 7.2", "< 9.0"
   spec.add_dependency "rufus-scheduler", "~> 3.6"
   spec.add_dependency "redlock", "~> 2.0"
   spec.add_dependency "rediconn", "~> 0.1.2"
-  # rediconn builds pools of redis-rb clients but doesn't declare the gem; the Bus needs it too
-  spec.add_dependency "redis", ">= 5.0"
 
+  spec.add_development_dependency "redis"
   spec.add_development_dependency 'rspec', '~> 3'
 end

data/lib/cable_room/host/action_cable_inbound.rb

@@ -0,0 +1,35 @@
+module CableRoom
+  class Host
+    # How rooms receive messages: straight off ActionCable's pubsub adapter, on the same stream
+    # names members publish to (Room::Base.room_port_key). This is the only place the room side
+    # touches ActionCable for inbound traffic; anything with the same two methods can be handed to
+    # `Host.new(inbound:)`.
+    #
+    # `subscribe` returns a handle that `unsubscribe` needs back. Handlers receive the payload
+    # exactly as published (a JSON string, for ActionCable) on the adapter's own thread, so they
+    # must be quick and hand the real work off. Host::Runner does that by queueing it on the room.
+    # `on_live` runs on that same thread once the adapter confirms the subscription.
+    class ActionCableInbound
+      # Subscribes from the calling thread. ActionCable's own stream_from posts the subscribe to
+      # the event loop first, but the adapter registers asynchronously either way, so that extra
+      # hop only widens the window in which the room can't hear anything.
+      #
+      # The adapter calls `on_live` while holding its own subscriber-map lock, so it must never
+      # wait on anything that might be waiting on the adapter (a room's mutex, say).
+      def subscribe(stream, on_live: nil, &on_message)
+        server.pubsub.subscribe(stream, on_message, on_live)
+        on_message
+      end
+
+      def unsubscribe(stream, handle)
+        server.pubsub.unsubscribe(stream, handle)
+      end
+
+      private
+
+      def server
+        ActionCable.server
+      end
+    end
+  end
+end

data/lib/cable_room/host/runner.rb

@@ -2,7 +2,7 @@ module CableRoom
   class Host
     # Runs one Room for the Host. It holds everything that is per-room but not the room's own
     # business: the ordered work queue, lifecycle state, the Redlock and watchdog, the inbound
-    # Bus subscriptions, and the periodic timers.
+    # stream subscriptions, and the periodic timers.
     #
     # A Room talks to its runner (`@runner`) for anything that touches threads or the outside
     # world. The Room DSL (`shutdown!`, `stop!`, `async`, `on_room_thread`, `ports[]`,
@@ -10,51 +10,40 @@ module CableRoom
     #
     # States: :initializing -> :starting -> :started -> :shutting_down -> :dead
     #
-    # A room being migrated off this host takes a detour: :started -> :freezing -> :frozen, then
-    # either `discard!` (-> :dead, quietly, once another host has it) or `stop!` (-> :dead with
-    # room_closed, when nobody took it). See `freeze!`. A room arriving on this host by migration
-    # runs the detour backwards: `restore!(hold_inbound: true)` brings it up straight into :frozen,
-    # and `thaw!` moves it to :started once the handoff has replayed what the old host relayed.
+    # Locking: `@mutex` guards lifecycle state and the stream and timer lists. It is never held
+    # across a call into the inbound transport or into the room's own callbacks, because the
+    # transport's threads call back into us and would deadlock against it. Those callbacks only
+    # ever take `@queue_lock`, a leaf lock around the work queue that is never held while calling
+    # out to anything. Lock order is always `@mutex` then `@queue_lock`.
     class Runner
-      FROZEN_STATES = %i[freezing frozen].freeze
-
       attr_reader :host, :room, :room_class, :key, :uuid, :tenant, :logger
 
-      # Monotonic time this runner was built. `Host#drain!` migrates rooms oldest first.
-      attr_reader :started_at
-
       delegate :worker_pool, :inbound, to: :host
 
-      def initialize(host, room_class, key, lock_info, watchdog_interval:, lock_duration:, tenant: nil)
+      def initialize(host, room_class, key, lock_info, watchdog_interval:, lock_duration:)
         @host = host
         @room_class = room_class
         @key = key
         @lock_info = lock_info
         @watchdog_interval = watchdog_interval
         @lock_duration = lock_duration
-        @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
         # Used mainly for logs and being able to follow a specific Room instance
         @uuid = SecureRandom.hex(6)
 
         @mutex = Monitor.new
-        @work_finished = @mutex.new_cond
+        @queue_lock = Mutex.new
         @current_state = :initializing
+        @stopping = false
         @processing_work = false
         @work_queue = []
-        @held_inbound = []
-        @hold_from_start = false
-        @streams = {}   # stream => the inbound transport's handle, needed to unsubscribe
-        @handlers = {}  # stream => the room's handler, so a message can be fed in by hand (see `inject`)
+        @streams = {}
         @periodic_timers = []
 
-        # Whatever the member's provision request said (RoomMembership defaults that to the
-        # ambient tenant at join_room's own call site -- see RoomMembership#initialize) or, for a
-        # migrated room, whatever its snapshot carried forward. Nothing here parses it out of
-        # `key` or guesses from ambient state itself: a Host thread has no request of its own, so
-        # anything read here would be leftover from whatever this thread ran last, not this room's
-        # tenant.
-        @tenant = tenant
+        # Nothing in the gem reads this, but the runner is the "connection" for its work items
+        # (see WorkerPool), and PandaPal's Worker :work hook and broadcasting_for read
+        # `connection.tenant` to pick the room's tenant. Don't remove it.
+        @tenant = Apartment::Tenant.current if defined?(Apartment)
 
         @logger = ActionCable::Connection::TaggedLoggerProxy.new(
           host.logger,
@@ -74,14 +63,8 @@ module CableRoom
         @current_state
       end
 
-      def frozen?
-        FROZEN_STATES.include?(state)
-      end
-
-      # The Redlock this runner holds, or nil once it has been released or handed off. Read-only:
-      # use `release_lock!` to let go of it.
-      def lock_info
-        @lock_info
+      def closed?
+        state == :dead || state == :shutting_down
       end
 
       # -- Lifecycle ---------------------------------------------------------------------------
@@ -89,37 +72,30 @@ module CableRoom
       # Run the room's startup callbacks and start its class-level timers, on the calling thread.
       # If startup fails the room is torn down and the error re-raised.
       def start!
-        start_with { room.send(:_startup) }
-      end
-
-      # Bring the room up from a CableRoom::Snapshot instead of from scratch: same as `start!`,
-      # except the room rebuilds its ports, users, reaper deadlines, and (through `restore_state`)
-      # its own state first, and doesn't tell its members it opened. The caller holds the room's
-      # lock already, the same as for `start!`.
-      #
-      # With `hold_inbound: true` the room comes up :frozen instead of :started: it subscribes its
-      # inbound streams as usual, but every message that arrives is held (see `held_inbound`) and
-      # its timers don't start until `thaw!`. This is how a migration's adopter takes a room: the
-      # messages the old host relayed have to run before anything that arrives live here.
-      def restore!(snapshot, hold_inbound: false)
-        @hold_from_start = hold_inbound
-        start_with(final_state: hold_inbound ? :frozen : :started) { room.send(:_restore, snapshot) }
+        @current_state = :starting
+        with_executor do
+          room.send(:_startup)
+          start_periodic_timers
+        end
+        # Startup may already have asked to shut down (a reaper, or a KILL that arrived early)
+        @queue_lock.synchronize { @current_state = :started if @current_state == :starting }
+      rescue => e
+        terminate!
+        raise e
       end
 
       # Stop right now, on the calling thread: run the room's shutdown callbacks and free
       # everything. Work still on the queue is dropped. `initiate_shutdown` is the graceful version.
-      # `reason` is what members see in `room_closed`; nil keeps whatever the room already set.
       #
-      # Only the state change happens under the mutex. Unsubscribing waits for the Bus to confirm,
-      # and the Bus thread needs this same mutex to queue an arriving message, so holding it here
-      # would deadlock the two whenever a message lands mid-shutdown.
-      def stop!(reason: nil)
+      # Only the state change happens under the mutex (see the class comment). `@stopping` makes
+      # sure the shutdown callbacks run once even if, say, a lost lock and a queued stop race.
+      def stop!
         @mutex.synchronize do
-          return if state == :dead
-          @current_state = :shutting_down
+          return if @stopping || state == :dead
+          @stopping = true
+          @queue_lock.synchronize { @current_state = :shutting_down }
         end
 
-        room.send(:_shutdown_reason=, reason) unless reason.nil?
         unsubscribe_all
         begin
           with_executor { room.send(:_shutdown) }
@@ -128,191 +104,42 @@ module CableRoom
         end
       end
 
-      # Shut down gracefully: stop listening, then stop once everything already queued has run.
-      # `reason` reaches the members in `room_closed`.
+      # Shut down gracefully: stop listening, then stop once everything already queued has run
       def initiate_shutdown(reason)
         @mutex.synchronize do
           return if closed?
 
           logger.info "Initiating shutdown: #{reason}"
-          room.send(:_shutdown_reason=, reason)
 
-          # The actual stop goes behind whatever is already waiting, so those messages still run
-          post_work(async: false) { stop! }
-
-          # From here on new messages are dropped at the queue (see `enqueue`), so nothing new
-          # lands after the stop even before the unsubscribe below is confirmed
-          @current_state = :shutting_down
+          # The actual stop goes behind whatever is already waiting, so those messages still run.
+          # Closing in the same step means nothing posted from here on can land after it.
+          @queue_lock.synchronize do
+            @work_queue << wrap_work(-> { stop! })
+            @current_state = :shutting_down
+          end
         end
 
-        unsubscribe_all
-        # A frozen room had stopped taking work off its queue; now that it's shutting down the
-        # stop posted above has to actually run
         schedule_work
+        unsubscribe_all
       end
 
-      # Free everything without running the room's shutdown callbacks. Members hear nothing (no
-      # room_closed); the lock is released unless the caller asks to keep it (see `discard!`).
-      def terminate!(release_lock: true)
+      # Free everything without running the room's shutdown callbacks
+      def terminate!
         lock_info = @mutex.synchronize do
-          @current_state = :dead
-          stop_periodic_timers
+          @queue_lock.synchronize do
+            @current_state = :dead
+            @work_queue.clear
+          end
           @lock_info.tap { @lock_info = nil }
         end
 
+        stop_periodic_timers
+
         # Unsubscribe before releasing the lock: the next runner for this key (which needs the
-        # lock) must not subscribe to the same Bus channel before this one has let go of it.
+        # lock) must not subscribe to the same streams before this one has let go of them.
         unsubscribe_all
-        CableRoom.lock_manager.unlock(lock_info) if release_lock && lock_info
+        CableRoom.lock_manager.unlock(lock_info) if lock_info
         host.untrack(self)
-        lock_info
-      end
-
-      # -- Migration ---------------------------------------------------------------------------
-      #
-      # The pieces the migration protocol (CableRoom::Migration) is built from, in the order it
-      # uses them: `freeze!`, `snapshot`, `release_lock!`, and finally `discard!` once another
-      # host has restored the room — or `stop!` if none did. Members never hear about any of it.
-
-      # Bring the room to a standstill so it can be snapshotted: stop the periodic timers, let
-      # whatever is already queued finish, and from then on hold inbound messages instead of
-      # handling them. The Bus subscription stays up, so the messages that arrive while the room
-      # is frozen still land here — in `held_inbound`, or with the block if one is given (called
-      # on the Bus thread with `(stream, message)`, so keep it quick), for the migration to relay
-      # to the new host. The lock is kept and keeps being renewed.
-      #
-      # Blocks until the room is quiet, or for `timeout` seconds (nil waits as long as it takes).
-      # Returns true once frozen, or false if the room died on the way (a queued message asked it
-      # to shut down, say) or the timeout passed — in which case the room is running again as if
-      # nothing happened, with the messages held meanwhile back on its queue, so the caller can
-      # still `stop!` it cleanly. A room that never goes quiet must not hang a drain.
-      def freeze!(timeout: nil, &hold)
-        @mutex.synchronize do
-          return false if closed?
-          return true if frozen?
-
-          logger.info "Freezing"
-          @current_state = :freezing
-          @hold_inbound = hold
-          stop_periodic_timers
-          # Nothing else may be running in this room once we return: wait for the queue to drain
-          # and the item in flight to finish. New inbound is already being held, so this ends.
-          deadline = timeout && monotonic_now + timeout
-          while state == :freezing && (@processing_work || @work_queue.any?)
-            remaining = deadline && deadline - monotonic_now
-            if remaining && remaining <= 0
-              logger.warn "Room did not go quiet within #{timeout}s; not freezing it"
-              resume_from_freeze
-              return false
-            end
-            @work_finished.wait(remaining)
-          end
-          return false unless state == :freezing
-
-          @current_state = :frozen
-          # Messages that arrived while the room was still going quiet were held, not handed to
-          # the block: had the freeze timed out they'd have to go back on the queue, and once
-          # relayed they'd be gone. Now that the freeze is final, hand them over first — under the
-          # mutex, so nothing arriving on the Bus thread can get ahead of them.
-          if hold
-            held = @held_inbound
-            @held_inbound = []
-            held.each { |stream, message| hold.call(stream, message) }
-          end
-        end
-        true
-      end
-
-      # Messages that arrived while frozen (and weren't handed to a `freeze!` block), as
-      # `[stream, message]` pairs in arrival order.
-      def held_inbound
-        @mutex.synchronize { @held_inbound.dup }
-      end
-
-      # The streams this runner is subscribed to right now (Bus channel names).
-      def subscribed_streams
-        @mutex.synchronize { @streams.keys }
-      end
-
-      # Feed `message` into the room as if it had arrived on `stream`, behind whatever is already
-      # queued — even while the room is frozen, when a live message would be held instead. This is
-      # how a migration's adopter replays what the old host relayed: the messages go onto the
-      # queue in the order given and run once the room thaws. Returns false (and drops the
-      # message, with a warning) if the room doesn't listen on that stream.
-      def inject(stream, message)
-        handler = @mutex.synchronize { @handlers[String(stream)] }
-        unless handler
-          logger.warn "Dropping a relayed message for #{stream}: this room doesn't listen on it"
-          return false
-        end
-
-        post_work(async: false, silent: true) { handler.call(message) }
-        true
-      end
-
-      # Take a frozen room back to :started. The block gets the inbound held so far (as
-      # `[stream, message]` pairs, in arrival order) and returns the pairs to run, in order —
-      # a migration's adopter uses it to drop the ones it has already replayed from the handoff
-      # list. Those are queued (after anything `inject`ed before), the timers start, and the room
-      # runs again. Everything happens under the mutex, so no message can land between the block
-      # seeing the held list and the room going live: it is either in the list or queued after.
-      # Returns false if the room isn't frozen.
-      def thaw!
-        @mutex.synchronize do
-          return false unless state == :frozen
-
-          held = @held_inbound
-          @held_inbound = []
-          @hold_inbound = nil
-          @hold_from_start = false
-          to_run = block_given? ? yield(held) : held
-          to_run.each { |stream, message| inject(stream, message) }
-          @current_state = :started
-          start_periodic_timers
-          ping_watchdog
-        end
-        logger.info "Thawed"
-        schedule_work
-        true
-      end
-
-      # The room's CableRoom::Snapshot. Only a frozen room can be snapshotted: that's the one
-      # state where nothing else is touching its state.
-      def snapshot
-        raise "#{room_class.name}[#{key}] must be frozen before it can be snapshotted (state: #{state})" unless state == :frozen
-
-        with_executor { Snapshot.take(room) }
-      end
-
-      # Give the room's lock up while staying alive, so another host can claim the room and this
-      # one can keep relaying inbound until it has. Returns false if there was no lock to release.
-      def release_lock!
-        lock_info = @mutex.synchronize { @lock_info.tap { @lock_info = nil } }
-        return false unless lock_info
-
-        CableRoom.lock_manager.unlock(lock_info)
-        true
-      end
-
-      # Take the room's lock again after `release_lock!`, when no other host claimed it. Returns
-      # false if someone else holds it (or this runner still holds it). With the lock back, `stop!`
-      # releases it the normal way.
-      def retake_lock!
-        return false if @lock_info
-
-        lock_info = CableRoom.lock_manager.lock(room_class.room_port_key(key), @lock_duration.in_milliseconds)
-        return false unless lock_info
-
-        @mutex.synchronize { @lock_info = lock_info }
-        true
-      end
-
-      # Drop a room that now lives somewhere else: no shutdown callbacks, no room_closed, and the
-      # lock — if this runner still holds one — is left alone, since it may belong to the new host
-      # by now. Returns the lock_info that was still held, or nil, so the caller can decide.
-      def discard!
-        logger.info "Discarding (the room has moved)"
-        terminate!(release_lock: false)
       end
 
       # -- Watchdog and lock -------------------------------------------------------------------
@@ -341,9 +168,6 @@ module CableRoom
           return
         end
 
-        # A frozen room is idle on purpose; its timers are stopped, so nobody pings the watchdog
-        return if frozen?
-
         unless @last_watchdog_ping_at && @last_watchdog_ping_at > @watchdog_interval.ago
           logger.warn "Watchdog timeout for room #{room_class.name}[#{key}], shutting down"
           initiate_shutdown("Watchdog timeout")
@@ -364,16 +188,10 @@ module CableRoom
       # Errors inside the work are reported (see `report_work_error`), never raised, so one bad
       # message can't take the room down with it.
       def post_work(async: false, silent: false, &blk)
-        work = proc do
-          worker_pool.invoke(blk, :call, connection: self)
-        rescue => e
-          report_work_error(e)
-        end
-
         if async
-          worker_pool.executor.post(&work)
+          worker_pool.executor.post(&wrap_work(blk))
         else
-          enqueue(work, silent: silent)
+          enqueue(wrap_work(blk), silent: silent)
         end
       end
 
@@ -394,37 +212,44 @@ module CableRoom
 
       # -- Inbound streams ---------------------------------------------------------------------
 
-      # Deliver every message published on `stream` to `handler`, on the room's own queue. The
-      # transport hands us decoded messages on its own thread; all we do there is queue, so one
-      # room's handler can't hold up another room's traffic. `on_live` runs once the transport
-      # confirms the subscription; anything published before that may be missed.
+      # Deliver every message published on `stream` to `handler`, on the room's own queue.
+      # Decoding happens on the room's thread too, so a bad payload is reported like any other
+      # work error instead of hurting the transport. `on_live` runs once the transport confirms
+      # the subscription; anything published before that may be missed.
       #
-      # The subscribe itself happens outside the mutex (it blocks until the Bus confirms, and the
-      # Bus thread needs the mutex to enqueue), so a room that stopped meanwhile undoes it.
-      def subscribe(stream, on_live: nil, &handler)
+      # The subscribe itself happens outside the mutex (see the class comment), so a room that
+      # stopped meanwhile undoes it.
+      def subscribe(stream, coder: ActiveSupport::JSON, on_live: nil, &handler)
         raise ArgumentError, "Block required" unless handler
 
         stream = String(stream)
         return if closed?
 
-        handle = inbound.subscribe(stream, on_live: on_live) do |message|
-          receive_inbound(stream, message, handler)
+        confirmed = lambda do
+          logger.info "#{room_class.name} is streaming from #{stream}"
+          on_live&.call
+        end
+
+        handle = inbound.subscribe(stream, on_live: confirmed) do |raw|
+          post_work(async: false, silent: true) do
+            handler.call(coder ? coder.decode(raw) : raw)
+          end
         end
 
         stopped = @mutex.synchronize do
-          closed? || (@streams[stream] = handle; @handlers[stream] = handler; false)
+          closed? || (@streams[stream] = handle; false)
         end
         inbound.unsubscribe(stream, handle) if stopped
       end
 
       def unsubscribe(stream)
         stream = String(stream)
-        handle = @mutex.synchronize { @handlers.delete(stream); @streams.delete(stream) }
+        handle = @mutex.synchronize { @streams.delete(stream) }
         inbound.unsubscribe(stream, handle) if handle
       end
 
       def unsubscribe_all
-        handles = @mutex.synchronize { @handlers.clear; @streams.to_a.tap { @streams.clear } }
+        handles = @mutex.synchronize { @streams.to_a.tap { @streams.clear } }
         handles.each { |stream, handle| inbound.unsubscribe(stream, handle) }
       end
 
@@ -439,94 +264,46 @@ module CableRoom
           post_work(async: false, silent: true) { callback.call }
         end
 
-        PeriodicTimer.new(job)
+        timer = PeriodicTimer.new(job)
+        @mutex.synchronize { @periodic_timers << timer }
+        timer
       end
 
       private
 
-      def closed?
-        state == :dead || state == :shutting_down
-      end
-
-      # `start!` and `restore!` differ only in what the room does first; everything around it —
-      # the executor, the timers, tearing down on failure — is the same. A room restored with
-      # `hold_inbound` ends up :frozen with no timers running; `thaw!` starts them.
-      def start_with(final_state: :started)
-        @current_state = :starting
-        with_executor do
-          yield
-          start_periodic_timers unless final_state == :frozen
-        end
-        @current_state = final_state
-      rescue => e
-        terminate!
-        raise e
-      end
-
-      # A message from the Bus, on the Bus thread. Normally it's queued for the room; while the
-      # room is frozen it's held for the migration to relay instead. Decided under the mutex so a
-      # message can't slip onto the queue in the moment the room freezes.
-      #
-      # A room restored with `hold_inbound` holds from its very first subscribe, while it is still
-      # :starting: the adopter has to see everything that arrives before it thaws the room.
-      def receive_inbound(stream, message, handler)
-        disposition = @mutex.synchronize do
-          next :queue unless frozen? || @hold_from_start
-          # Only a fully frozen room relays; while still :freezing it holds (see `freeze!`)
-          next :relay if @hold_inbound && state == :frozen
-
-          @held_inbound << [stream, message]
-          :held
-        end
-
-        case disposition
-        when :queue
-          # A handoff marker is the migration protocol talking to itself (see Migration); it is
-          # never for the room. One can only reach a live room on a failure path, so drop it here.
-          return if Migration.marker?(message)
+      # Timers declared on the Room class with `periodically`. None if startup already asked the
+      # room to shut down.
+      def start_periodic_timers
+        return if closed?
 
-          post_work(async: false, silent: true) { handler.call(message) }
-        when :relay then @hold_inbound.call(stream, message)
+        room_class.periodic_timers.each do |callback, every|
+          start_periodic_timer(-> { room.instance_exec(&callback) }, every: every)
         end
       end
 
-      # Undo a freeze that didn't complete: the room runs on as if `freeze!` had never been
-      # called, with the messages held meanwhile queued in arrival order. Caller holds the mutex,
-      # and the state is :freezing, so nothing else is running in the room.
-      def resume_from_freeze
-        held = @held_inbound
-        @held_inbound = []
-        @hold_inbound = nil
-        @current_state = :started
-        held.each { |stream, message| inject(stream, message) }
-        start_periodic_timers
-        ping_watchdog
-      end
-
-      def monotonic_now
-        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      def stop_periodic_timers
+        timers = @mutex.synchronize { @periodic_timers.slice!(0..) }
+        timers.each(&:shutdown)
       end
 
-      # Timers declared on the Room class with `periodically`
-      def start_periodic_timers
-        room_class.periodic_timers.each do |callback, every|
-          @periodic_timers << start_periodic_timer(-> { room.instance_exec(&callback) }, every: every)
+      def wrap_work(blk)
+        proc do
+          worker_pool.invoke(blk, :call, connection: self)
+        rescue => e
+          report_work_error(e)
         end
       end
 
-      def stop_periodic_timers
-        @periodic_timers.each(&:shutdown)
-        @periodic_timers.clear
-      end
-
       def enqueue(work, silent:)
-        @mutex.synchronize do
-          if closed?
-            raise "Attempt to post work to dead or shutting down room" unless silent
-            return
-          end
-
+        accepted = @queue_lock.synchronize do
+          next false if closed?
           @work_queue << work
+          true
+        end
+
+        unless accepted
+          raise "Attempt to post work to dead or shutting down room" unless silent
+          return
         end
 
         schedule_work
@@ -536,28 +313,18 @@ module CableRoom
       # room is running already. When it finishes, come back for the next one. This is the whole
       # ordering guarantee: one thread per room at a time, in arrival order.
       def schedule_work
-        @mutex.synchronize do
-          return if @processing_work
-          # A frozen room keeps its queue but runs nothing from it (while :freezing it still
-          # drains, so that `freeze!` can return to a quiet room)
-          return if state == :frozen
-
-          work = @work_queue.shift
-          return unless work
-
-          @processing_work = true
-
-          worker_pool.executor.post do
-            begin
-              work.call
-            ensure
-              @mutex.synchronize do
-                @processing_work = false
-                @work_finished.broadcast
-              end
-              schedule_work
-            end
-          end
+        work = @queue_lock.synchronize do
+          next if @processing_work
+
+          @work_queue.shift.tap { |w| @processing_work = true if w }
+        end
+        return unless work
+
+        worker_pool.executor.post do
+          work.call
+        ensure
+          @queue_lock.synchronize { @processing_work = false }
+          schedule_work
         end
       end