cable_room 0.6.2 → 0.7.0.beta1

data/lib/cable_room/host/runner.rb

@@ -0,0 +1,571 @@
+module CableRoom
+  class Host
+    # Runs one Room for the Host. It holds everything that is per-room but not the room's own
+    # business: the ordered work queue, lifecycle state, the Redlock and watchdog, the inbound
+    # Bus subscriptions, and the periodic timers.
+    #
+    # A Room talks to its runner (`@runner`) for anything that touches threads or the outside
+    # world. The Room DSL (`shutdown!`, `stop!`, `async`, `on_room_thread`, `ports[]`,
+    # `periodically`) is built on this API.
+    #
+    # States: :initializing -> :starting -> :started -> :shutting_down -> :dead
+    #
+    # A room being migrated off this host takes a detour: :started -> :freezing -> :frozen, then
+    # either `discard!` (-> :dead, quietly, once another host has it) or `stop!` (-> :dead with
+    # room_closed, when nobody took it). See `freeze!`. A room arriving on this host by migration
+    # runs the detour backwards: `restore!(hold_inbound: true)` brings it up straight into :frozen,
+    # and `thaw!` moves it to :started once the handoff has replayed what the old host relayed.
+    class Runner
+      FROZEN_STATES = %i[freezing frozen].freeze
+
+      attr_reader :host, :room, :room_class, :key, :uuid, :tenant, :logger
+
+      # Monotonic time this runner was built. `Host#drain!` migrates rooms oldest first.
+      attr_reader :started_at
+
+      delegate :worker_pool, :inbound, to: :host
+
+      def initialize(host, room_class, key, lock_info, watchdog_interval:, lock_duration:)
+        @host = host
+        @room_class = room_class
+        @key = key
+        @lock_info = lock_info
+        @watchdog_interval = watchdog_interval
+        @lock_duration = lock_duration
+        @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        # Used mainly for logs and being able to follow a specific Room instance
+        @uuid = SecureRandom.hex(6)
+
+        @mutex = Monitor.new
+        @work_finished = @mutex.new_cond
+        @current_state = :initializing
+        @processing_work = false
+        @work_queue = []
+        @held_inbound = []
+        @hold_from_start = false
+        @streams = {}   # stream => the inbound transport's handle, needed to unsubscribe
+        @handlers = {}  # stream => the room's handler, so a message can be fed in by hand (see `inject`)
+        @periodic_timers = []
+
+        @tenant = Apartment::Tenant.current if defined?(Apartment)
+
+        @logger = ActionCable::Connection::TaggedLoggerProxy.new(
+          host.logger,
+          tags: ["#{room_class.name} #{@uuid}"]
+        )
+
+        logger.info "Initializing new #{room_class.name}"
+        logger.info "  UUID: #{@uuid}"
+        logger.info "  Key: #{room_class.room_port_key(key)}"
+
+        @room = room_class.new(self, key)
+        host.track(self)
+        ping_watchdog
+      end
+
+      def state
+        @current_state
+      end
+
+      def frozen?
+        FROZEN_STATES.include?(state)
+      end
+
+      # The Redlock this runner holds, or nil once it has been released or handed off. Read-only:
+      # use `release_lock!` to let go of it.
+      def lock_info
+        @lock_info
+      end
+
+      # -- Lifecycle ---------------------------------------------------------------------------
+
+      # Run the room's startup callbacks and start its class-level timers, on the calling thread.
+      # If startup fails the room is torn down and the error re-raised.
+      def start!
+        start_with { room.send(:_startup) }
+      end
+
+      # Bring the room up from a CableRoom::Snapshot instead of from scratch: same as `start!`,
+      # except the room rebuilds its ports, users, reaper deadlines, and (through `restore_state`)
+      # its own state first, and doesn't tell its members it opened. The caller holds the room's
+      # lock already, the same as for `start!`.
+      #
+      # With `hold_inbound: true` the room comes up :frozen instead of :started: it subscribes its
+      # inbound streams as usual, but every message that arrives is held (see `held_inbound`) and
+      # its timers don't start until `thaw!`. This is how a migration's adopter takes a room: the
+      # messages the old host relayed have to run before anything that arrives live here.
+      def restore!(snapshot, hold_inbound: false)
+        @hold_from_start = hold_inbound
+        start_with(final_state: hold_inbound ? :frozen : :started) { room.send(:_restore, snapshot) }
+      end
+
+      # Stop right now, on the calling thread: run the room's shutdown callbacks and free
+      # everything. Work still on the queue is dropped. `initiate_shutdown` is the graceful version.
+      # `reason` is what members see in `room_closed`; nil keeps whatever the room already set.
+      #
+      # Only the state change happens under the mutex. Unsubscribing waits for the Bus to confirm,
+      # and the Bus thread needs this same mutex to queue an arriving message, so holding it here
+      # would deadlock the two whenever a message lands mid-shutdown.
+      def stop!(reason: nil)
+        @mutex.synchronize do
+          return if state == :dead
+          @current_state = :shutting_down
+        end
+
+        room.send(:_shutdown_reason=, reason) unless reason.nil?
+        unsubscribe_all
+        begin
+          with_executor { room.send(:_shutdown) }
+        ensure
+          terminate!
+        end
+      end
+
+      # Shut down gracefully: stop listening, then stop once everything already queued has run.
+      # `reason` reaches the members in `room_closed`.
+      def initiate_shutdown(reason)
+        @mutex.synchronize do
+          return if closed?
+
+          logger.info "Initiating shutdown: #{reason}"
+          room.send(:_shutdown_reason=, reason)
+
+          # The actual stop goes behind whatever is already waiting, so those messages still run
+          post_work(async: false) { stop! }
+
+          # From here on new messages are dropped at the queue (see `enqueue`), so nothing new
+          # lands after the stop even before the unsubscribe below is confirmed
+          @current_state = :shutting_down
+        end
+
+        unsubscribe_all
+        # A frozen room had stopped taking work off its queue; now that it's shutting down the
+        # stop posted above has to actually run
+        schedule_work
+      end
+
+      # Free everything without running the room's shutdown callbacks. Members hear nothing (no
+      # room_closed); the lock is released unless the caller asks to keep it (see `discard!`).
+      def terminate!(release_lock: true)
+        lock_info = @mutex.synchronize do
+          @current_state = :dead
+          stop_periodic_timers
+          @lock_info.tap { @lock_info = nil }
+        end
+
+        # Unsubscribe before releasing the lock: the next runner for this key (which needs the
+        # lock) must not subscribe to the same Bus channel before this one has let go of it.
+        unsubscribe_all
+        CableRoom.lock_manager.unlock(lock_info) if release_lock && lock_info
+        host.untrack(self)
+        lock_info
+      end
+
+      # -- Migration ---------------------------------------------------------------------------
+      #
+      # The pieces the migration protocol (CableRoom::Migration) is built from, in the order it
+      # uses them: `freeze!`, `snapshot`, `release_lock!`, and finally `discard!` once another
+      # host has restored the room — or `stop!` if none did. Members never hear about any of it.
+
+      # Bring the room to a standstill so it can be snapshotted: stop the periodic timers, let
+      # whatever is already queued finish, and from then on hold inbound messages instead of
+      # handling them. The Bus subscription stays up, so the messages that arrive while the room
+      # is frozen still land here — in `held_inbound`, or with the block if one is given (called
+      # on the Bus thread with `(stream, message)`, so keep it quick), for the migration to relay
+      # to the new host. The lock is kept and keeps being renewed.
+      #
+      # Blocks until the room is quiet, or for `timeout` seconds (nil waits as long as it takes).
+      # Returns true once frozen, or false if the room died on the way (a queued message asked it
+      # to shut down, say) or the timeout passed — in which case the room is running again as if
+      # nothing happened, with the messages held meanwhile back on its queue, so the caller can
+      # still `stop!` it cleanly. A room that never goes quiet must not hang a drain.
+      def freeze!(timeout: nil, &hold)
+        @mutex.synchronize do
+          return false if closed?
+          return true if frozen?
+
+          logger.info "Freezing"
+          @current_state = :freezing
+          @hold_inbound = hold
+          stop_periodic_timers
+          # Nothing else may be running in this room once we return: wait for the queue to drain
+          # and the item in flight to finish. New inbound is already being held, so this ends.
+          deadline = timeout && monotonic_now + timeout
+          while state == :freezing && (@processing_work || @work_queue.any?)
+            remaining = deadline && deadline - monotonic_now
+            if remaining && remaining <= 0
+              logger.warn "Room did not go quiet within #{timeout}s; not freezing it"
+              resume_from_freeze
+              return false
+            end
+            @work_finished.wait(remaining)
+          end
+          return false unless state == :freezing
+
+          @current_state = :frozen
+          # Messages that arrived while the room was still going quiet were held, not handed to
+          # the block: had the freeze timed out they'd have to go back on the queue, and once
+          # relayed they'd be gone. Now that the freeze is final, hand them over first — under the
+          # mutex, so nothing arriving on the Bus thread can get ahead of them.
+          if hold
+            held = @held_inbound
+            @held_inbound = []
+            held.each { |stream, message| hold.call(stream, message) }
+          end
+        end
+        true
+      end
+
+      # Messages that arrived while frozen (and weren't handed to a `freeze!` block), as
+      # `[stream, message]` pairs in arrival order.
+      def held_inbound
+        @mutex.synchronize { @held_inbound.dup }
+      end
+
+      # The streams this runner is subscribed to right now (Bus channel names).
+      def subscribed_streams
+        @mutex.synchronize { @streams.keys }
+      end
+
+      # Feed `message` into the room as if it had arrived on `stream`, behind whatever is already
+      # queued — even while the room is frozen, when a live message would be held instead. This is
+      # how a migration's adopter replays what the old host relayed: the messages go onto the
+      # queue in the order given and run once the room thaws. Returns false (and drops the
+      # message, with a warning) if the room doesn't listen on that stream.
+      def inject(stream, message)
+        handler = @mutex.synchronize { @handlers[String(stream)] }
+        unless handler
+          logger.warn "Dropping a relayed message for #{stream}: this room doesn't listen on it"
+          return false
+        end
+
+        post_work(async: false, silent: true) { handler.call(message) }
+        true
+      end
+
+      # Take a frozen room back to :started. The block gets the inbound held so far (as
+      # `[stream, message]` pairs, in arrival order) and returns the pairs to run, in order —
+      # a migration's adopter uses it to drop the ones it has already replayed from the handoff
+      # list. Those are queued (after anything `inject`ed before), the timers start, and the room
+      # runs again. Everything happens under the mutex, so no message can land between the block
+      # seeing the held list and the room going live: it is either in the list or queued after.
+      # Returns false if the room isn't frozen.
+      def thaw!
+        @mutex.synchronize do
+          return false unless state == :frozen
+
+          held = @held_inbound
+          @held_inbound = []
+          @hold_inbound = nil
+          @hold_from_start = false
+          to_run = block_given? ? yield(held) : held
+          to_run.each { |stream, message| inject(stream, message) }
+          @current_state = :started
+          start_periodic_timers
+          ping_watchdog
+        end
+        logger.info "Thawed"
+        schedule_work
+        true
+      end
+
+      # The room's CableRoom::Snapshot. Only a frozen room can be snapshotted: that's the one
+      # state where nothing else is touching its state.
+      def snapshot
+        raise "#{room_class.name}[#{key}] must be frozen before it can be snapshotted (state: #{state})" unless state == :frozen
+
+        with_executor { Snapshot.take(room) }
+      end
+
+      # Give the room's lock up while staying alive, so another host can claim the room and this
+      # one can keep relaying inbound until it has. Returns false if there was no lock to release.
+      def release_lock!
+        lock_info = @mutex.synchronize { @lock_info.tap { @lock_info = nil } }
+        return false unless lock_info
+
+        CableRoom.lock_manager.unlock(lock_info)
+        true
+      end
+
+      # Take the room's lock again after `release_lock!`, when no other host claimed it. Returns
+      # false if someone else holds it (or this runner still holds it). With the lock back, `stop!`
+      # releases it the normal way.
+      def retake_lock!
+        return false if @lock_info
+
+        lock_info = CableRoom.lock_manager.lock(room_class.room_port_key(key), @lock_duration.in_milliseconds)
+        return false unless lock_info
+
+        @mutex.synchronize { @lock_info = lock_info }
+        true
+      end
+
+      # Drop a room that now lives somewhere else: no shutdown callbacks, no room_closed, and the
+      # lock — if this runner still holds one — is left alone, since it may belong to the new host
+      # by now. Returns the lock_info that was still held, or nil, so the caller can decide.
+      def discard!
+        logger.info "Discarding (the room has moved)"
+        terminate!(release_lock: false)
+      end
+
+      # -- Watchdog and lock -------------------------------------------------------------------
+
+      def ping_watchdog
+        return if state == :dead
+
+        logger.debug "Ping watchdog"
+        @last_watchdog_ping_at = Time.current
+      end
+
+      # Called on every beat: renew the Redlock, and shut down if nobody has pinged the watchdog
+      # within the interval. Losing the lock stops the room immediately, since another process
+      # may already be running it.
+      def check_room_watchdog
+        lock_info = @mutex.synchronize do
+          return if closed?
+          @lock_info
+        end
+        return unless lock_info
+
+        relock = CableRoom.lock_manager.lock(lock_info[:resource], @lock_duration.in_milliseconds, extend: lock_info)
+        unless relock
+          logger.warn "Lost lock, shutting down"
+          stop!
+          return
+        end
+
+        # A frozen room is idle on purpose; its timers are stopped, so nobody pings the watchdog
+        return if frozen?
+
+        unless @last_watchdog_ping_at && @last_watchdog_ping_at > @watchdog_interval.ago
+          logger.warn "Watchdog timeout for room #{room_class.name}[#{key}], shutting down"
+          initiate_shutdown("Watchdog timeout")
+        end
+      end
+
+      def beat
+        post_work(async: true) { check_room_watchdog }
+      end
+
+      # -- Work --------------------------------------------------------------------------------
+
+      # Run `blk` for this room. `async: false` (the default) puts it on the room's own queue,
+      # where items run one at a time in arrival order. `async: true` runs it on the shared pool
+      # straight away, alongside whatever the room is doing. Posting to a room that is shutting
+      # down raises unless `silent: true`, in which case the work is dropped.
+      #
+      # Errors inside the work are reported (see `report_work_error`), never raised, so one bad
+      # message can't take the room down with it.
+      def post_work(async: false, silent: false, &blk)
+        work = proc do
+          worker_pool.invoke(blk, :call, connection: self)
+        rescue => e
+          report_work_error(e)
+        end
+
+        if async
+          worker_pool.executor.post(&work)
+        else
+          enqueue(work, silent: silent)
+        end
+      end
+
+      # Log the backtrace and hand the error to the application so the failure is still
+      # discoverable even though the room carries on.
+      def report_work_error(error)
+        logger.error "Error during work execution: #{error.class.name}: #{error.message}"
+        Array(error.backtrace).first(20).each { |line| logger.error "  #{line}" }
+
+        CableRoom.report_error(
+          error,
+          room: room,
+          room_class: room_class,
+          room_key: key,
+          runner: self
+        )
+      end
+
+      # -- Inbound streams ---------------------------------------------------------------------
+
+      # Deliver every message published on `stream` to `handler`, on the room's own queue. The
+      # transport hands us decoded messages on its own thread; all we do there is queue, so one
+      # room's handler can't hold up another room's traffic. `on_live` runs once the transport
+      # confirms the subscription; anything published before that may be missed.
+      #
+      # The subscribe itself happens outside the mutex (it blocks until the Bus confirms, and the
+      # Bus thread needs the mutex to enqueue), so a room that stopped meanwhile undoes it.
+      def subscribe(stream, on_live: nil, &handler)
+        raise ArgumentError, "Block required" unless handler
+
+        stream = String(stream)
+        return if closed?
+
+        handle = inbound.subscribe(stream, on_live: on_live) do |message|
+          receive_inbound(stream, message, handler)
+        end
+
+        stopped = @mutex.synchronize do
+          closed? || (@streams[stream] = handle; @handlers[stream] = handler; false)
+        end
+        inbound.unsubscribe(stream, handle) if stopped
+      end
+
+      def unsubscribe(stream)
+        stream = String(stream)
+        handle = @mutex.synchronize { @handlers.delete(stream); @streams.delete(stream) }
+        inbound.unsubscribe(stream, handle) if handle
+      end
+
+      def unsubscribe_all
+        handles = @mutex.synchronize { @handlers.clear; @streams.to_a.tap { @streams.clear } }
+        handles.each { |stream, handle| inbound.unsubscribe(stream, handle) }
+      end
+
+      # -- Timers ------------------------------------------------------------------------------
+
+      # Run `callback` every `every`, on the room's queue, until the timer is shut down or the
+      # room stops. Returns a PeriodicTimer.
+      def start_periodic_timer(callback, every:)
+        raise "Attempt to start periodic timer on a dead room" if closed?
+
+        job = host.scheduler.schedule_every(every) do
+          post_work(async: false, silent: true) { callback.call }
+        end
+
+        PeriodicTimer.new(job)
+      end
+
+      private
+
+      def closed?
+        state == :dead || state == :shutting_down
+      end
+
+      # `start!` and `restore!` differ only in what the room does first; everything around it —
+      # the executor, the timers, tearing down on failure — is the same. A room restored with
+      # `hold_inbound` ends up :frozen with no timers running; `thaw!` starts them.
+      def start_with(final_state: :started)
+        @current_state = :starting
+        with_executor do
+          yield
+          start_periodic_timers unless final_state == :frozen
+        end
+        @current_state = final_state
+      rescue => e
+        terminate!
+        raise e
+      end
+
+      # A message from the Bus, on the Bus thread. Normally it's queued for the room; while the
+      # room is frozen it's held for the migration to relay instead. Decided under the mutex so a
+      # message can't slip onto the queue in the moment the room freezes.
+      #
+      # A room restored with `hold_inbound` holds from its very first subscribe, while it is still
+      # :starting: the adopter has to see everything that arrives before it thaws the room.
+      def receive_inbound(stream, message, handler)
+        disposition = @mutex.synchronize do
+          next :queue unless frozen? || @hold_from_start
+          # Only a fully frozen room relays; while still :freezing it holds (see `freeze!`)
+          next :relay if @hold_inbound && state == :frozen
+
+          @held_inbound << [stream, message]
+          :held
+        end
+
+        case disposition
+        when :queue
+          # A handoff marker is the migration protocol talking to itself (see Migration); it is
+          # never for the room. One can only reach a live room on a failure path, so drop it here.
+          return if Migration.marker?(message)
+
+          post_work(async: false, silent: true) { handler.call(message) }
+        when :relay then @hold_inbound.call(stream, message)
+        end
+      end
+
+      # Undo a freeze that didn't complete: the room runs on as if `freeze!` had never been
+      # called, with the messages held meanwhile queued in arrival order. Caller holds the mutex,
+      # and the state is :freezing, so nothing else is running in the room.
+      def resume_from_freeze
+        held = @held_inbound
+        @held_inbound = []
+        @hold_inbound = nil
+        @current_state = :started
+        held.each { |stream, message| inject(stream, message) }
+        start_periodic_timers
+        ping_watchdog
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      # Timers declared on the Room class with `periodically`
+      def start_periodic_timers
+        room_class.periodic_timers.each do |callback, every|
+          @periodic_timers << start_periodic_timer(-> { room.instance_exec(&callback) }, every: every)
+        end
+      end
+
+      def stop_periodic_timers
+        @periodic_timers.each(&:shutdown)
+        @periodic_timers.clear
+      end
+
+      def enqueue(work, silent:)
+        @mutex.synchronize do
+          if closed?
+            raise "Attempt to post work to dead or shutting down room" unless silent
+            return
+          end
+
+          @work_queue << work
+        end
+
+        schedule_work
+      end
+
+      # Take the next item off the queue and run it on the pool, but only if nothing from this
+      # room is running already. When it finishes, come back for the next one. This is the whole
+      # ordering guarantee: one thread per room at a time, in arrival order.
+      def schedule_work
+        @mutex.synchronize do
+          return if @processing_work
+          # A frozen room keeps its queue but runs nothing from it (while :freezing it still
+          # drains, so that `freeze!` can return to a quiet room)
+          return if state == :frozen
+
+          work = @work_queue.shift
+          return unless work
+
+          @processing_work = true
+
+          worker_pool.executor.post do
+            begin
+              work.call
+            ensure
+              @mutex.synchronize do
+                @processing_work = false
+                @work_finished.broadcast
+              end
+              schedule_work
+            end
+          end
+        end
+      end
+
+      # Room startup and shutdown used to run inside ActionCable's subscribe/unsubscribe
+      # callbacks, which Rails wraps in its executor (database connections, reloading, and so
+      # on). Keep that behavior. Nesting is fine: the executor yields straight through when it's
+      # already active on this thread.
+      def with_executor(&blk)
+        if defined?(Rails) && Rails.application
+          Rails.application.executor.wrap(&blk)
+        else
+          yield
+        end
+      end
+    end
+  end
+end