cable_room 0.6.2 → 0.7.0.beta1

data/lib/cable_room/migration.rb

@@ -0,0 +1,586 @@
+module CableRoom
+  # Moves one room from this host to a peer while its members stay connected. `Host#drain!`
+  # builds one per room and runs it; `Migration.adopt` is the receiving side, which a peer's
+  # `Host#ensure_room` runs when it wins the room's lock while a snapshot is on offer — usually
+  # because its Placement claimed our `handoff: true` provision request, but a member's own
+  # request for the room ends up there too, so nobody starts the room fresh underneath us.
+  #
+  # The old host's states, in order (`state`):
+  #
+  #   :pending     → :frozen        the room stops taking work; inbound is relayed instead (freeze!)
+  #   :frozen      → :snapshotted   the gem's state plus `snapshot_state` (Runner#snapshot)
+  #   :snapshotted → :offered       SET cr:{key}:snapshot (TTL 60 s), release the lock, publish
+  #                                 provision(handoff) — any peer may now claim it
+  #   :offered     → :relaying      a peer claimed the lock and said so (`claimed`)
+  #   :relaying    → :adopted       the peer has restored the room and replayed the list (`adopted`)
+  #   :adopted     → :migrated      markers, discard the room, tell the peer (`released`)
+  #   :offered     → :restarting    no peer within handoff_timeout: take the lock back
+  #   :restarting  → :closed        stop! with room_closed; members re-provision on their next ping
+  #   anything     → :closed        a snapshot error, or a room that won't go quiet
+  #   :offered     → :abandoned     a peer holds the lock but never finished; the room is theirs
+  #   :pending     → :gone          the room died before it could be frozen
+  #
+  # From :frozen until the handoff completes, the old host stays subscribed to the room's inbound
+  # streams and RPUSHes every message to cr:{key}:handoff (`relay`), in arrival order, with a
+  # running `seq`. Members notice nothing: their streams are unchanged, and the new host picks up
+  # the room's broadcasts.
+  #
+  # == Why nothing is lost or doubled
+  #
+  # The gap the protocol has to close: the adopter can't subscribe the room's inbound streams at
+  # the very instant the old host stops relaying, so for a while both hear every message. Messages
+  # the adopter took from the list must not run again when they also arrive live; messages
+  # published before the adopter's subscribe must not be missed. Redis gives one fact to build
+  # on: everything published on one channel reaches every subscriber in the same order. So:
+  #
+  #   1. The adopter restores the room *holding* its inbound (Runner#restore! with hold_inbound):
+  #      it subscribes, but every live message is held, not run. Then it replays the list as it
+  #      stands (LRANGE 0..-1), and publishes `adopted`.
+  #   2. The old host, on `adopted`, publishes a marker message on each inbound stream it is
+  #      subscribed to. The marker is published after the adopter's subscribe (adopted came
+  #      after it), so the adopter receives the marker and every message published between its
+  #      subscribe and the marker — and the old host receives that same sequence.
+  #   3. Every message on a stream *before* its marker was relayed to the list by the old host
+  #      (it relays until it sees the marker), and every message *after* the marker reached the
+  #      adopter live. The old host stops relaying that stream at the marker, then, once it has
+  #      seen every marker, unsubscribes and publishes `released` with how many entries the list
+  #      holds and which streams got a marker.
+  #   4. The adopter, on `released`, replays the list entries it hasn't yet (its first LRANGE up
+  #      to `relayed`), then thaws the room: of the held live messages it drops, per marked
+  #      stream, everything up to and including the marker (those came from the list) and runs
+  #      the rest; a stream with no marker (the old host wasn't listening on it) runs in full.
+  #      Runner#thaw! does the filtering and the switch to live under one lock, so no message can
+  #      land in between.
+  #
+  # Within a stream the adopter runs: list entries in relay order (= publish order, since the old
+  # host relays from its single Bus thread as it receives), then live messages after the marker,
+  # in publish order. Every message is in exactly one of those two groups. Order across streams
+  # was never guaranteed and still isn't.
+  #
+  # If the old host dies after `adopted` and never publishes `released`, the adopter waits
+  # `handoff_timeout`, then replays whatever is in the list and runs everything it held: a few
+  # messages may run twice, none are lost except ones the dead host received and never relayed.
+  # Only a crash mid-handoff can do that.
+  class Migration
+    # The message the old host publishes on each inbound stream to mark where the list ends and
+    # live traffic begins. Never reaches a room's handler: the old host is frozen, the adopter
+    # filters it, and Runner#receive_inbound drops one that arrives anywhere else.
+    MARKER_TYPE = "cable_room:handoff_marker".freeze
+
+    SNAPSHOT_TTL = 60 # seconds; the design's number: a dead handoff cleans itself up
+
+    # A safety net only: the adopter deletes the list when it is done with it, so the TTL matters
+    # only when both hosts die mid-handoff.
+    HANDOFF_LIST_TTL = 10.minutes
+
+    FINAL_STATES = %i[migrated closed abandoned gone].freeze
+
+    class << self
+      def marker?(message)
+        message.is_a?(Hash) && message["type"] == MARKER_TYPE
+      end
+
+      # The receiving side: with the room's lock already claimed (`lock_info`) and the offered
+      # `snapshot` in hand, rebuild the room here, replay what the old host relayed, and take over
+      # its inbound (see the class comment for the ordering). `Host#ensure_room` calls this
+      # whenever it wins a lock while a snapshot is on offer — whether the request that led there
+      # was the old host's handoff or a member asking for the room — so a migrating room is never
+      # started fresh underneath the old host. Returns true once the room is live here. Errors are
+      # reported and answered with false (the lock released), never raised.
+      def adopt(host, room_class, room_key, lock_info:, snapshot:, handoff_timeout: nil)
+        Adoption.new(host, room_class, room_key, lock_info: lock_info, snapshot: snapshot, handoff_timeout: handoff_timeout).run
+      end
+    end
+
+    attr_reader :host, :runner, :reason, :state, :relayed_messages, :to_host, :error, :handoff_timeout
+
+    def initialize(host, runner, reason:, handoff_timeout: nil)
+      @host = host
+      @runner = runner
+      @reason = reason
+      @handoff_timeout = (handoff_timeout || CableRoom.config.handoff_timeout).to_f
+      @state = :pending
+      @relayed_messages = 0
+      @to_host = nil
+      @error = nil
+      @nonce = SecureRandom.hex(8)
+
+      @mutex = Mutex.new
+      @changed = ConditionVariable.new
+      @adopted = false
+      @marked_streams = []
+      @adopted_handle = nil
+      @started_at = nil
+      @finished_at = nil
+    end
+
+    def room
+      runner.room
+    end
+
+    def room_class
+      runner.room_class
+    end
+
+    def room_key
+      runner.key
+    end
+
+    # The room's cluster-wide identity ("RoomClass:key"): the lock key, and what the Bus names
+    # the handoff list, snapshot, and adopted channel after.
+    def room_id
+      room_class.room_port_key(room_key)
+    end
+
+    def finished?
+      FINAL_STATES.include?(state)
+    end
+
+    def migrated?
+      state == :migrated
+    end
+
+    # Seconds from freeze to the final state (or so far, while running).
+    def duration
+      return nil unless @started_at
+      (@finished_at || monotonic_now) - @started_at
+    end
+
+    # Run the whole handoff for this room, blocking. Never raises: whatever goes wrong, the room
+    # ends up either on a peer or closed with `room_closed`, and the error is reported. Returns
+    # the final state.
+    def run
+      @started_at = monotonic_now
+      logger.info "Migrating #{room_id} (#{reason})"
+
+      case freeze
+      when :gone then return finish(:gone)
+      when :closed then return finish(:closed)
+      end
+      snapshot = take_snapshot or return finish(:closed)
+      offer(snapshot)
+
+      case wait_for_adoption
+      when :adopted then complete_handoff
+      when :restarting then restart_room
+      when :abandoned then abandon
+      end
+    rescue => e
+      fail_with(e)
+    ensure
+      unsubscribe_adopted_channel
+    end
+
+    private
+
+    # -- The steps -----------------------------------------------------------------------------
+
+    # :pending → :frozen. A room that won't go quiet within handoff_timeout is closed instead
+    # (:closed): its members re-provision, and the drain moves on. One that died before we got
+    # to it is :gone.
+    def freeze
+      frozen = runner.freeze!(timeout: handoff_timeout) { |stream, message| relay(stream, message) }
+      unless frozen
+        return :gone if runner.state == :dead
+
+        logger.warn "#{room_id} would not freeze; closing it"
+        runner.stop!(reason: "Room could not be migrated: it did not go quiet")
+        return :closed
+      end
+
+      transition(:frozen)
+      :frozen
+    end
+
+    # :frozen → :snapshotted. A room whose `snapshot_state` isn't JSON (or raises) can't move;
+    # it closes with room_closed and the error is reported, and the drain moves on.
+    def take_snapshot
+      snapshot = runner.snapshot
+      transition(:snapshotted)
+      snapshot
+    rescue => e
+      @error = e
+      logger.error "#{room_id} can't be snapshotted (#{e.class}: #{e.message}); closing it"
+      CableRoom.report_error(e, migration: self, room: room, room_class: room_class, room_key: room_key)
+      runner.stop!(reason: "Room could not be migrated: #{e.class}")
+      nil
+    end
+
+    # :snapshotted → :offered. Listen for the peer first, so its `claimed` can't beat us to the
+    # channel; park the snapshot; let the lock go; ask the fleet. Our own Placement hears the
+    # request too and ignores it, since this host is draining.
+    def offer(snapshot)
+      @adopted_handle = host.inbound.subscribe(adopted_channel) { |message| on_adopted_channel(message) }
+      bus.set(snapshot_key, snapshot, ttl: SNAPSHOT_TTL)
+      runner.release_lock!
+      bus.publish(Bus.provision_channel, {
+        type: Placement::REQUEST_TYPE,
+        handoff: true,
+        reason: reason,
+        room_class: room_class.name,
+        room_key: ::ActiveJob::Arguments.serialize([room_key]),
+        snapshot_key: snapshot_key,
+        from_host: host.id,
+        requested_at: Time.current,
+      })
+      transition(:offered)
+    end
+
+    # Wait for `adopted`. When handoff_timeout passes without it, try to take the lock back: free
+    # means no peer is on the room (:restarting). Held means a peer claimed it and is still
+    # restoring, or died holding it — a dead peer's lock lapses within LOCK_DURATION, so keep
+    # waiting until even that would have happened, then leave the room to whoever holds the lock
+    # (:abandoned).
+    def wait_for_adoption
+      deadline = monotonic_now + handoff_timeout
+      hard_deadline = deadline + room_class::LOCK_DURATION.to_f + handoff_timeout
+
+      loop do
+        wait_until(deadline) { @adopted }
+        return :adopted if @mutex.synchronize { @adopted }
+
+        if runner.retake_lock!
+          logger.warn "No host adopted #{room_id} within #{handoff_timeout}s; restarting it"
+          transition(:restarting)
+          return :restarting
+        end
+
+        return :abandoned if monotonic_now >= hard_deadline
+
+        logger.info "#{room_id}: a peer holds the lock but hasn't finished adopting; waiting"
+        deadline = [monotonic_now + handoff_timeout, hard_deadline].min
+      end
+    end
+
+    # :adopted → :migrated. Mark every stream, wait for the markers to come back through `relay`
+    # (see the class comment), let go of the room, and tell the peer the list is complete.
+    def complete_handoff
+      streams = runner.subscribed_streams
+      marker = { type: MARKER_TYPE, nonce: @nonce, from_host: host.id }
+      streams.each { |stream| bus.publish(stream, marker) }
+
+      deadline = monotonic_now + handoff_timeout
+      wait_until(deadline) { (streams - @marked_streams).empty? }
+      missing = @mutex.synchronize { streams - @marked_streams }
+      unless missing.empty?
+        # The adopter can't tell list entries from live ones on these streams; it runs both, so
+        # a few messages may run twice there. Better than holding a dead host's room forever.
+        logger.warn "#{room_id}: no marker came back on #{missing.join(', ')} within #{handoff_timeout}s"
+      end
+
+      # Unsubscribes (waiting for Redis to confirm), so nothing is relayed after `released`. The
+      # lock was released at the offer and belongs to the peer now; discard! leaves it alone.
+      runner.discard!
+
+      marked, relayed = @mutex.synchronize { [@marked_streams.dup, @relayed_messages] }
+      bus.publish(adopted_channel, { type: "released", host: host.id, nonce: @nonce, relayed: relayed, streams: marked })
+      bus.del(snapshot_key)
+
+      finish(:migrated)
+      ActiveSupport::Notifications.instrument("room_migrated.cable_room", {
+        room: room,
+        room_class: room_class,
+        room_key: room_key,
+        duration: duration,
+        relayed_messages: relayed,
+        to_host: to_host,
+        from_host: host.id,
+        reason: reason,
+      })
+      logger.info "Migrated #{room_id} to #{to_host} in #{duration.round(3)}s, relaying #{relayed} message(s)"
+      state
+    end
+
+    # :restarting → :closed. We hold the lock again, so no late peer can claim the room. Stop it
+    # first (that unsubscribes, so nothing more lands in the list), then clean up the keys.
+    def restart_room
+      runner.stop!(reason: "Room could not be migrated: no host adopted it")
+      bus.del(snapshot_key, handoff_list)
+      finish(:closed)
+    end
+
+    # A peer took the lock and never said `adopted`. The room is theirs now, or will be free
+    # again when their lock lapses; either way it isn't ours. Leave the list for them (it has a
+    # TTL) and go quietly.
+    def abandon
+      error = RuntimeError.new("#{room_id}: a peer claimed the room but never adopted it; giving it up")
+      @error = error
+      CableRoom.report_error(error, migration: self, room: room, room_class: room_class, room_key: room_key)
+      runner.discard!
+      finish(:abandoned)
+    end
+
+    # A bug, or Redis failing mid-handoff. Whatever state the room is in, its members must not be
+    # left hanging on a frozen room: take the lock back if we let it go and close the room. If a
+    # peer holds the lock by now, the room is theirs and closing ours would tell its members the
+    # wrong thing; just drop it.
+    def fail_with(error)
+      @error = error
+      logger.error "Migrating #{room_id} failed: #{error.class}: #{error.message}"
+      CableRoom.report_error(error, migration: self, room: room, room_class: room_class, room_key: room_key)
+      begin
+        if runner.state == :dead
+          nil
+        elsif runner.lock_info || runner.retake_lock!
+          runner.stop!(reason: "Room could not be migrated: #{error.class}")
+          bus.del(snapshot_key, handoff_list)
+        else
+          runner.discard!
+        end
+      rescue => cleanup_error
+        CableRoom.report_error(cleanup_error, migration: self, room: room)
+      end
+      finish(:closed)
+    end
+
+    # -- On the Bus thread ---------------------------------------------------------------------
+
+    # The `freeze!` block: every inbound message while the room is frozen, on the Bus thread, in
+    # arrival order. Relayed to the list until the stream's marker comes back; from then on the
+    # adopter has that stream live. Our own marker is the signal that a stream is done. Keep
+    # this quick: one RPUSH.
+    def relay(stream, message)
+      if self.class.marker?(message)
+        return unless message["nonce"] == @nonce
+
+        @mutex.synchronize do
+          @marked_streams << stream unless @marked_streams.include?(stream)
+          @changed.broadcast
+        end
+        return
+      end
+
+      seq = @mutex.synchronize do
+        next nil if @marked_streams.include?(stream)
+        @relayed_messages += 1
+      end
+      return unless seq
+
+      bus.rpush(handoff_list, { seq: seq, stream: stream, message: message }, ttl: HANDOFF_LIST_TTL)
+    end
+
+    def on_adopted_channel(message)
+      return unless message.is_a?(Hash)
+
+      case message["type"]
+      when "claimed"
+        @mutex.synchronize do
+          @to_host = message["host"]
+          @state = :relaying if @state == :offered
+          @changed.broadcast
+        end
+      when "adopted"
+        @mutex.synchronize do
+          @to_host = message["host"]
+          @adopted = true
+          @state = :adopted unless finished?
+          @changed.broadcast
+        end
+      end
+    end
+
+    # -- Plumbing ------------------------------------------------------------------------------
+
+    def transition(new_state)
+      @mutex.synchronize { @state = new_state }
+      logger.debug "#{room_id}: #{new_state}"
+    end
+
+    def finish(new_state)
+      @finished_at = monotonic_now
+      transition(new_state)
+      new_state
+    end
+
+    # Block until the block is true (evaluated under the mutex) or `deadline` passes.
+    def wait_until(deadline)
+      @mutex.synchronize do
+        until yield
+          remaining = deadline - monotonic_now
+          break if remaining <= 0
+          @changed.wait(@mutex, remaining)
+        end
+      end
+    end
+
+    def unsubscribe_adopted_channel
+      handle = @adopted_handle
+      @adopted_handle = nil
+      host.inbound.unsubscribe(adopted_channel, handle) if handle
+    rescue => e
+      CableRoom.report_error(e, migration: self, room: room)
+    end
+
+    def handoff_list
+      Bus.handoff_list(room_id)
+    end
+
+    def snapshot_key
+      Bus.snapshot_key(room_id)
+    end
+
+    def adopted_channel
+      Bus.adopted_channel(room_id)
+    end
+
+    def bus
+      host.bus
+    end
+
+    def logger
+      host.logger
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    # The receiving side of one handoff. See the class comment on Migration for the ordering it
+    # relies on; the numbered steps there are the numbered comments here.
+    class Adoption
+      attr_reader :host, :room_class, :room_key, :lock_info, :snapshot
+
+      def initialize(host, room_class, room_key, lock_info:, snapshot:, handoff_timeout: nil)
+        @host = host
+        @room_class = room_class
+        @room_key = room_key
+        @lock_info = lock_info
+        @snapshot = snapshot
+        @handoff_timeout = (handoff_timeout || CableRoom.config.handoff_timeout).to_f
+        @mutex = Mutex.new
+        @changed = ConditionVariable.new
+        @released = nil
+        @handle = nil
+      end
+
+      def room_id
+        room_class.room_port_key(room_key)
+      end
+
+      def run
+        # Listen before saying anything, so `released` can't slip past us
+        @handle = host.inbound.subscribe(adopted_channel) { |message| on_adopted_channel(message) }
+        bus.publish(adopted_channel, { type: "claimed", host: host.id })
+
+        # (1) Restore holding inbound: subscribed, but nothing runs yet. On failure the lock is
+        # released for us (Host#restore_room) and the old host takes the room back.
+        runner = host.restore_room(snapshot, lock_info, hold_inbound: true)
+
+        first_batch = bus.lrange(handoff_list)
+        first_batch.each { |entry| runner.inject(entry["stream"], entry["message"]) }
+        bus.publish(adopted_channel, { type: "adopted", host: host.id })
+
+        # (2)+(3) happen on the old host. (4): wait for `released`, replay the rest of the list,
+        # and go live, dropping the held messages that the list already covered.
+        released = wait_for_released
+        rest = bus.lrange(handoff_list).drop(first_batch.size)
+        marked_streams = []
+        nonce = nil
+        if released
+          rest = rest.first([released["relayed"].to_i - first_batch.size, 0].max)
+          marked_streams = Array(released["streams"])
+          nonce = released["nonce"]
+        else
+          logger.warn "#{room_id}: the old host never said it released the room; running everything held " \
+                      "(a few messages may run twice)"
+        end
+
+        runner.thaw! do |held|
+          rest.map { |entry| [entry["stream"], entry["message"]] } + live_after_markers(held, marked_streams, nonce)
+        end
+
+        bus.del(handoff_list)
+        logger.info "Adopted #{room_id} from #{released&.dig('host') || 'its old host'}: replayed " \
+                    "#{first_batch.size + rest.size} relayed message(s)"
+        true
+      rescue => e
+        logger.error "Adopting #{room_id} failed: #{e.class}: #{e.message}"
+        CableRoom.report_error(e, adoption: self, room_class: room_class, room_key: room_key)
+        recover(runner)
+      ensure
+        host.inbound.unsubscribe(adopted_channel, @handle) if @handle
+      end
+
+      private
+
+      # The held live messages that the list didn't cover: on a marked stream, only what came
+      # after the marker; on an unmarked stream, everything. Markers themselves never run.
+      def live_after_markers(held, marked_streams, nonce)
+        seen_marker = {}
+        held.select do |stream, message|
+          if Migration.marker?(message)
+            seen_marker[stream] = true if nonce.nil? || message["nonce"] == nonce
+            false
+          elsif marked_streams.include?(stream)
+            seen_marker[stream]
+          else
+            true
+          end
+        end
+      end
+
+      def wait_for_released
+        deadline = monotonic_now + @handoff_timeout
+        @mutex.synchronize do
+          until @released
+            remaining = deadline - monotonic_now
+            break if remaining <= 0
+            @changed.wait(@mutex, remaining)
+          end
+          @released
+        end
+      end
+
+      def on_adopted_channel(message)
+        return unless message.is_a?(Hash) && message["type"] == "released"
+
+        @mutex.synchronize do
+          @released = message
+          @changed.broadcast
+        end
+      end
+
+      # Something broke. After the room was restored here it holds the lock and its members'
+      # ports, so it must not stay frozen: run it with everything it held (at least once beats
+      # never). If the failure came before or during the restore, make sure the lock is free
+      # (Host#restore_room has usually done it; unlocking twice is harmless) so the old host can
+      # take the room back, and answer false: nothing was claimed.
+      def recover(runner)
+        if runner.nil? || runner.state == :dead
+          CableRoom.lock_manager.unlock(lock_info)
+          return false
+        end
+
+        runner.thaw!
+        true
+      rescue => e
+        CableRoom.report_error(e, adoption: self, room_class: room_class, room_key: room_key)
+        false
+      end
+
+      def handoff_list
+        Bus.handoff_list(room_id)
+      end
+
+      def snapshot_key
+        Bus.snapshot_key(room_id)
+      end
+
+      def adopted_channel
+        Bus.adopted_channel(room_id)
+      end
+
+      def bus
+        host.bus
+      end
+
+      def logger
+        host.logger
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/cable_room/periodic_timer.rb

@@ -0,0 +1,18 @@
+module CableRoom
+  # A running `periodically` timer or reaper. Rooms stop timers with #shutdown, but Rufus jobs
+  # are cancelled with #unschedule. Adapt the one to the other here rather than adding #shutdown
+  # to Rufus::Scheduler::Job for the whole process.
+  class PeriodicTimer
+    attr_reader :job
+
+    delegate :unschedule, :scheduled?, :next_time, to: :job
+
+    def initialize(job)
+      @job = job
+    end
+
+    def shutdown
+      job.unschedule
+    end
+  end
+end