cable_room 0.6.2 → 0.7.0.beta1

data/lib/cable_room/placement.rb

@@ -0,0 +1,258 @@
+require 'concurrent'
+require 'active_support/inflector'
+
+module CableRoom
+  # The host side of provisioning. A `create: true` membership publishes a provision request on
+  # `Bus.provision_channel`; every Host runs one Placement, which hears every request and decides
+  # whether this host should be the one to start the room.
+  #
+  # The decision is a race weighted by load. Each host waits `provision_delay_ms` for every room
+  # it already runs (plus up to one room's worth of jitter, to break ties), then tries to take the
+  # room's Redlock through `Host#ensure_room`. The lightest host wakes first and usually wins;
+  # everyone else finds the lock held and does nothing. A host that is draining or shut down never
+  # claims. The same code runs in :inline and :remote — the only difference is which process the
+  # Host lives in.
+  #
+  # Requests arrive on the Bus subscriber thread, which must stay quick, so `handle` only checks
+  # and schedules; the wait and the claim run later on the Host's worker pool via a timer. That way
+  # a long wait never blocks other Bus traffic, and a slow room startup never blocks another claim.
+  #
+  # Specs make the race deterministic by passing `jitter: -> { 0 }` (or any callable returning
+  # 0..1) and a small `provision_delay_ms`; `delay_for(load)` is the only place the numbers meet.
+  class Placement
+    # Raised (and reported, never propagated) for a request this host can't act on: not a hash, an
+    # unknown room class, a class that isn't a Room, or a key that won't deserialize.
+    class InvalidRequest < ArgumentError; end
+
+    REQUEST_TYPE = "provision".freeze
+
+    attr_reader :host, :channel
+
+    # `jitter` returns a Float in 0..1 (defaults to `rand`); `executor` is where the claim runs
+    # (defaults to the Host's worker pool); `config` defaults to `CableRoom.config`.
+    def initialize(host, jitter: nil, executor: nil, config: nil)
+      @host = host
+      @jitter = jitter || -> { Kernel.rand }
+      @executor = executor
+      @config = config
+      @channel = Bus.provision_channel
+
+      @mutex = Mutex.new
+      @idle = ConditionVariable.new
+      @pending = {} # room identity => the scheduled claim, so one host never queues two claims for one room
+      @running = false # subscribed to the channel
+      @stopped = false # `stop` was called; no claim may go through until `start` again
+      @handle = nil
+    end
+
+    # Subscribe to the provision channel on the Host's inbound transport. Blocks until the Bus
+    # confirms, so a request published after this returns is heard. Calling it again while running
+    # re-subscribes (harmless; the Bus replaces the handler), which lets a Host re-arm its listener.
+    def start
+      @mutex.synchronize do
+        @running = true
+        @stopped = false
+      end
+      @handle = host.inbound.subscribe(channel) { |request| handle(request) }
+      self
+    end
+
+    # Stop hearing requests and drop every claim still waiting on its timer. Waits (up to `wait`
+    # seconds) for a claim that is already mid-attempt to finish, so nothing starts a room after
+    # this returns. The unsubscribe blocks until Redis confirms it, which doubles as a barrier: a
+    # request published before the stop has either been handled or is gone for good.
+    def stop(wait: 5)
+      was_running, handle = @mutex.synchronize do
+        @stopped = true
+        [@running, @handle].tap do
+          @running = false
+          @handle = nil
+        end
+      end
+
+      host.inbound.unsubscribe(channel, handle) if was_running
+
+      # A cancelled timer never runs its block (and so never runs the `finish` in it), so forget
+      # it here. A claim that already started keeps its entry until it finishes; wait for those.
+      tasks = @mutex.synchronize { @pending.to_a }
+      tasks.each { |room_id, task| finish(room_id) if task.cancel }
+      wait_idle(timeout: wait)
+      self
+    end
+
+    def running?
+      @mutex.synchronize { @running }
+    end
+
+    def stopped?
+      @mutex.synchronize { @stopped }
+    end
+
+    # How many claims are waiting on their timer or mid-attempt right now.
+    def pending_count
+      @mutex.synchronize { @pending.size }
+    end
+
+    def idle?
+      pending_count.zero?
+    end
+
+    # Block until every scheduled claim has finished (won, lost, or been cancelled). Returns true,
+    # or false if `timeout` passes first. Specs use this to observe a losing host give up.
+    def wait_idle(timeout: 5)
+      deadline = monotonic_now + timeout
+      @mutex.synchronize do
+        until @pending.empty?
+          remaining = deadline - monotonic_now
+          return false if remaining <= 0
+          @idle.wait(@mutex, remaining)
+        end
+      end
+      true
+    end
+
+    # One request off the Bus. Runs on the subscriber thread, so it does only the cheap checks
+    # (is this a room we know, do we already run it, are we allowed to claim) and then schedules
+    # the delayed claim. Returns the scheduled task, or nil when there is nothing to do.
+    def handle(request)
+      room_class, room_key = parse(request)
+      room_id = room_class.room_port_key(room_key)
+
+      return skip(room_id, "placement is stopped") if stopped?
+      return skip(room_id, "host is shutting down") if host.shutdown?
+      return skip(room_id, "host is draining") if host.draining?
+      return skip(room_id, "already running here") if running_here?(room_id)
+
+      load = host.open_rooms_count
+      delay = delay_for(load)
+
+      # A handoff request gets a claim of its own even while a member's plain request for the
+      # same room is pending: the plain claim may have lost the lock to the old host moments
+      # before it let go, and the handoff must not be skipped for it. The lock keeps them safe.
+      pending_key = request["handoff"] == true ? "#{room_id} (handoff)" : room_id
+
+      @mutex.synchronize do
+        return skip(room_id, "a claim is already pending here") if @pending.key?(pending_key)
+
+        task = Concurrent::ScheduledTask.new(delay, executor: executor) do
+          begin
+            claim(room_class, room_key, room_id, request, delay: delay, load: load)
+          ensure
+            finish(pending_key)
+          end
+        end
+        @pending[pending_key] = task
+        task.execute
+      end
+    rescue InvalidRequest => e
+      CableRoom.report_error(e, placement: self, request: request)
+      nil
+    end
+
+    # Seconds to wait before racing for a room, given how many rooms this host already runs:
+    # `provision_delay_ms × (load + jitter)`, jitter in 0..1. A host with more rooms always waits
+    # longer than one with fewer, and the jitter only decides the order between hosts with the same
+    # load. A host running nothing waits at most one `provision_delay_ms`.
+    def delay_for(load)
+      base = config.provision_delay_ms / 1000.0
+      base * (load + @jitter.call.to_f)
+    end
+
+    private
+
+    # The claim itself, after the delay, on the worker pool. Re-checks the cheap conditions first,
+    # since the world may have moved while we waited, then races for the Redlock. Losing is
+    # normal — it means a lighter host got there first — so it's only a debug line.
+    def claim(room_class, room_key, room_id, request, delay:, load:)
+      return if stopped?
+      return if host.shutdown? || host.draining?
+      return if running_here?(room_id)
+
+      payload = {
+        host: host,
+        room_class: room_class,
+        room_key: room_key,
+        delay: delay,
+        open_rooms: load,
+        request: request,
+      }
+
+      claimed = ActiveSupport::Notifications.instrument("provision_claimed.cable_room", payload) do |notification|
+        # `handoff: true` (with a `reason`) is a draining host offering a room to its peers. The
+        # race is the same — the lock decides — but the winner rebuilds the room from the offered
+        # snapshot and replays what the old host relayed instead of starting it from scratch (see
+        # CableRoom::Migration.adopt). A plain request starts the room fresh.
+        handoff = request["handoff"] == true
+        notification[:handoff] = handoff
+        notification[:claimed] = host.ensure_room(room_class, room_key, handoff_only: handoff)
+      end
+
+      if claimed
+        logger.info "Claimed #{room_id} after #{(delay * 1000).round}ms with #{load} room(s) open"
+      else
+        logger.debug "Lost the race for #{room_id}: another host holds its lock"
+      end
+    rescue => e
+      logger.error "Failed to claim #{room_id}: #{e.class}: #{e.message}"
+      CableRoom.report_error(e, placement: self, room_class: room_class, room_key: room_key, request: request)
+    end
+
+    def finish(room_id)
+      @mutex.synchronize do
+        @pending.delete(room_id)
+        @idle.broadcast if @pending.empty?
+      end
+    end
+
+    def skip(room_id, why)
+      logger.debug "Ignoring provision request for #{room_id}: #{why}"
+      nil
+    end
+
+    def running_here?(room_id)
+      host.runners.any? { |runner| runner.room_class.room_port_key(runner.key) == room_id }
+    end
+
+    # Turn a decoded Bus message into a Room class and a key. Only `Room::Base` descendants may be
+    # provisioned: the class name comes off the wire, and `constantize` on an arbitrary string is
+    # not something a request should be able to do.
+    def parse(request)
+      raise InvalidRequest, "provision request must be a Hash (got #{request.class})" unless request.is_a?(Hash)
+
+      type = request["type"]
+      raise InvalidRequest, "expected a #{REQUEST_TYPE.inspect} request (got #{type.inspect})" unless type == REQUEST_TYPE
+
+      name = request["room_class"]
+      klass = name.is_a?(String) ? ActiveSupport::Inflector.safe_constantize(name) : nil
+      unless klass.is_a?(Class) && klass < Room::Base
+        raise InvalidRequest, "#{name.inspect} is not a CableRoom::Room::Base subclass"
+      end
+
+      [klass, deserialize_key(request["room_key"])]
+    end
+
+    # Keys travel the way `extra` does (ActiveJob's argument serializer), so a record key comes
+    # back as the record and a string comes back as a string.
+    def deserialize_key(serialized)
+      ::ActiveJob::Arguments.deserialize(Array(serialized)).first
+    rescue StandardError => e
+      raise InvalidRequest, "room_key can't be deserialized: #{e.class}: #{e.message}"
+    end
+
+    def executor
+      @executor || host.worker_pool.executor
+    end
+
+    def config
+      @config || CableRoom.config
+    end
+
+    def logger
+      host.logger
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/cable_room/ports.rb

@@ -1,4 +1,17 @@
 module CableRoom
+  # The `ports[...]` DSL shared by a Room and by a member's RoomMembership. A port is a named
+  # lane between a room and its members; `ports[:x] << msg` sends on it and `ports[:x].stream`
+  # listens on it. Which transport that touches depends on which side you're on, and each side
+  # implements the three methods below:
+  #
+  #   * Member side (RoomMembership): sending publishes on the Bus channel for the room's port
+  #     (member→room), listening is an ActionCable stream (room→member, fed by the broadcaster).
+  #   * Room side (Room::Base and Room::HostAdapter): sending goes through the configured
+  #     CableRoom::Broadcaster, listening subscribes on the Bus through the room's Host.
+  #
+  # So a member's `ports[:custom] << msg` and a room's `ports[:custom].stream` meet on the Bus,
+  # and a room's `ports[:custom] << msg` and a member's `ports[:custom].stream` meet on the
+  # broadcaster's streams. Neither side needs to know that.
   module Ports
     extend ActiveSupport::Concern
 
@@ -6,28 +19,23 @@ module CableRoom
       @ports_proxy ||= PortsProxy.new(self)
     end
 
+    # Listen on `port`, handing each decoded message to the block. Ports streamed with
+    # `auto_close: true` are stopped by `close_streamed_ports!`.
     def stream_port(port, auto_close: true, &blk)
-      @cable_channel.stream_from(room_port_key(port), coder: ActiveSupport::JSON, &blk)
-      _streamed_ports << port if auto_close
+      raise NotImplementedError, "#{self.class} must implement stream_port"
     end
 
     def close_streamed_ports!
-      _streamed_ports.each do |port|
-        @cable_channel.stop_stream_from(room_port_key(port))
-      end
-      _streamed_ports.clear
+      raise NotImplementedError, "#{self.class} must implement close_streamed_ports!"
     end
 
+    # Send `data` on `port`.
     def port_transmit(port, data)
-      ActionCable.server.broadcast(room_port_key(port), data)
+      raise NotImplementedError, "#{self.class} must implement port_transmit"
     end
 
     protected
 
-    def _room_channel_class
-      room_class::Channel
-    end
-
     def _streamed_ports
       @_streamed_ports ||= Set.new
     end

data/lib/cable_room/railtie.rb

@@ -5,21 +5,12 @@ require "active_model/railtie"
 
 module CableRoom
   class Railtie < Rails::Railtie # :nodoc:
-    rake_tasks do
-    end
-
-    console do |app|
-    end
-
-    runner do
-    end
-
     initializer "cable_room.hook_action_cable_restart" do
       module ActionCableServerExtensions
+        # ActionCable restarts when the app reloads; stop every room with it. A process that
+        # hosts no rooms (a web process in :remote) has nothing to stop.
         def restart
-          CableRoom::ChannelTracker.instance.each_room_channel do |chan|
-            chan.unsubscribe_from_channel
-          end
+          CableRoom::Host.current&.stop_all_rooms!
           super
         end
       end

data/lib/cable_room/room/base.rb

@@ -8,52 +8,50 @@ module CableRoom
       WATCH_DOG_INTERVAL = 15.seconds
 
       class << self
+        # Start this room in the current process if nobody else is running it. Takes the room's
+        # Redlock first, so exactly one process wins; returns false when the lock is held.
+        #
+        # Only a process that hosts rooms can do this: every process in :inline, and only
+        # `cable_room server` in :remote. Anywhere else `Host.instance` raises Host::NotHosting
+        # rather than letting a room start where it doesn't belong.
         def ensure(key = nil)
-          lock_key = room_port_key(key)
-          lock_info = CableRoom.lock_manager.lock(lock_key, self::LOCK_DURATION.in_milliseconds)
-          return false unless lock_info
-
-          vchannel = self::Channel.new(
-            lock_info,
-            self,
-            key,
-            {
-              watchdog_interval: self::WATCH_DOG_INTERVAL,
-              lock_duration: self::LOCK_DURATION,
-            }
-          )
-          vchannel.subscribe_to_channel
-
-          true
-        rescue => e
-          CableRoom.lock_manager.unlock(lock_info) if lock_info
-          raise e
+          Host.instance.ensure_room(self, key)
         end
 
+        # The stream name for a room, or for one of its ports: "RoomClass:key" or
+        # "RoomClass:key:port". Members and rooms both use this, so it has to be stable.
         def room_port_key(room_key, port = nil)
-          full_key = [room_key]
-          full_key << port if port
-          self::Channel.broadcasting_for(full_key)
+          parts = [name, room_key]
+          parts << port if port
+          parts.map { |part| serialize_key_part(part) }.join(":")
         end
 
+        # The Bus channel members publish on to reach a room's inbound port `port`, and the one
+        # the room's Host subscribes to for it. `room_port_key(room_key)` ("RoomClass:key") is the
+        # room's cluster-wide identity — it's already the Redlock key — so the main port lands on
+        # the design's `cr:{room_key}:in` and a custom port on `cr:{room_key}:in:{port}`.
+        def inbound_channel(room_key, port = ROOM_IN_CHANNEL)
+          port = port.to_s == ROOM_IN_CHANNEL.to_s ? nil : serialize_key_part(port)
+          Bus.inbound_channel(room_port_key(room_key), port)
+        end
+
+        # Send a message to a room from anywhere in the app: same Bus channel a member uses
         def send_message(room_key, data, port: ROOM_IN_CHANNEL)
-          ActionCable.server.broadcast(room_port_key(room_key, port), data)
+          CableRoom.bus.publish(inbound_channel(room_key, port), data)
         end
 
+        # Every instance of this room class running in this process. Empty when the process
+        # hosts no rooms at all (a web process in :remote).
         def locally_running_instances
-          ChannelTracker.instance.room_channels.select do |chan|
-            chan.room_class == self
-          end.map(&:room)
+          Room.locally_open_rooms.select { |room| room.class == self }
         end
 
-        protected
+        private
 
-        def periodically(method, every:, &blk)
-          blk ||= method
-          blk = -> { send(method) } unless blk.is_a?(Proc)
-          self::Channel.periodically(every:) do
-            room.instance_exec(&blk)
-          end
+        # Same rule ActionCable uses to name broadcasts: records go by their GlobalID, anything
+        # else by `to_param`
+        def serialize_key_part(part)
+          part.respond_to?(:to_gid_param) ? part.to_gid_param : part.to_param
         end
       end
 
@@ -61,9 +59,10 @@ module CableRoom
 
       include Callbacks
       include Threading
-      include ChannelAdapter
+      include HostAdapter
 
       include Lifecycle
+      include Snapshotting
       include Reaping
       include InputHandling
 
@@ -74,18 +73,25 @@ module CableRoom
 
       attr_reader :key
 
-      delegate :logger, :params, to: :@cable_channel
+      delegate :logger, to: :@runner
 
-      def initialize(vchannel, key = nil)
+      def initialize(runner, key = nil)
         super()
         @key = key
-        @cable_channel = vchannel
+        @runner = runner
       end
 
       def <<(data)
         port_transmit(ROOM_OUT_CHANNEL, data)
       end
 
+      # The room's sending side of Ports: room→member messages go out on the stream named by
+      # `room_port_key(port)` through the configured CableRoom::Broadcaster. (The listening side
+      # is in Room::HostAdapter.) Public because `ports[:x] << msg` reaches it through a PortProxy.
+      def port_transmit(port, data)
+        broadcaster.broadcast(room_port_key(port), data)
+      end
+
       protected
 
       def room_class
@@ -96,8 +102,8 @@ module CableRoom
         self.class.room_port_key(key, sub_channel)
       end
 
-      def start_periodic_timer(callback, every:)
-        @cable_channel.send(:start_periodic_timer, -> { room.instance_exec(&callback) }, every:)
+      def broadcaster
+        Broadcaster.current
       end
     end
   end

data/lib/cable_room/room/host_adapter.rb

@@ -0,0 +1,52 @@
+module CableRoom
+  module Room
+    # The Room's side of its connection to the Host: which streams it listens on and which
+    # timers it runs. Everything here goes through the Room's runner (`@runner`, a Host::Runner).
+    module HostAdapter
+      extend ActiveSupport::Concern
+
+      included do
+        # `[callback, every]` pairs declared with `periodically`. A class_attribute so subclasses
+        # inherit their parents' timers and add their own without touching the parent's list.
+        class_attribute :periodic_timers, instance_writer: false, instance_predicate: false, default: []
+      end
+
+      class_methods do
+        protected
+
+        # Run `method` (or the block) on the Room's thread every `every`, for as long as the
+        # Room is running
+        def periodically(method, every:, &blk)
+          blk ||= method
+          blk = -> { send(method) } unless blk.is_a?(Proc)
+          self.periodic_timers += [[blk, every]]
+        end
+      end
+
+      # The room's listening side of Ports: subscribe, through the Host, to the Bus channel
+      # members publish on for `port` (see Room::Base.inbound_channel). `on_live` runs once the
+      # subscription is confirmed, the same as it does for members.
+      def stream_port(port, auto_close: true, on_live: nil, &blk)
+        @runner.subscribe(inbound_channel(port), on_live: on_live, &blk)
+        _streamed_ports << port if auto_close
+      end
+
+      def close_streamed_ports!
+        _streamed_ports.each do |port|
+          @runner.unsubscribe(inbound_channel(port))
+        end
+        _streamed_ports.clear
+      end
+
+      protected
+
+      def inbound_channel(port)
+        self.class.inbound_channel(key, port)
+      end
+
+      def start_periodic_timer(callback, every:)
+        @runner.start_periodic_timer(-> { instance_exec(&callback) }, every: every)
+      end
+    end
+  end
+end

data/lib/cable_room/room/lifecycle.rb

@@ -5,8 +5,10 @@ module CableRoom
 
       included do
         after_startup do
-          logger.info "Started"
-          self << { type: 'room_opened' }
+          logger.info restored? ? "Restored" : "Started"
+          # A restored room's members were attached the whole time; telling them the room opened
+          # would make every one of them re-announce for nothing.
+          self << { type: 'room_opened' } unless restored?
         end
 
         before_shutdown do
@@ -22,27 +24,42 @@ module CableRoom
       protected
 
       def lifecycle_state
-        @cable_channel.state
+        @runner.state
       end
 
-      # Requests that the Room shut down gracefully, processing any pending messages
+      # Requests that the Room shut down gracefully, processing any pending messages. A frozen
+      # room (mid-migration) shuts down the same way; that's the "no peer took it" path.
       def shutdown!(reason = "Room requested shutdown")
         @shutdown_reason = reason
         logger.info "Shutdown requested: #{reason}"
-        @cable_channel.initiate_shutdown(reason)
+        @runner.initiate_shutdown(reason)
       end
 
       # Stops the room synchronously, ignoring remaining messages
       def stop!
-        @cable_channel.unsubscribe_from_channel
+        @runner.stop!
       end
 
       private
 
-      def _startup
-        ActiveSupport::Notifications.instrument("room_opened.cable_room", { room: self }) do
+      # What `room_closed` will say. The runner sets it when the host, not the room, decides to
+      # close (a server shutting down, a migration nobody adopted).
+      def _shutdown_reason=(reason)
+        @shutdown_reason = reason
+      end
+
+      # The startup chain. A restore (see Snapshotting#_restore) runs the same callbacks — they
+      # are what subscribe the inbound port and start the reapers — but swaps `startup` for
+      # `restore_state(app_state)` when the room defines it.
+      def _startup(restoring: false, app_state: nil)
+        event = restoring ? "room_restored.cable_room" : "room_opened.cable_room"
+        ActiveSupport::Notifications.instrument(event, { room: self }) do
           run_callbacks :startup do
-            startup if respond_to?(:startup)
+            if restoring && respond_to?(:restore_state, true)
+              restore_state(app_state)
+            else
+              startup if respond_to?(:startup)
+            end
           end
         end
       end

data/lib/cable_room/room/port_management.rb

@@ -68,6 +68,13 @@ module CableRoom
         { client_port: client_port || tag, **kwargs }
       end
 
+      # How many member ports are attached right now. Public, unlike `connected_clients`, because
+      # the Host reads it from outside the room's thread to total up `Host#open_ports`; a plain
+      # size is safe to read that way and a moment's staleness doesn't matter for a gauge.
+      def open_port_count
+        @_port_clients.size
+      end
+
       def reply(message = nil, **kwargs)
         raise ArgumentError, "Can only use reply when handling a message" unless message_origin
         raise ArgumentError, "Must provide message or block" unless message || block_given?
@@ -151,6 +158,29 @@ module CableRoom
         end
       end
 
+      # -- Snapshot and restore (see CableRoom::Snapshot) ----------------------------------------
+
+      def _snapshot_port_clients
+        @_port_clients.values.map(&:to_snapshot)
+      end
+
+      # Re-create every port exactly as it was, without running port_connected callbacks: the
+      # members are still there and were already acknowledged. A port whose user can't be found
+      # any more (its record was deleted) is dropped rather than failing the whole room; that
+      # member will be re-acknowledged when it next pings.
+      def _restore_port_clients(entries)
+        Array(entries).each do |entry|
+          client = PortClient.new(self, entry["token"])
+          begin
+            client.restore!(entry)
+          rescue ::ActiveJob::DeserializationError => e
+            logger.warn "Dropping port #{entry['token']} from the snapshot: #{e.message}"
+            next
+          end
+          @_port_clients[client.token] = client
+        end
+      end
+
       def check_port_inactivity
         return unless @_port_clients
 
@@ -208,6 +238,33 @@ module CableRoom
         def tag!(*tags)
           self[:tags].merge(tags.flatten.map(&:to_sym))
         end
+
+        SNAPSHOT_OWN_KEYS = %w[tags as last_seen_at].freeze
+
+        # This port as one `port_clients` entry of a CableRoom::Snapshot. `as` goes through the
+        # same serializer the member used to send it (a GlobalID for a record). Everything else on
+        # the port — what the member's `extra:` merged in, plus anything the room stored with
+        # `message_origin[:x] = ...` — travels under `metadata`, the same way, so records survive.
+        def to_snapshot
+          metadata = @metadata.to_h.except(*SNAPSHOT_OWN_KEYS)
+          {
+            token: token,
+            tags: self[:tags].map(&:to_s),
+            as: Snapshot.serialize_argument(self[:as]),
+            last_seen_at: Snapshot.encode_time(self[:last_seen_at]),
+            metadata: Snapshot.serialize_argument(metadata),
+          }
+        end
+
+        # The inverse of `to_snapshot`. `last_seen_at` is kept, not reset: the port is exactly as
+        # old as it was, and its member's next ping refreshes it as usual.
+        def restore!(entry)
+          merge!(Snapshot.deserialize_argument(entry["metadata"]))
+          tag!(entry["tags"])
+          self[:as] = Snapshot.deserialize_argument(entry["as"])
+          self[:last_seen_at] = Snapshot.decode_time(entry["last_seen_at"]) || self[:last_seen_at]
+          self
+        end
       end
     end
   end