openlanes 3.0.1 → 3.1.0

data/lib/openlanes/live/memory_store.rb

@@ -126,6 +126,17 @@ module OpenLanes
       # @return [Integer]
       def xlen(key) = locked { (live(key) || []).size }
 
+      # @return [Integer] how many were removed
+      def xtrim(key, minid:)
+        locked do
+          stream = live(key) or next 0
+          floor = id_of(minid.include?("-") ? minid : "#{minid}-0")
+          before = stream.size
+          stream.reject! { (id_of(it.id) <=> floor).negative? }
+          before - stream.size
+        end
+      end
+
       # @return [Boolean] whether the group is new
       def xgroup_create(key, group, from: "0")
         locked do
@@ -141,7 +152,7 @@ module OpenLanes
       # @return [Array<Store::Entry>]
       def xreadgroup(key, group, consumer, id: ">", count: 100)
         locked do
-          state = @groups[key][group] or raise Unavailable, "NOGROUP #{group} on #{key}"
+          state = @groups[key][group] or raise GroupMissing, "NOGROUP #{group} on #{key}"
           stream = live(key) || []
           if id == ">"
             fresh = stream.select { |e| (id_of(e.id) <=> state[:last]).positive? }.first(count)

data/lib/openlanes/live/news.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "json"
+
+module OpenLanes
+  module Live
+    # The nudge that tells every open page's event stream to look again
+    # ({Keys#news}). It carries no data: whoever wrote something the page
+    # shows writes it first, then publishes `{"kind": ...}`, and the stream
+    # re-reads Redis as it always has. The data stays in its keys and
+    # streams, so a nudge lost to a dropped connection costs latency, never
+    # correctness, and the stream's cursors work exactly as before.
+    #
+    # Who publishes, and why each one does:
+    #
+    # - `board`: {Publisher}, on the Dispatcher's tick, only when the
+    #   snapshot changed apart from its clock, so an idle engine is quiet.
+    # - `reply`: {Commands}, on the Dispatcher's tick, for each reply.
+    # - `copilot`: {Copilot}, in the API server, for each turn written.
+    # - `app`: {Apps::Runner}, when a process changes state or the app stops.
+    #   Its once-a-second heartbeat does not: nothing on the page moves.
+    # - `chat`: {Chat}, after a step handed a page something to show.
+    #   Its events live in this process, not in Redis, so the nudge is the
+    #   only way a stream blocked on Redis can hear of them.
+    # - `stop`: {Web::Server#stop}, so open streams end now rather than at
+    #   their next ping.
+    #
+    # The `--demo` engine publishes `board` and `reply` the way the real one does.
+    module News
+      # Every kind a nudge may name. The stream does not branch on them; they
+      # are there for whoever watches the channel with `redis-cli SUBSCRIBE`.
+      KINDS = %w[board reply copilot app chat stop].freeze
+
+      module_function
+
+      # Nudge every stream following +keys+' project on +store+'s database.
+      # Best effort: the write it announces is already in Redis, and a
+      # stream that misses the nudge still finds it at its next ping.
+      #
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param keys [OpenLanes::Live::Keys]
+      # @param kind [String] one of {KINDS}
+      # @return [Integer] how many subscriptions heard it, 0 when Redis did not answer
+      def publish(store:, keys:, kind:)
+        bus = EventBus.for(store)
+        bus.publish(keys.news(db: bus.db), JSON.generate(kind:))
+      rescue Unavailable
+        0
+      end
+
+      # A subscription of its own to the nudges, for one event stream. The
+      # caller closes it: each is a Redis connection.
+      #
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param keys [OpenLanes::Live::Keys]
+      # @return [OpenLanes::Live::Subscriber, OpenLanes::Live::MemorySubscriber]
+      # @raise [OpenLanes::Live::Unavailable]
+      def subscribe(store:, keys:) = EventBus.for(store).then { it.subscribe(channels: [keys.news(db: it.db)]) }
+    end
+  end
+end

data/lib/openlanes/live/node.rb

@@ -46,7 +46,27 @@ module OpenLanes
       ID_LENGTH = 24
       CHECKOUT_LENGTH = 12
 
+      # A declared `user@host`: one of each, no spaces, no second `@`.
+      IDENTITY = /\A(?<user>[^@\s]+)@(?<host>[^@\s]+)\z/
+
+      # A declared identity that is not `user@host`.
+      class Invalid < ArgumentError; end
+
       class << self
+        # This checkout as a machine of another name: `openlanes web --demo
+        # --identity demo@second-machine` registers as a second machine on the
+        # same Redis and repository, through the same digest every process
+        # is already named by, so nothing reads a second naming scheme.
+        #
+        # @param identity [String] `user@host`
+        # @param root [String] the checkout
+        # @return [OpenLanes::Live::Node]
+        # @raise [Invalid]
+        def declared(identity, root:, **)
+          match = IDENTITY.match(identity.to_s.strip) or raise Invalid, "an identity is user@host, not #{identity.inspect}"
+          current(root:, user: match[:user], host: match[:host], **)
+        end
+
         # @param root [String] the checkout
         # @param user [String]
         # @param host [String]

data/lib/openlanes/live/outage.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Live
+    # What the engine does when its Redis stops answering (W32).
+    #
+    # A blip is not an outage: Redis that is back within {GRACE} seconds
+    # changes nothing. Past that, every running agent is paused through its
+    # control file ({Execution::Control::PAUSE}), nothing new starts, and the
+    # outage goes into the run's state file, the record that survives Redis.
+    # Redis is then asked again after 1, 1, 2, 3, 5, 8 … seconds, the
+    # Fibonacci numbers scaled by `redis_backoff.initial` and capped at
+    # `redis_backoff.cap` ({MAX_GAP} by default), so a short outage is noticed
+    # quickly, a long one is not hammered, and a Redis that comes back is
+    # never left unnoticed for longer than the cap. With full jitter (the
+    # default) each gap is drawn uniformly below that, so engines sharing one
+    # Redis do not all ask in the same second. When it answers, the agents
+    # carry on, their clocks having stood still meanwhile. When it has not answered for
+    # `redis_outage_hours` (an hour by default), the run shuts down the way
+    # `q` does, and says why.
+    #
+    # Run on the dispatcher's tick, ahead of everything that needs Redis.
+    class Outage
+      # Seconds Redis may be unreachable before the agents are paused.
+      GRACE = 10
+
+      # Seconds between pings while Redis answers.
+      PING_EVERY = 1
+
+      # The longest gap between two asks while paused, in seconds. Without
+      # it the Fibonacci gaps pass 16 minutes within fifteen asks, and a Redis
+      # back after 17 minutes would leave every agent paused for 10 more.
+      MAX_GAP = 30
+
+      # How many past outages the state file keeps.
+      HISTORY = 20
+
+      # One outage that ended in Redis answering: wall-clock bounds, and the
+      # length by the monotonic clock, which a changed system time cannot
+      # skew. {OutageReport} turns it into `run.outage` (plan 022).
+      Window = Data.define(:started_at, :ended_at, :seconds)
+
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param limit [Numeric] seconds paused before the run gives up
+      # @param clock [#call] monotonic seconds
+      # @param wall [#call] wall time, for the state file
+      # @param initial [Numeric] seconds before the first ask; the Fibonacci gaps are its multiples
+      # @param cap [Numeric] the longest gap between two asks
+      # @param jitter [Boolean] full jitter: each gap is drawn from [0, gap)
+      # @param random [#rand] the jitter's source; a spec hands in a fixed one
+      def initialize(store:, limit:, clock: -> { UI.monotonic }, wall: -> { Time.now },
+        initial: 1, cap: MAX_GAP, jitter: true, random: Random.new)
+        @store = store
+        @limit = limit
+        @initial = initial
+        @cap = cap
+        @jitter = jitter
+        @random = random
+        @clock = clock
+        @wall = wall
+        @state = :up
+        @down_since = nil
+        @last_ping = nil
+        @resumed = []
+      end
+
+      # @return [Symbol] :up, :paused or :gave_up
+      attr_reader :state
+
+      # One tick.
+      #
+      # @param dispatcher [OpenLanes::Engine::Dispatcher] for its state file
+      # @return [void]
+      def call(dispatcher)
+        @state_file = dispatcher.state_file
+        now = @clock.call
+        case @state
+        when :up then watch(now)
+        when :paused then probe(now)
+        end
+      end
+
+      # @return [Boolean] whether agents are paused for this outage
+      def paused? = @state == :paused
+
+      # @return [Proc] {#call}, for `Dispatcher#before_tick`
+      def to_proc = method(:call).to_proc
+
+      # Something to run once Redis answers again, after the agents resume.
+      #
+      # @yieldparam window [Window]
+      # @return [void]
+      def on_resume(&hook) = @resumed << hook
+
+      # @return [Window, nil] the last outage that ended in Redis answering
+      attr_reader :window
+
+      private
+
+      # @return [void]
+      def watch(now)
+        return if @last_ping && now - @last_ping < PING_EVERY
+
+        @last_ping = now
+        if answers?
+          @down_since = nil
+        else
+          @down_since ||= now
+          pause(now) if now - @down_since >= GRACE
+        end
+      end
+
+      # @return [void]
+      def pause(now)
+        @state = :paused
+        @delays = fibonacci
+        @attempts = 0
+        @next_probe = next_probe(now)
+        @record = { "since" => (@wall.call - (now - @down_since)).iso8601, "paused_at" => @wall.call.iso8601,
+                    "limit_seconds" => @limit.round, "attempts" => 0 }
+        UI.log("Redis has not answered for #{GRACE}s: pausing every agent, starting nothing new, " \
+               "and giving up after #{hours(@limit)}",
+          level: :warn)
+        Execution::Control.pause!
+        save
+      end
+
+      # @return [void]
+      def probe(now)
+        return if now < @next_probe
+
+        @attempts += 1
+        @record["attempts"] = @attempts
+        @record["last_attempt_at"] = @wall.call.iso8601
+        if answers?
+          resume(now)
+        elsif now - @down_since >= @limit
+          give_up(now)
+        else
+          @next_probe = next_probe(now)
+          save
+        end
+      end
+
+      # @return [void]
+      def resume(now)
+        @state = :up
+        @record["resumed_at"] = @wall.call.iso8601
+        @record["seconds"] = (now - @down_since).round
+        UI.log("Redis answers again after #{(now - @down_since).round}s and #{@attempts} attempts: resuming every agent")
+        Execution::Control.resume!
+        @window = Window.new(started_at: @wall.call - (now - @down_since),
+          ended_at: @wall.call,
+          seconds: now - @down_since)
+        @down_since = nil
+        save(over: true)
+        @resumed.each { |hook| hook.call(@window) }
+      end
+
+      # @return [void]
+      def give_up(now)
+        @state = :gave_up
+        @record["gave_up_at"] = @wall.call.iso8601
+        @record["seconds"] = (now - @down_since).round
+        UI.log("Redis has not answered for #{hours(@limit)}: shutting the run down", level: :error)
+        Execution::Control.quit!
+        save(over: true)
+      end
+
+      # The next Fibonacci gap, scaled, capped, jittered, and never past the
+      # limit: the last ask is the one at the limit.
+      #
+      # @return [Float]
+      def next_probe(now) = [now + gap, @down_since + @limit].min
+
+      # @return [Float] seconds to the next ask
+      def gap
+        gap = [@delays.next * @initial, @cap].min.to_f
+        @jitter ? gap * @random.rand : gap
+      end
+
+      # @return [Boolean]
+      def answers?
+        @store.ping == true
+      rescue Unavailable, SystemCallError
+        false
+      end
+
+      # 1, 1, 2, 3, 5, 8, …
+      #
+      # @return [Enumerator]
+      def fibonacci
+        Enumerator.new do |out|
+          a, b = 1, 1
+          loop do
+            out << a
+            a, b = b, a + b
+          end
+        end
+      end
+
+      # The current outage, or, once it is over, into the history. Saved at
+      # once: this is the record a person reads to learn why agents stood
+      # still, and the run may not live to save it later.
+      #
+      # @param over [Boolean]
+      # @return [void]
+      def save(over: false)
+        return unless @state_file
+
+        @state_file.outage = over ? nil : @record
+        @state_file.outages = (@state_file.outages + [@record]).last(HISTORY) if over
+        @state_file.save
+      rescue SystemCallError => e
+        UI.log("outage not recorded: #{e.message}", level: :warn)
+      end
+
+      # @return [String]
+      def hours(seconds)
+        hours = seconds / 3600.0
+        hours == hours.round ? "#{hours.round} hour#{"s" unless hours.round == 1}" : "#{hours.round(2)} hours"
+      end
+    end
+  end
+end

data/lib/openlanes/live/outage_report.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module OpenLanes
+  module Live
+    # What the engine says once Redis is back: how long it was gone, how
+    # that was measured, and which copy of the run state won.
+    #
+    # The duration prefers this node's last heartbeat in Redis, because it
+    # is the last moment Redis provably had the engine's word. A heartbeat
+    # far older than the first failure is not that: it is what an RDB
+    # rollback restores, and measuring from it would overstate the outage by
+    # however long the snapshot was stale. Then the engine's own monotonic
+    # window is the answer, and `measured_by` says so.
+    class OutageReport
+      # Seconds a heartbeat may precede the first failure and still count.
+      # The publisher beats every tick, so a live one is a few seconds old.
+      FRESH = 15
+
+      # @param store [#hget]
+      # @param keys [OpenLanes::Live::Keys]
+      # @param node [OpenLanes::Live::Node]
+      # @param window [OpenLanes::Live::Outage::Window]
+      # @param reconciled_from [Symbol, nil] `:redis`, `:file` or nil, from
+      #   {RunRecord#reconcile}
+      def initialize(store:, keys:, node:, window:, reconciled_from:)
+        @store = store
+        @keys = keys
+        @node = node
+        @window = window
+        @reconciled_from = reconciled_from
+      end
+
+      # The `run.outage` wire shape (plan 022, shared with the page).
+      #
+      # @return [Hash]
+      def to_h
+        { started_at:      iso(started_at),
+          ended_at:        iso(@window.ended_at),
+          seconds:         seconds,
+          measured_by:     measured_by,
+          reconciled_from: @reconciled_from&.to_s }
+      end
+
+      # @return [String] one log line
+      def line
+        from = @reconciled_from ? "run state from #{@reconciled_from}" : "no run state to reconcile"
+        "Redis is back after #{seconds}s (by #{measured_by}), #{from}; resuming"
+      end
+
+      # @return [Integer]
+      def seconds
+        (heartbeat ? @window.ended_at - heartbeat : @window.seconds).round
+      end
+
+      # @return [String] `"heartbeat"` or `"clock"`
+      def measured_by = heartbeat ? "heartbeat" : "clock"
+
+      private
+
+      # @return [Time]
+      def started_at = heartbeat || @window.started_at
+
+      # This node's heartbeat, when it is from just before the first failure.
+      #
+      # @return [Time, nil]
+      def heartbeat
+        return @heartbeat if defined?(@heartbeat)
+
+        @heartbeat = read_heartbeat
+      end
+
+      # @return [Time, nil]
+      def read_heartbeat
+        json = @store.hget(@keys.nodes, @node.name) or return nil
+        at = Time.iso8601(JSON.parse(json)["heartbeat_at"].to_s)
+        at if at <= @window.started_at && @window.started_at - at <= FRESH
+      rescue JSON::ParserError, ArgumentError, Unavailable
+        nil
+      end
+
+      # @param time [Time]
+      # @return [String]
+      def iso(time) = time.utc.iso8601
+    end
+  end
+end

data/lib/openlanes/live/publisher.rb

@@ -60,6 +60,12 @@ module OpenLanes
         @on_lost = block
       end
 
+      # {OutageReport#to_h} of the outage this run last recovered from,
+      # published as `run.outage` from the next snapshot on.
+      #
+      # @return [Hash, nil]
+      attr_accessor :outage
+
       # @return [OpenLanes::Live::Meter]
       attr_reader :meter
 
@@ -77,11 +83,12 @@ module OpenLanes
         @meter.record(board.rows, now:)
         snapshot = build(board, now)
         json = JSON.generate(snapshot)
-        @store.multi do |tx|
+        EventBus.for(@store).transaction do |tx|
           tx.set(@keys.snapshot, json, ex: Keys::SNAPSHOT_TTL)
-          tx.xadd(@keys.board, { seq: snapshot[:seq], json: }, maxlen: Keys::BOARD_LENGTH)
+          tx.append(@keys.board, { seq: snapshot[:seq], json: }, maxlen: Keys::BOARD_LENGTH)
           tx.hset(@keys.nodes, @node.name, JSON.generate(@node.wire(heartbeat_at: now)))
         end
+        nudge(snapshot)
         lost = @claims&.renew || []
         unless lost.empty?
           UI.log("claims lost to another machine: #{lost.join(", ")}; stopping their agents")
@@ -153,9 +160,42 @@ module OpenLanes
           claims:      @claims,
           run_state:   @run_state,
           meter:       @meter,
+          # The URL the engine publishes through, not one read back from the
+          # environment: the page's status bar says which Redis the claims and
+          # run state really live in, and only the store that wrote them knows.
+          redis_url:   @store.url,
+          outage:      @outage,
           log:).to_h
       end
 
+      # Fields that move with the clock alone: the sequence number, the
+      # snapshot's time, this node's heartbeat, and how long each plan has
+      # been worked on. A car's `elapsed` is left in on purpose: while an
+      # agent runs, its seconds counter is what the page is there to show.
+      #
+      # @param snapshot [Hash]
+      # @return [Hash] what the page would see change
+      def content(snapshot)
+        snapshot.except(:seq, :at)
+                .merge(node:  snapshot[:node].except(:heartbeat_at),
+                  plans: Array(snapshot[:plans]).map { it.except(:elapsed) })
+      end
+
+      # Nudges the open pages ({News}) when this tick's snapshot says
+      # something the last one did not. The snapshot is still written every
+      # tick, because its expiry is how a node is known to be alive; only
+      # the nudge waits for news, so an engine with nothing to do is quiet.
+      #
+      # @param snapshot [Hash]
+      # @return [void]
+      def nudge(snapshot)
+        now = content(snapshot)
+        return if now == @published
+
+        News.publish(store: @store, keys: @keys, kind: "board")
+        @published = now
+      end
+
       # One more than the last published, even across an engine restart:
       # the API server reads a change of `seq` as news, and a restart that
       # counted from one again would look like the past.

data/lib/openlanes/live/run_record.rb

@@ -50,7 +50,7 @@ module OpenLanes
         @state_file = dispatcher.state_file
         redis = read_redis
         file = @state_file&.live
-        newer = [redis, file].compact.max_by { it["seq"].to_i }
+        newer = newest(redis, file)
         # A copy that is missing is behind, unless both are: then there is
         # nothing to keep until a person changes something.
         @in_redis = redis ? redis["seq"].to_i : (newer && -1) || 0
@@ -62,6 +62,37 @@ module OpenLanes
           ordinal, agent = key.split("/", 2)
           dispatcher.prefer_model(ordinal, agent, model)
         end
+        restore_concurrency(dispatcher)
+      end
+
+      # A `/concurrency` set before the restart holds after it (plan 020.00).
+      # One this run cannot take (a shared checkout runs one agent) is said
+      # once and dropped, rather than stopping the engine from starting.
+      #
+      # @param dispatcher [OpenLanes::Engine::Dispatcher]
+      # @return [void]
+      def restore_concurrency(dispatcher)
+        limit = @run_state.concurrency or return
+        dispatcher.concurrency = limit
+      rescue OpenLanes::Error => e
+        UI.log("concurrency #{limit} from before the restart not kept: #{e.message}")
+      end
+
+      # After Redis comes back: which copy is the newer, by the same `seq`
+      # rule as {#restore}. An RDB rollback leaves Redis behind or empty and
+      # the file wins. The engine's own state kept going through the outage,
+      # so nothing is restored from either; both marks are reset to what is
+      # really there, so the next {#flush} rewrites whichever copy is behind.
+      #
+      # @return [Symbol, nil] `:redis` or `:file`, or nil with neither copy
+      def reconcile
+        @warned = false
+        redis = read_redis
+        file = @state_file&.live
+        @in_redis = redis ? redis["seq"].to_i : -1
+        @in_file = file ? file["seq"].to_i : -1
+        newer = newest(redis, file) or return nil
+        newer.equal?(redis) ? :redis : :file
       end
 
       # Bring both copies up to the state's `seq`. The file is written by the
@@ -98,6 +129,13 @@ module OpenLanes
 
       private
 
+      # The copy with the higher `seq`; Redis on a tie, being listed first.
+      #
+      # @param redis [Hash, nil]
+      # @param file [Hash, nil]
+      # @return [Hash, nil]
+      def newest(redis, file) = [redis, file].compact.max_by { it["seq"].to_i }
+
       # @return [void]
       def mirror
         return unless @state_file && @in_file != @run_state.seq

data/lib/openlanes/live/run_state.rb

@@ -9,7 +9,8 @@ module OpenLanes
     # Held in the engine, and kept by {RunRecord} in Redis and in
     # `.plans/openlanes-state.json` (W6), so an engine that restarts still
     # knows them. Every change moves {#seq} on by one, which is how the
-    # record tells which of its two copies is newer. A model choice also
+    # record tells which of its two copies is newer. So does the concurrency
+    # a person set with `/concurrency` (plan 020.00). A model choice also
     # lives in the executor, which is what applies it; this copy is what the
     # page is shown and what a restart hands back to the executor.
     class RunState
@@ -18,9 +19,24 @@ module OpenLanes
         @held_questions = []
         @held_cars = []
         @models = {}
+        @concurrency = nil
         @seq = 0
       end
 
+      # @return [Integer, nil] the limit `/concurrency` set; nil when nobody did
+      def concurrency = @mutex.synchronize { @concurrency }
+
+      # @param limit [Integer]
+      # @return [void]
+      def concurrency=(limit)
+        @mutex.synchronize do
+          next if @concurrency == limit
+
+          @concurrency = limit
+          @seq += 1
+        end
+      end
+
       # @param id [String] `"<ordinal>/B<n>"`
       # @param held [Boolean]
       # @return [void]
@@ -67,7 +83,7 @@ module OpenLanes
       def to_h
         @mutex.synchronize do
           { "seq" => @seq, "held_questions" => @held_questions.dup, "held_cars" => @held_cars.dup,
-            "models" => @models.dup }
+            "models" => @models.dup }.merge(@concurrency ? { "concurrency" => @concurrency } : {})
         end
       end
 
@@ -80,6 +96,7 @@ module OpenLanes
           @held_questions = Array(data["held_questions"]).map(&:to_s)
           @held_cars = Array(data["held_cars"]).map(&:to_s)
           @models = (data["models"].is_a?(Hash) ? data["models"] : {}).to_h { |k, v| [k.to_s, v.to_s] }
+          @concurrency = data["concurrency"].is_a?(Integer) && data["concurrency"].positive? ? data["concurrency"] : nil
           @seq = data["seq"].to_i
         end
         self

data/lib/openlanes/live/session.rb

@@ -19,13 +19,16 @@ module OpenLanes
       # @param node [OpenLanes::Live::Node]
       # @param repo [String]
       # @param clock [Proc] wall time
+      # @param outage [OpenLanes::Live::Outage] what pauses the run while
+      #   Redis is gone; a spec hands in one on its own clock
       def initialize(store:, root:, tree:, agents:, project: nil, node: Node.current(root:), repo: Repo.for(root),
-        clock: -> { Time.now })
+        clock: -> { Time.now }, outage: Outage.new(store:, limit: Config.current.outage_limit, **Config.current.redis_backoff))
         @store = store
         @node = node
         @keys = Keys.new(repo:, node: node.name)
         @run_state = RunState.new
         @record = RunRecord.new(store:, keys: @keys, run_state: @run_state)
+        @outage = outage
         @claims = Claims.new(store:, keys: @keys)
         @publisher = Publisher.new(store:,
           keys: @keys,
@@ -69,20 +72,45 @@ module OpenLanes
       # @return [OpenLanes::Live::RunRecord]
       attr_reader :record
 
+      # @return [OpenLanes::Live::Outage]
+      attr_reader :outage
+
       # @return [Proc] the runner's `on_board:`
       def on_board = @publisher.to_proc
 
       # Wire the run in: the run state a previous engine left is taken back
-      # on the first tick, commands are applied at the top of each tick, and
-      # the log is heard from now on.
+      # on the first tick, commands are applied at the top of each tick, mail
+      # published to Redis is drained into the plans' `state.json`, and the
+      # log is heard from now on.
       #
       # @param dispatcher [OpenLanes::Engine::Dispatcher]
       # @return [void]
       def attach(dispatcher)
         @publisher.listen
         @publisher.on_lost { |ordinals| dispatcher.fence(ordinals) }
+        # First, so a tick that finds Redis gone pauses before anything else
+        # on it asks Redis for something (W32).
+        dispatcher.before_tick(&@outage)
         dispatcher.before_tick(&@record)
         dispatcher.before_tick(&@commands)
+        dispatcher.mail = Mail.new(store: @store, keys: @keys)
+        @outage.on_resume { recover(it) }
+      end
+
+      # Redis answers again: say how long it was gone and which copy of the
+      # run state won, and publish that as `run.outage`.
+      #
+      # @param window [OpenLanes::Live::Outage::Window]
+      # @return [OpenLanes::Live::OutageReport]
+      def recover(window)
+        report = OutageReport.new(store: @store,
+          keys: @keys,
+          node: @node,
+          window:,
+          reconciled_from: @record.reconcile)
+        UI.log(report.line)
+        @publisher.outage = report.to_h
+        report
       end
 
       # The run is over: the run state is written down, claims go back, and