openlanes 3.0.0 → 3.1.0

data/lib/openlanes/live/keys.rb

@@ -62,6 +62,112 @@ module OpenLanes
       # @return [String]
       def run(of = node) = "#{base}:#{of}:run"
 
+      # Pub/sub: `{json}`, one line of this node's {Engine::EventJournal},
+      # mirrored after it is fsync'd. The file is the record; a subscriber
+      # that misses a message reads the file.
+      #
+      # @param of [String]
+      # @return [String]
+      def journal(of = node) = "#{base}:#{of}:journal"
+
+      # Stream: `{json}`, one `CopilotTurn`: the conversation with this
+      # node's copilot (W17), the person's messages and its answers in the
+      # order they were said. {Copilot} writes and reads it.
+      #
+      # @param of [String]
+      # @return [String]
+      def copilot(of = node) = "#{base}:#{of}:copilot"
+
+      # Hash: `session`, `model`, `up` and `down`, the copilot's `claude`
+      # session id and what it has spent in it. Kept beside {#copilot} so a
+      # reloaded page or a restarted server carries on the same conversation
+      # (W17: one session, resumed on each message).
+      #
+      # @param of [String]
+      # @return [String]
+      def copilot_session(of = node) = "#{base}:#{of}:copilot-session"
+
+      # Stream: `{json}`, one mailbox message and the plan it belongs to,
+      # what `openlanes mail send` publishes after writing `state.json`
+      # (plan 003, B3). One stream per checkout; the plan rides in the entry,
+      # and each plan reads it through a group of its own ({Keys#mail_group}).
+      #
+      # @param of [String]
+      # @return [String]
+      def mail(of = node) = "#{base}:#{of}:mail"
+
+      # The consumer group one plan drains {#mail} through. Every group sees
+      # every entry, so a plan's harness thread acks what it read, its own
+      # or not, without taking anything from another plan's.
+      #
+      # @param plan [#to_s] e.g. "003.00"
+      # @return [String]
+      def mail_group(plan) = "mail:#{plan}"
+
+      # Pub/sub channel: one chat's messages, joins, leaves and idle notices
+      # (plan 006). Redis delivers a `PUBLISH` to every database's
+      # subscribers, so the database is part of every chat channel's name: a
+      # chat between nodes on database 13 never reaches a page on database 0.
+      #
+      # @param id [String] the chat's UUID
+      # @param db [Integer] the Redis database the nodes share
+      # @return [String]
+      def chat_room(id, db:) = "#{chat_base(db)}:room:#{id}"
+
+      # Pub/sub pattern matching every {#chat_room} on +db+, which each API
+      # server subscribes to once, so joining a chat needs no new subscription.
+      #
+      # @param db [Integer]
+      # @return [String]
+      def chat_rooms(db:) = "#{chat_base(db).gsub(/[*?\[\]\\]/) { "\\#{it}" }}:room:*"
+
+      # Pub/sub channel: one machine's chat inbox, where invites to it and
+      # their answers arrive. Every API server for that machine listens, so
+      # an invite reaches every page open there (D9).
+      #
+      # @param machine [String] a node id, 24 hex digits of `user@host`
+      # @param db [Integer]
+      # @return [String]
+      def chat_inbox(machine, db:) = "#{chat_base(db)}:machine:#{machine}"
+
+      # String: how an invite was settled (`accepted`, `declined`, `expired`),
+      # `SET NX EX`, so the first answer from any page wins (D9).
+      #
+      # @param invite [String] the invite's UUID
+      # @return [String]
+      def chat_answer(invite) = "#{base}:chat:invite:#{invite}"
+
+      # @param db [Integer]
+      # @return [String]
+      def chat_base(db) = "#{base}:chat:#{db}"
+
+      # Pub/sub channel: "look again", one per project, published by
+      # whoever wrote something an open page shows ({News}). Every node's
+      # engine on the project publishes here, because every page shows every
+      # node's board. The database is in the name for the reason it is in
+      # {#chat_room}'s: a `PUBLISH` reaches every database's subscribers.
+      #
+      # @param db [Integer] the Redis database the nodes share
+      # @return [String]
+      def news(db:) = "#{base}:news:#{db}"
+
+      # Hash: plan ordinal to that plan's running app, its process statuses
+      # and a heartbeat ({Apps::Listing}). One per machine, not per
+      # checkout: the app runs in a plan's worktree and the control room in
+      # the main checkout, and both are this machine.
+      #
+      # @param machine [String] a node id, 24 hex digits of `user@host`
+      # @return [String]
+      def apps(machine = node.split(":").first) = "#{base}:apps:#{machine}"
+
+      # Stream: `{json}`, one request from the Preview tab's toolbar
+      # (restart, stop), read by `openlanes app start`. Not {#commands}: the
+      # engine never runs an app, so it would never read them.
+      #
+      # @param machine [String]
+      # @return [String]
+      def app_commands(machine = node.split(":").first) = "#{apps(machine)}:commands"
+
       # String: the node name holding a plan, `SET NX EX`.
       #
       # @param ordinal [#to_s] e.g. "004.00"
@@ -80,6 +186,14 @@ module OpenLanes
       BOARD_LENGTH = 100
       REPLIES_LENGTH = 1000
 
+      # Length of {#copilot}, kept approximately: a conversation the page
+      # can scroll back through, not an archive.
+      COPILOT_LENGTH = 500
+
+      # Seconds a message stays in {#mail}. `state.json` is the record; the
+      # stream only has to outlast a harness tick, and an hour is generous.
+      MAIL_TTL = 3600
+
       # Seconds a snapshot outlives its last refresh.
       SNAPSHOT_TTL = 30
 

data/lib/openlanes/live/mail.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module OpenLanes
+  module Live
+    # Mail between agents, over Redis as well as through `state.json`
+    # (plan 003, B3).
+    #
+    # `state.json` stays the record: `openlanes mail send` writes it first
+    # and publishes here only when Redis answers, so a run without Redis
+    # loses nothing. What the stream adds is delivery to the harness thread
+    # beside each agent, which drains it once a second (B4) instead of
+    # re-reading every plan's file.
+    #
+    # A stream with a consumer group per plan, not pub/sub: a message
+    # published while nobody drains waits for the next drain instead of
+    # vanishing. Every publish trims what is older than {Keys::MAIL_TTL}
+    # and renews the key's expiry, so the stream never outgrows an hour.
+    #
+    # Delivery is acked once the drain's block kept it, never before: an
+    # entry acked ahead of a failed write would be gone from the stream with
+    # only a log line left. An entry not kept stays pending and is offered
+    # again on the next drain. An ack says the bytes arrived, nothing more: a message is read when its recipient runs `openlanes mail ack`,
+    # and that lands in `state.json`, never here.
+    class Mail
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param keys [OpenLanes::Live::Keys]
+      # @param now [Proc] wall time, which the trim's cutoff is taken from
+      def initialize(store:, keys:, now: -> { Time.now })
+        @store = store
+        @keys = keys
+        @now = now
+      end
+
+      # Publish one message that `state.json` already holds.
+      #
+      # @param plan [#to_s] the plan's number, e.g. "003.00"
+      # @param message [OpenLanes::Plans::Mailbox::Message, Hash]
+      # @return [void]
+      def publish(plan:, message:)
+        payload = JSON.generate("plan" => plan.to_s, "message" => wire(message))
+        EventBus.for(@store).transaction do |tx|
+          tx.append(@keys.mail, { json: payload })
+          tx.xtrim(@keys.mail, minid: cutoff)
+          tx.expire(@keys.mail, Keys::MAIL_TTL)
+        end
+      end
+
+      # Every message for +plan+ that this plan's group has not had kept
+      # yet: first the ones offered before and left pending, then the new.
+      # Entries for other plans, and entries that are not messages, are
+      # acked in this plan's group straight away, which leaves other plans'
+      # groups untouched.
+      #
+      # With a block, each message is yielded and acked only when the block
+      # returns truthy; one it returns false for, or raises on, stays pending
+      # for the next drain. Without one, each is acked as it is returned.
+      #
+      # @param plan [#to_s]
+      # @param consumer [String] who is draining, e.g. the agent's name
+      # @yieldparam message [Hash] as `state.json` holds it
+      # @yieldreturn [Boolean] whether it was kept
+      # @return [Array<Hash>] messages as `state.json` holds them, oldest
+      #   first; with a block, the ones left pending
+      def drain(plan:, consumer:)
+        group = @keys.mail_group(plan)
+        @store.expire(@keys.mail, Keys::MAIL_TTL) if @store.xgroup_create(@keys.mail, group)
+        entries = @store.xreadgroup(@keys.mail, group, consumer, id: "0") + @store.xreadgroup(@keys.mail, group, consumer)
+        entries.filter_map do |entry|
+          message = message_in(entry, plan.to_s)
+          kept = message.nil? || !block_given? || yield(message)
+          @store.xack(@keys.mail, group, entry.id) if kept
+          message if message && (kept ? !block_given? : true)
+        end
+      end
+
+      # Which plans have mail in the stream after +cursor+, so a drain can
+      # reach a plan with nobody running on it: a worktree's `mail send` has
+      # no other way into the main checkout's `state.json`, and the stream
+      # forgets it after {Keys::MAIL_TTL}.
+      #
+      # @param cursor [String] the last stream id already looked at, "0" for
+      #   all of it
+      # @return [Array(Array<String>, String)] the plans, and the new cursor
+      def plans_since(cursor)
+        plans = []
+        loop do
+          entries = @store.xread(@keys.mail, id: cursor)
+          break if entries.empty?
+
+          cursor = entries.last.id
+          plans.concat(entries.filter_map { plan_in(it) })
+        end
+        [plans.uniq, cursor]
+      end
+
+      private
+
+      # The oldest id a publish keeps: {Keys::MAIL_TTL} seconds ago.
+      #
+      # @return [String]
+      def cutoff = "#{((@now.call.to_f - Keys::MAIL_TTL) * 1000).to_i}-0"
+
+      # @param message [OpenLanes::Plans::Mailbox::Message, Hash]
+      # @return [Hash] string keys, `at` in ISO 8601 as `state.json` has it
+      def wire(message)
+        hash = message.to_h.transform_keys(&:to_s)
+        hash.merge("at" => iso8601(hash["at"])).compact
+      end
+
+      # @param at [String, Time, nil]
+      # @return [String, nil]
+      def iso8601(at)
+        return at.iso8601 if at.respond_to?(:iso8601)
+
+        Time.parse(at.to_s).iso8601
+      rescue ArgumentError
+        at&.to_s
+      end
+
+      # @param entry [OpenLanes::Live::Store::Entry]
+      # @return [String, nil] the plan the entry is for
+      def plan_in(entry)
+        data = JSON.parse(entry.fields["json"].to_s)
+        data["plan"] if data.is_a?(Hash) && data["plan"].is_a?(String)
+      rescue JSON::ParserError
+        nil
+      end
+
+      # @param entry [OpenLanes::Live::Store::Entry]
+      # @param plan [String]
+      # @return [Hash, nil] the message, when the entry is one and for +plan+
+      def message_in(entry, plan)
+        data = JSON.parse(entry.fields["json"].to_s)
+        data["message"] if data.is_a?(Hash) && data["plan"] == plan && data["message"].is_a?(Hash)
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/openlanes/live/memory_store.rb

@@ -22,6 +22,7 @@ module OpenLanes
         @groups = Hash.new { |h, k| h[k] = {} }
         @last_ms = 0
         @last_seq = 0
+        @subscribers = []
       end
 
       # @return [String] what {Live.mask} shows for it
@@ -125,6 +126,17 @@ module OpenLanes
       # @return [Integer]
       def xlen(key) = locked { (live(key) || []).size }
 
+      # @return [Integer] how many were removed
+      def xtrim(key, minid:)
+        locked do
+          stream = live(key) or next 0
+          floor = id_of(minid.include?("-") ? minid : "#{minid}-0")
+          before = stream.size
+          stream.reject! { (id_of(it.id) <=> floor).negative? }
+          before - stream.size
+        end
+      end
+
       # @return [Boolean] whether the group is new
       def xgroup_create(key, group, from: "0")
         locked do
@@ -140,7 +152,7 @@ module OpenLanes
       # @return [Array<Store::Entry>]
       def xreadgroup(key, group, consumer, id: ">", count: 100)
         locked do
-          state = @groups[key][group] or raise Unavailable, "NOGROUP #{group} on #{key}"
+          state = @groups[key][group] or raise GroupMissing, "NOGROUP #{group} on #{key}"
           stream = live(key) || []
           if id == ">"
             fresh = stream.select { |e| (id_of(e.id) <=> state[:last]).positive? }.first(count)
@@ -193,6 +205,29 @@ module OpenLanes
       # @return [Array<String>]
       def scan(prefix) = locked { @data.keys.select { |k| k.start_with?(prefix) && !live(k).nil? }.sort }
 
+      # ---- pub/sub ----------------------------------------------------------
+
+      # @return [Integer] how many subscriptions received it
+      def publish(channel, message)
+        receivers = locked { @subscribers.dup }
+        receivers.sum do |subscriber|
+          subscriber.deliver(channel, message.to_s)
+          subscriber.matches(channel)
+        end
+      end
+
+      # @return [OpenLanes::Live::MemorySubscriber]
+      def subscriber(channels: [], patterns: [])
+        MemorySubscriber.new(store: self, channels:, patterns:).tap { |s| locked { @subscribers << s } }
+      end
+
+      # @param subscriber [OpenLanes::Live::MemorySubscriber]
+      # @return [void]
+      def unsubscribe(subscriber) = locked { @subscribers.delete(subscriber) }
+
+      # @return [Integer] always 0: one process, one keyspace
+      def db = 0
+
       # ---- transactions -----------------------------------------------------
 
       # @yieldparam tx [Store::Transaction]

data/lib/openlanes/live/memory_subscriber.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Live
+    # {Subscriber}'s methods over {MemoryStore#publish}: a queue in this
+    # process, matched by exact channel or by Redis-style glob.
+    class MemorySubscriber
+      # @param store [OpenLanes::Live::MemoryStore]
+      # @param channels [Array<String>]
+      # @param patterns [Array<String>]
+      def initialize(store:, channels: [], patterns: [])
+        @store = store
+        @channels = channels
+        @patterns = patterns
+        @queue = Thread::Queue.new
+      end
+
+      # @param channel [String]
+      # @return [Integer] how many of its subscriptions match, as Redis counts receivers
+      def matches(channel)
+        @channels.count(channel) + @patterns.count { File.fnmatch?(it, channel) }
+      end
+
+      # @param channel [String]
+      # @param message [String]
+      # @return [void]
+      def deliver(channel, message) = matches(channel).times { @queue << [channel, message] }
+
+      # @param timeout [Float]
+      # @return [Array(String, String), nil]
+      def next_message(timeout) = @queue.pop(timeout: [timeout, 0].max)
+
+      # @return [void]
+      def close
+        @store.unsubscribe(self)
+        @queue.close
+      end
+    end
+  end
+end

data/lib/openlanes/live/news.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "json"
+
+module OpenLanes
+  module Live
+    # The nudge that tells every open page's event stream to look again
+    # ({Keys#news}). It carries no data: whoever wrote something the page
+    # shows writes it first, then publishes `{"kind": ...}`, and the stream
+    # re-reads Redis as it always has. The data stays in its keys and
+    # streams, so a nudge lost to a dropped connection costs latency, never
+    # correctness, and the stream's cursors work exactly as before.
+    #
+    # Who publishes, and why each one does:
+    #
+    # - `board`: {Publisher}, on the Dispatcher's tick, only when the
+    #   snapshot changed apart from its clock, so an idle engine is quiet.
+    # - `reply`: {Commands}, on the Dispatcher's tick, for each reply.
+    # - `copilot`: {Copilot}, in the API server, for each turn written.
+    # - `app`: {Apps::Runner}, when a process changes state or the app stops.
+    #   Its once-a-second heartbeat does not: nothing on the page moves.
+    # - `chat`: {Chat}, after a step handed a page something to show.
+    #   Its events live in this process, not in Redis, so the nudge is the
+    #   only way a stream blocked on Redis can hear of them.
+    # - `stop`: {Web::Server#stop}, so open streams end now rather than at
+    #   their next ping.
+    #
+    # The `--demo` engine publishes `board` and `reply` the way the real one does.
+    module News
+      # Every kind a nudge may name. The stream does not branch on them; they
+      # are there for whoever watches the channel with `redis-cli SUBSCRIBE`.
+      KINDS = %w[board reply copilot app chat stop].freeze
+
+      module_function
+
+      # Nudge every stream following +keys+' project on +store+'s database.
+      # Best effort: the write it announces is already in Redis, and a
+      # stream that misses the nudge still finds it at its next ping.
+      #
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param keys [OpenLanes::Live::Keys]
+      # @param kind [String] one of {KINDS}
+      # @return [Integer] how many subscriptions heard it, 0 when Redis did not answer
+      def publish(store:, keys:, kind:)
+        bus = EventBus.for(store)
+        bus.publish(keys.news(db: bus.db), JSON.generate(kind:))
+      rescue Unavailable
+        0
+      end
+
+      # A subscription of its own to the nudges, for one event stream. The
+      # caller closes it: each is a Redis connection.
+      #
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param keys [OpenLanes::Live::Keys]
+      # @return [OpenLanes::Live::Subscriber, OpenLanes::Live::MemorySubscriber]
+      # @raise [OpenLanes::Live::Unavailable]
+      def subscribe(store:, keys:) = EventBus.for(store).then { it.subscribe(channels: [keys.news(db: it.db)]) }
+    end
+  end
+end

data/lib/openlanes/live/node.rb

@@ -46,7 +46,27 @@ module OpenLanes
       ID_LENGTH = 24
       CHECKOUT_LENGTH = 12
 
+      # A declared `user@host`: one of each, no spaces, no second `@`.
+      IDENTITY = /\A(?<user>[^@\s]+)@(?<host>[^@\s]+)\z/
+
+      # A declared identity that is not `user@host`.
+      class Invalid < ArgumentError; end
+
       class << self
+        # This checkout as a machine of another name: `openlanes web --demo
+        # --identity demo@second-machine` registers as a second machine on the
+        # same Redis and repository, through the same digest every process
+        # is already named by, so nothing reads a second naming scheme.
+        #
+        # @param identity [String] `user@host`
+        # @param root [String] the checkout
+        # @return [OpenLanes::Live::Node]
+        # @raise [Invalid]
+        def declared(identity, root:, **)
+          match = IDENTITY.match(identity.to_s.strip) or raise Invalid, "an identity is user@host, not #{identity.inspect}"
+          current(root:, user: match[:user], host: match[:host], **)
+        end
+
         # @param root [String] the checkout
         # @param user [String]
         # @param host [String]

data/lib/openlanes/live/outage.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+module OpenLanes
+  module Live
+    # What the engine does when its Redis stops answering (W32).
+    #
+    # A blip is not an outage: Redis that is back within {GRACE} seconds
+    # changes nothing. Past that, every running agent is paused through its
+    # control file ({Execution::Control::PAUSE}), nothing new starts, and the
+    # outage goes into the run's state file, the record that survives Redis.
+    # Redis is then asked again after 1, 1, 2, 3, 5, 8 … seconds, the
+    # Fibonacci numbers scaled by `redis_backoff.initial` and capped at
+    # `redis_backoff.cap` ({MAX_GAP} by default), so a short outage is noticed
+    # quickly, a long one is not hammered, and a Redis that comes back is
+    # never left unnoticed for longer than the cap. With full jitter (the
+    # default) each gap is drawn uniformly below that, so engines sharing one
+    # Redis do not all ask in the same second. When it answers, the agents
+    # carry on, their clocks having stood still meanwhile. When it has not answered for
+    # `redis_outage_hours` (an hour by default), the run shuts down the way
+    # `q` does, and says why.
+    #
+    # Run on the dispatcher's tick, ahead of everything that needs Redis.
+    class Outage
+      # Seconds Redis may be unreachable before the agents are paused.
+      GRACE = 10
+
+      # Seconds between pings while Redis answers.
+      PING_EVERY = 1
+
+      # The longest gap between two asks while paused, in seconds. Without
+      # it the Fibonacci gaps pass 16 minutes within fifteen asks, and a Redis
+      # back after 17 minutes would leave every agent paused for 10 more.
+      MAX_GAP = 30
+
+      # How many past outages the state file keeps.
+      HISTORY = 20
+
+      # One outage that ended in Redis answering: wall-clock bounds, and the
+      # length by the monotonic clock, which a changed system time cannot
+      # skew. {OutageReport} turns it into `run.outage` (plan 022).
+      Window = Data.define(:started_at, :ended_at, :seconds)
+
+      # @param store [OpenLanes::Live::Store, OpenLanes::Live::MemoryStore]
+      # @param limit [Numeric] seconds paused before the run gives up
+      # @param clock [#call] monotonic seconds
+      # @param wall [#call] wall time, for the state file
+      # @param initial [Numeric] seconds before the first ask; the Fibonacci gaps are its multiples
+      # @param cap [Numeric] the longest gap between two asks
+      # @param jitter [Boolean] full jitter: each gap is drawn from [0, gap)
+      # @param random [#rand] the jitter's source; a spec hands in a fixed one
+      def initialize(store:, limit:, clock: -> { UI.monotonic }, wall: -> { Time.now },
+        initial: 1, cap: MAX_GAP, jitter: true, random: Random.new)
+        @store = store
+        @limit = limit
+        @initial = initial
+        @cap = cap
+        @jitter = jitter
+        @random = random
+        @clock = clock
+        @wall = wall
+        @state = :up
+        @down_since = nil
+        @last_ping = nil
+        @resumed = []
+      end
+
+      # @return [Symbol] :up, :paused or :gave_up
+      attr_reader :state
+
+      # One tick.
+      #
+      # @param dispatcher [OpenLanes::Engine::Dispatcher] for its state file
+      # @return [void]
+      def call(dispatcher)
+        @state_file = dispatcher.state_file
+        now = @clock.call
+        case @state
+        when :up then watch(now)
+        when :paused then probe(now)
+        end
+      end
+
+      # @return [Boolean] whether agents are paused for this outage
+      def paused? = @state == :paused
+
+      # @return [Proc] {#call}, for `Dispatcher#before_tick`
+      def to_proc = method(:call).to_proc
+
+      # Something to run once Redis answers again, after the agents resume.
+      #
+      # @yieldparam window [Window]
+      # @return [void]
+      def on_resume(&hook) = @resumed << hook
+
+      # @return [Window, nil] the last outage that ended in Redis answering
+      attr_reader :window
+
+      private
+
+      # @return [void]
+      def watch(now)
+        return if @last_ping && now - @last_ping < PING_EVERY
+
+        @last_ping = now
+        if answers?
+          @down_since = nil
+        else
+          @down_since ||= now
+          pause(now) if now - @down_since >= GRACE
+        end
+      end
+
+      # @return [void]
+      def pause(now)
+        @state = :paused
+        @delays = fibonacci
+        @attempts = 0
+        @next_probe = next_probe(now)
+        @record = { "since" => (@wall.call - (now - @down_since)).iso8601, "paused_at" => @wall.call.iso8601,
+                    "limit_seconds" => @limit.round, "attempts" => 0 }
+        UI.log("Redis has not answered for #{GRACE}s: pausing every agent, starting nothing new, " \
+               "and giving up after #{hours(@limit)}",
+          level: :warn)
+        Execution::Control.pause!
+        save
+      end
+
+      # @return [void]
+      def probe(now)
+        return if now < @next_probe
+
+        @attempts += 1
+        @record["attempts"] = @attempts
+        @record["last_attempt_at"] = @wall.call.iso8601
+        if answers?
+          resume(now)
+        elsif now - @down_since >= @limit
+          give_up(now)
+        else
+          @next_probe = next_probe(now)
+          save
+        end
+      end
+
+      # @return [void]
+      def resume(now)
+        @state = :up
+        @record["resumed_at"] = @wall.call.iso8601
+        @record["seconds"] = (now - @down_since).round
+        UI.log("Redis answers again after #{(now - @down_since).round}s and #{@attempts} attempts: resuming every agent")
+        Execution::Control.resume!
+        @window = Window.new(started_at: @wall.call - (now - @down_since),
+          ended_at: @wall.call,
+          seconds: now - @down_since)
+        @down_since = nil
+        save(over: true)
+        @resumed.each { |hook| hook.call(@window) }
+      end
+
+      # @return [void]
+      def give_up(now)
+        @state = :gave_up
+        @record["gave_up_at"] = @wall.call.iso8601
+        @record["seconds"] = (now - @down_since).round
+        UI.log("Redis has not answered for #{hours(@limit)}: shutting the run down", level: :error)
+        Execution::Control.quit!
+        save(over: true)
+      end
+
+      # The next Fibonacci gap, scaled, capped, jittered, and never past the
+      # limit: the last ask is the one at the limit.
+      #
+      # @return [Float]
+      def next_probe(now) = [now + gap, @down_since + @limit].min
+
+      # @return [Float] seconds to the next ask
+      def gap
+        gap = [@delays.next * @initial, @cap].min.to_f
+        @jitter ? gap * @random.rand : gap
+      end
+
+      # @return [Boolean]
+      def answers?
+        @store.ping == true
+      rescue Unavailable, SystemCallError
+        false
+      end
+
+      # 1, 1, 2, 3, 5, 8, …
+      #
+      # @return [Enumerator]
+      def fibonacci
+        Enumerator.new do |out|
+          a, b = 1, 1
+          loop do
+            out << a
+            a, b = b, a + b
+          end
+        end
+      end
+
+      # The current outage, or, once it is over, into the history. Saved at
+      # once: this is the record a person reads to learn why agents stood
+      # still, and the run may not live to save it later.
+      #
+      # @param over [Boolean]
+      # @return [void]
+      def save(over: false)
+        return unless @state_file
+
+        @state_file.outage = over ? nil : @record
+        @state_file.outages = (@state_file.outages + [@record]).last(HISTORY) if over
+        @state_file.save
+      rescue SystemCallError => e
+        UI.log("outage not recorded: #{e.message}", level: :warn)
+      end
+
+      # @return [String]
+      def hours(seconds)
+        hours = seconds / 3600.0
+        hours == hours.round ? "#{hours.round} hour#{"s" unless hours.round == 1}" : "#{hours.round(2)} hours"
+      end
+    end
+  end
+end