samagotchi 0.102.0 → 0.104.0

data/lib/samagotchi/terminal_ui/local_link.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require_relative "attach_link"
+require_relative "../session"
+require_relative "../bridge_client"
+require_relative "../worker_sidecar"
+require_relative "../delivery"
+require_relative "../session_inbox"
+require_relative "../image_store"
+require_relative "../session_metrics"
+require_relative "../children_status"
+require_relative "../session_manager"
+require_relative "../installed_versions"
+require_relative "../model_profile"
+require_relative "../parent_report"
+require_relative "../chi_invocation"
+
+module Samagotchi
+  class TerminalUI
+    # The AttachLink of a session on this machine: its worker's Bridge
+    # (found through the sidecar in the session folder) and the session's
+    # files under +state_dir+.
+    class LocalLink
+      include AttachLink
+
+      # How long /exit --delete and /archive wait for the worker to let go
+      # (AttachedLoop's defaults too).
+      DELETE_WAIT = 10
+
+      attr_reader :client
+
+      # @param client [BridgeClient] the live worker's
+      # @param state_dir [String] the state dir the worker was found in
+      # @param delete_session [#call] session id -> deletes it
+      # @param archive_session [#call] session id -> archives it
+      #   (SessionManager.archive_session's result)
+      # @param installed_version [#call] -> the newest chi installed
+      # @param children_counts [#call] session id -> ChildrenStatus::Counts
+      def initialize(client:, state_dir: Session.default_state_dir,
+                     delete_session: ->(id) { SessionManager.delete_session(id, stop: true, wait: DELETE_WAIT) },
+                     archive_session: ->(id) { SessionManager.archive_session(id, wait: DELETE_WAIT) },
+                     installed_version: -> { InstalledVersions.new.newest },
+                     children_counts: ->(id) { ChildrenStatus.counts(id, state_dir: state_dir) })
+        @client = client
+        @state_dir = state_dir
+        @delete_session = delete_session
+        @archive_session = archive_session
+        @installed_version = installed_version
+        @children_counts = children_counts
+      end
+
+      # The sidecar's worker, nil while none runs; gone once the session was
+      # stopped (no worker is coming): its stop marker in +state_dir+.
+      def live_worker
+        id = @client.session_id
+        return [:gone, "stopped"] if Session.stopped_marker?(id, state_dir: @state_dir)
+
+        [BridgeClient.discover(id, session_dir: session_dir, host: @client.host), nil]
+      end
+
+      # A new worker's Bridge, on the port its event names.
+      def switched(event)
+        @client = BridgeClient.new(session_id: @client.session_id, port: AttachLink.event_port(event), host: @client.host)
+      end
+
+      # No sidecar to read: true, its Bridge answers (a P0-P2 worker refuses
+      # a queue).
+      def queues? = feature?(Delivery::QUEUE_FEATURE)
+
+      # No sidecar to read: true, its Bridge answers (an older one's 404
+      # not_found says so).
+      def unqueues? = feature?(SessionInbox::UNQUEUE_FEATURE)
+
+      # The worker's stats; a worker from before the stats route answers
+      # only /state.
+      def stats_metrics
+        @client.get_json("stats")&.dig("metrics") || @client.get_json("state")&.dig("session_state_snapshot", "metrics")
+      end
+
+      # Stored in the session's images/ here (the worker's state dir).
+      def ingest_images(paths)
+        paths.map do |path|
+          ref = ImageStore.ingest(session_dir, path: path)
+          { file: ref[:file], name: ref[:name] }
+        end
+      end
+
+      # SessionMetrics, in this machine's state dir: none for a prompt with
+      # no turn id (an older session) or a session with no analytics.
+      def saved_tool_records(turn_id)
+        SessionMetrics.saved_tool_records(session_dir, turn_id)
+      rescue Session::InvalidId
+        []
+      end
+
+      def saved_turn_record(turn_id)
+        SessionMetrics.saved_turn_record(session_dir, turn_id)
+      rescue Session::InvalidId
+        nil
+      end
+
+      def children_counts = @children_counts.call(@client.session_id)
+
+      def delete_session = @delete_session.call(@client.session_id)
+
+      def archive_session = @archive_session.call(@client.session_id)
+
+      def installed_version = @installed_version.call
+
+      def default_model_name = ModelProfile.default_model_name
+
+      def parent_strategy = ParentReport.strategy_for(@client.session_id, state_dir: @state_dir)
+
+      def completions = Completions::ALL
+
+      def reconnect_delays = BridgeClient::EventStream::REDISCOVER_DELAYS
+
+      def via = ChiInvocation::HERE
+
+      # The closing line says it all (unreachable, unknown_session).
+      def failure = nil
+
+      def close = nil
+
+      private
+
+      def session_dir = Session.session_dir(@client.session_id, state_dir: @state_dir)
+
+      def feature?(name)
+        sidecar = WorkerSidecar.read(session_dir)
+        sidecar.nil? || sidecar.features.include?(name)
+      end
+    end
+  end
+end

data/lib/samagotchi/terminal_ui/remote_connector.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "socket"
+require_relative "../session_access"
+require_relative "../session_access/remote"
+require_relative "../session_access/http_transport"
+require_relative "../owner_lock"
+require_relative "../launch_mode"
+require_relative "attach_launcher"
+require_relative "remote_link"
+
+module Samagotchi
+  class TerminalUI
+    # AttachLauncher.run's connector for a session on another machine: POST
+    # …/wake through that machine's chi web (the live worker, else one
+    # started there) and the RemoteLink the attached loop runs on. Its
+    # failures are AttachLauncher::Error, in the words chi web sent (the
+    # exceptions' own: one condition, one message).
+    class RemoteConnector
+      # @param remote [SessionAccess::Remote]
+      # @param session_id [String] resolved there already
+      # @param hostname [String] this machine's, for #client_id
+      # @param err [IO] where the wait for a leaving worker is said
+      def initialize(remote, session_id:, hostname: Socket.gethostname, err: $stderr)
+        @remote = remote
+        @session_id = session_id
+        @hostname = hostname
+        @err = err
+      end
+
+      # A worker there still leaving (its recap on leave) is said here
+      # (LaunchMode::RECAP_WAIT, as a local attach says it) and waited for
+      # by the wake, whose read then allows for that wait too.
+      # @return [RemoteLink]
+      # @raise [AttachLauncher::Error]
+      def connect
+        woke = @remote.attach_wake(@session_id, wait: AttachLauncher::BRIDGE_WAIT, read_timeout: wake_read_timeout)
+        RemoteLink.new(remote: @remote, session_id: @session_id, woke: woke)
+      rescue SessionManager::OwnedByTUI, SessionAccess::Refused, SessionAccess::RemoteFailed, ArgumentError => e
+        raise AttachLauncher::Error, e.message
+      end
+
+      # Seconds past the transport's read timeout a wake may take beyond its
+      # leave and Bridge waits there.
+      WAKE_READ_SLACK = 15.0
+
+      # nil (the transport's) unless the worker there is leaving: then its
+      # leave's budget left, OwnerLock::LEAVE_MARGIN, the Bridge wait and
+      # some slack, at least the transport's.
+      def wake_read_timeout
+        state = @remote.attach_worker(@session_id)
+        return nil unless state.leaving
+
+        @err.puts(LaunchMode::RECAP_WAIT)
+        budget = state.leave_seconds.to_f + OwnerLock::LEAVE_MARGIN + AttachLauncher::BRIDGE_WAIT + WAKE_READ_SLACK
+        [budget, SessionAccess::HTTPTransport::READ_TIMEOUT].max
+      end
+
+      # tui:PID@HOST (P4-D6): an interactive, human client for the UIs
+      # there (ClientId), with the machine it types on (its short name).
+      def self.client_id(hostname = Socket.gethostname) = "tui:#{Process.pid}@#{hostname.to_s.split(".").first}"
+
+      def client_id = RemoteConnector.client_id(@hostname)
+    end
+  end
+end

data/lib/samagotchi/terminal_ui/remote_link.rb

@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require "fileutils"
+require_relative "attach_link"
+require_relative "../session_access"
+require_relative "../session_access/remote"
+require_relative "../session_access/web_bridge_client"
+require_relative "../children_status"
+require_relative "../image_store"
+require_relative "../delivery"
+require_relative "../session_inbox"
+require_relative "../log"
+
+module Samagotchi
+  class TerminalUI
+    # The AttachLink of a session on another machine (chi --remote NAME
+    # --attach ID), reached through that machine's chi web: one
+    # SessionAccess::WebBridgeClient for the whole attach (a new worker
+    # changes its worker_key, never the object), the worker's features from
+    # the wake and GET …/worker, and that machine's default model, newest
+    # chi and parent strategy. What stays this machine's: the prompt
+    # history, the run flags' baselines, the @path files (ingested here,
+    # uploaded there).
+    class RemoteLink
+      include AttachLink
+
+      # The registry ids (SessionCommands' local: true, :attached) the
+      # attached loop runs remotely as it does here: its own leaving, the
+      # deliveries, the dequeue route, and the exit, archive, stats and
+      # recap through chi web's routes.
+      HANDLED_COMMANDS = %i[detach cut queue unqueue exit archive stats recap].freeze
+
+      # About a minute of reconnects (P4-D5); a Bridge here keeps
+      # EventStream::REDISCOVER_DELAYS.
+      RECONNECT_DELAYS = [0.25, 0.5, 1.0, 2.0, 2.0, 3.0, 5.0, 5.0, 5.0, 5.0, 10.0, 10.0, 10.0].freeze
+      # How long chi web must answer "no live worker" (not stopped) before
+      # the reconnect wakes one (P4-D9): a restart's new worker comes
+      # sooner, so it isn't raced.
+      WAKE_AFTER = 3.0
+      # The wake's wait for a woken worker's Bridge (AttachLauncher's).
+      WAKE_WAIT = 10.0
+
+      attr_reader :client
+
+      # @param remote [SessionAccess::Remote]
+      # @param session_id [String]
+      # @param woke [SessionAccess::WakeReply] the launch's wake
+      # @param clock [#call] monotonic seconds
+      def initialize(remote:, session_id:, woke:, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+        @remote = remote
+        @woke = woke
+        @features = Array(woke.worker&.fetch("features", nil))
+        @client = remote.attach_client(session_id, worker: woke.worker || {})
+        @clock = clock
+        @no_worker_since = nil
+        @woken_once = false
+        @failure = nil
+        @default_model = woke.default_model
+        @last_records = nil
+      end
+
+      # GET …/worker, with the reason when there is none: [client, nil]
+      # (its key updated in place: EventStream sees a move when it
+      # changed), [nil, reason] to retry, [:gone, reason] to stop. A worker
+      # chi web says isn't live and isn't stopped is woken once per
+      # outage, after WAKE_AFTER (P4-D9).
+      def live_worker
+        state = @remote.attach_worker(session_id)
+        return [:gone, "stopped"] if state.stopped
+        return follow(state.worker) if state.live
+
+        wake_once || [nil, "worker_gone"]
+      rescue SessionAccess::AuthRefused, SessionAccess::RemoteTooOld => e
+        @failure = e.message
+        [:gone, e.is_a?(SessionAccess::AuthRefused) ? "auth_failed" : "too_old"]
+      rescue SessionAccess::RemoteFailed => e
+        @failure = e.message
+        [nil, "unreachable"]
+      rescue ArgumentError
+        [:gone, "unknown_session"]
+      rescue StandardError => e
+        Log.debug(:attached, "rediscover_failed", error: e.class.name, message: e.message)
+        @failure = e.message
+        [nil, "unreachable"]
+      end
+
+      # Still the same client (its key moved already: the stream's epoch
+      # watch, or #live_worker); the new worker's features read again.
+      def switched(event)
+        epoch = event[:epoch] || event["epoch"]
+        @client.worker_key = epoch if epoch
+        refresh_features
+        @client
+      end
+
+      def queues? = @features.include?(Delivery::QUEUE_FEATURE)
+
+      def unqueues? = @features.include?(SessionInbox::UNQUEUE_FEATURE)
+
+      # Each @path image ingested here (converted and downscaled as chi send
+      # does), then uploaded into the session there.
+      def ingest_images(paths)
+        refs = paths.map { |path| ImageStore.ingest(image_dir, path: path) }
+        @remote.upload_images(session_id, refs, from: image_dir)
+      rescue SessionAccess::Refused => e
+        raise ImageStore::Error, e.message
+      rescue SessionAccess::RemoteFailed => e
+        raise SessionAccess::WebBridgeClient::Unreachable, e.message
+      end
+
+      # A turn's saved records there (GET …/turns/:turn_id/records): [] and
+      # nil when they can't be read, as for an older session. A join asks
+      # for the tool records, then the turn record, of the same turn: one
+      # read gives both, so the second takes what the first read.
+      def saved_tool_records(turn_id)
+        @last_records = [turn_id, turn_records(turn_id)]
+        Array(@last_records[1]&.fetch("tool_records", nil))
+      end
+
+      def saved_turn_record(turn_id)
+        read = @last_records&.first == turn_id ? @last_records[1] : turn_records(turn_id)
+        @last_records = nil
+        record = read&.fetch("turn_record", nil)
+        record.is_a?(Hash) ? record : nil
+      end
+
+      # The delegates' counts there (GET …/children/counts); none when they
+      # can't be read (the status row keeps no children segment).
+      def children_counts
+        json = @client.children_counts
+        ChildrenStatus::Counts.new(running: json["running"].to_i, waiting: json["waiting"].to_i,
+                                   unreported: json["unreported"].to_i)
+      rescue StandardError => e
+        Log.debug(:attached, "children_counts_failed", error: e.class.name, message: e.message)
+        ChildrenStatus::Counts.none
+      end
+
+      # GET …/stats (nil when the worker didn't give them).
+      # @raise [SessionAccess::WebBridgeClient::Unreachable] chi web couldn't
+      #   be asked, or has no worker for it: the loop says why
+      def stats_metrics = @client.stats
+
+      # /exit --delete, once the worker let go: DELETE …?force=true there
+      # (SessionManager.delete_session's refusals, raised from chi web's
+      # fields).
+      def delete_session
+        @remote.delete(session_id, force: true)
+      rescue SessionAccess::RemoteFailed => e
+        raise SessionAccess::WebBridgeClient::Unreachable, e.message
+      end
+
+      # /archive: POST …/archive there ({discarded: […]}, as
+      # SessionManager.archive_session's).
+      def archive_session
+        @remote.archive(session_id)
+      rescue SessionAccess::RemoteFailed => e
+        raise SessionAccess::WebBridgeClient::Unreachable, e.message
+      end
+
+      # That machine's newest chi (the wake's).
+      def installed_version = @woke.installed
+
+      # That machine's config's default model: the wake's, then the newest
+      # worker read's (a restart may come with another config).
+      def default_model_name = @default_model
+
+      # Read again now (a parent's report at the end of a run), the wake's
+      # when chi web can't be asked.
+      def parent_strategy
+        @remote.attach_worker(session_id).strategy
+      rescue StandardError
+        @woke.strategy
+      end
+
+      # / commands and this machine's @path files (images are ingested here);
+      # no memory names: they would be this machine's (P4-D11).
+      def completions = Completions.new(paths: true, memories: false)
+
+      def reconnect_delays = RECONNECT_DELAYS
+
+      def via = @remote.invocation
+
+      # Why chi web couldn't be asked, the last time it couldn't (its
+      # message: the remote's label and the cause), for the closing line.
+      attr_reader :failure
+
+      # The images' scratch dir goes with the attach.
+      def close
+        FileUtils.rm_rf(@image_dir) if @image_dir
+        @image_dir = nil
+      end
+
+      private
+
+      def session_id = @client.session_id
+
+      def image_dir = @image_dir ||= Dir.mktmpdir("chi-attach-images")
+
+      def follow(worker)
+        @no_worker_since = nil
+        @woken_once = false
+        @features = Array(worker&.fetch("features", nil))
+        key = worker && (worker["epoch"] || worker["started_at"])
+        @client.worker_key = key if key
+        [@client, nil]
+      end
+
+      # nil while chi web hasn't said "no worker" for WAKE_AFTER yet, or
+      # once this outage's wake was made (or failed).
+      def wake_once
+        now = @clock.call
+        @no_worker_since ||= now
+        return nil if @woken_once || now - @no_worker_since < WAKE_AFTER
+
+        @woken_once = true
+        woke = @remote.attach_wake(session_id, wait: WAKE_WAIT)
+        follow(woke.worker)
+      rescue SessionAccess::Refused, SessionManager::OwnedByTUI => e
+        @failure = e.message
+        nil
+      end
+
+      # nil without a turn id (an older session's prompt) or when chi web
+      # can't give them (a join reads them within JOIN_READ_TIMEOUT).
+      def turn_records(turn_id)
+        return nil if turn_id.to_s.empty?
+
+        json = @client.turn_records(turn_id)
+        json.is_a?(Hash) ? json : nil
+      rescue StandardError => e
+        Log.debug(:attached, "turn_records_failed", error: e.class.name, message: e.message)
+        nil
+      end
+
+      def refresh_features
+        state = @remote.attach_worker(session_id)
+        @features = Array(state.worker&.fetch("features", nil)) if state.live
+        @default_model = state.default_model if state.default_model
+      rescue StandardError => e
+        Log.debug(:attached, "worker_features_failed", error: e.class.name, message: e.message)
+      end
+    end
+  end
+end

data/lib/samagotchi/terminal_ui/version_lines.rb

@@ -2,6 +2,7 @@
 
 require_relative "../installed_versions"
 require_relative "../version"
+require_relative "../chi_invocation"
 
 module Samagotchi
   class TerminalUI
@@ -15,9 +16,11 @@ module Samagotchi
       # @param worker [String, nil] the worker's chi_version (nil: a worker
       #   from before it said so; nothing is said then)
       # @param installed [String, nil] InstalledVersions#newest
+      # @param via [ChiInvocation] how the commands in it are spelled
       # @return [String, nil] one "chi> ..." line, or nil when all is current
-      def at_attach(worker:, installed:, session_id:, terminal: VERSION)
+      def at_attach(worker:, installed:, session_id:, terminal: VERSION, via: ChiInvocation::HERE)
         return nil if worker.to_s.empty?
+        return across_machines(worker: worker, installed: installed, session_id: session_id, terminal: terminal, via: via) if via.remote?
 
         newest = [installed, terminal].select { |v| Gem::Version.correct?(v.to_s) && !v.to_s.empty? }
                                       .max_by { |v| Gem::Version.new(v) }
@@ -26,8 +29,8 @@ module Samagotchi
         return nil unless worker_old || terminal_old
 
         id = session_id.to_s[0, 8]
-        restart = "chi sessions restart #{id}"
-        attach = "/detach, then chi --attach #{id}"
+        restart = via.command("sessions restart #{id}")
+        attach = "/detach, then #{via.attach(id)}"
         if worker_old && terminal_old
           "chi> chi #{newest} is installed; this session's worker runs #{worker} and this terminal #{terminal}. " \
             "#{restart} moves the worker; #{attach} for the terminal."
@@ -39,6 +42,28 @@ module Samagotchi
         end
       end
 
+      # A remote attach (P4-D10): the worker and +installed+ are that
+      # machine's, the terminal this one's, so each is advised where it
+      # runs: the worker older than that machine's newest moves with chi
+      # --remote NAME sessions restart; this terminal older than the newer
+      # of the two updates here.
+      # @return [String, nil]
+      def across_machines(worker:, installed:, session_id:, terminal:, via:)
+        there = [installed, worker].select { |v| Gem::Version.correct?(v.to_s) && !v.to_s.empty? }
+                                   .max_by { |v| Gem::Version.new(v) }
+        worker_old = InstalledVersions.newer?(installed, worker)
+        terminal_old = InstalledVersions.newer?(there, terminal)
+        return nil unless worker_old || terminal_old
+
+        lines = []
+        if worker_old
+          lines << "the session's worker runs chi #{worker}; #{via.remote} has #{installed} installed: " \
+                   "#{via.command("sessions restart #{session_id.to_s[0, 8]}")}"
+        end
+        lines << "#{worker_old ? "This" : "this"} terminal runs chi #{terminal} (#{via.remote} has #{there}): chi update here" if terminal_old
+        "chi> #{lines.join(". ")}"
+      end
+
       # After the stream moved to a new worker (a restart).
       def restarted(worker)
         worker.to_s.empty? ? "chi> the session's worker restarted" : "chi> the session's worker restarted on chi #{worker}"

data/lib/samagotchi/terminal_ui.rb

@@ -867,7 +867,10 @@ module Samagotchi
       route = ComposerRoute.call(input, running: false, registry: command_registry)
       return @surface.commit(STEER_CHOICE_USAGE) if route.kind == :usage
 
-      input = route.text if route.choice
+      if route.choice
+        input = route.text
+        @surface.commit(STEER_CHOICE_IDLE)
+      end
 
       # /model, /models, !rollback, !cmd, /continue (shared with workers);
       # an anytime command's cards print as it shows them (btw's
@@ -890,8 +893,8 @@ module Samagotchi
           show_command_result(command)
           refresh_status_row(clear_unknown_context: true) if command.changed.include?(:model)
           refresh_status_row if command.changed.include?(:title)
-          # /recap on|off|default doesn't save itself (a worker saves a changed command).
-          save_session(session) if command.changed.include?(:recap)
+          # /recap and /title don't save themselves (a worker saves a changed command).
+          save_session(session) if command.changed.intersect?(%i[recap title])
           persist_recent_history(input) if command.shell
           return
         end

data/lib/samagotchi/testing/waiting.rb

@@ -5,12 +5,71 @@ require_relative "../process_group"
 module Samagotchi
   module Testing
     # Waiting with a deadline, for a test that waits on a thread or a
-    # process (Helpers has them).
+    # process (Helpers has them). Every deadline is the given +timeout+ ×
+    # Waiting.time_scale (SPEC_TIME_SCALE, CI sets 2): a slow runner gets
+    # more headroom, a fast one never waits longer than its condition takes.
     module Waiting
+      # A wait whose deadline passed: its message names what it waited for
+      # (Waiting.timed_out_message).
+      class TimedOut < StandardError; end
+
+      # The most of a +seen:+ value's inspect a message shows.
+      SEEN_LIMIT = 300
+
+      class << self
+        # SPEC_TIME_SCALE as a Float, at least 1 (unset or empty: 1). A value
+        # that isn't a number gives 1 and one warning. Read per call.
+        def time_scale
+          raw = ENV.fetch("SPEC_TIME_SCALE", "").strip
+          return 1.0 if raw.empty?
+
+          [Float(raw), 1.0].max
+        rescue ArgumentError
+          unless @warned_scale
+            @warned_scale = true
+            warn("SPEC_TIME_SCALE=#{raw.inspect} is not a number: waiting with scale 1")
+          end
+          1.0
+        end
+
+        # +timeout+ s scaled: the deadline a helper waits for.
+        def scaled(timeout, scale = time_scale) = timeout * scale
+
+        # How long a wait took to give up: "5 s", or with a scale
+        # "10.0 s (5 s × SPEC_TIME_SCALE 2)".
+        def waited(timeout, scale = time_scale)
+          return "#{timeout} s" if scale == 1
+
+          "#{scaled(timeout, scale).round(3).to_f} s (#{timeout} s × SPEC_TIME_SCALE #{number(scale)})"
+        end
+
+        # The one failure text of wait_for, join! and pop!:
+        # "timed out after 5 s waiting for <what>", then "; last saw: <value>"
+        # when +seen+ (a callable) is given, its value's inspect cut to
+        # SEEN_LIMIT chars.
+        def timed_out_message(what, timeout, scale: time_scale, seen: nil)
+          message = "timed out after #{waited(timeout, scale)} waiting for #{what}"
+          seen ? "#{message}; last saw: #{seen_text(seen)}" : message
+        end
+
+        private
+
+        def number(value) = value == value.to_i ? value.to_i.to_s : value.to_s
+
+        def seen_text(seen)
+          text = seen.call.inspect
+          text.length > SEEN_LIMIT ? "#{text[0, SEEN_LIMIT]}…" : text
+        rescue StandardError => e
+          "(raised #{e.class}: #{e.message})"
+        end
+      end
+
       # The block's value once truthy, polled every +interval+ s; after
-      # +timeout+ s its last (falsy) value.
+      # +timeout+ s (scaled) its last (falsy) value. For a wait whose
+      # timeout is fine (a bounded best-effort wait); a wait that must hold
+      # is #wait_for.
       def wait_until(timeout: 5, interval: 0.01)
-        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + Waiting.scaled(timeout)
         until (value = yield)
           return yield if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
 
@@ -19,15 +78,39 @@ module Samagotchi
         value
       end
 
-      # #wait_until, raising "timed out after N s waiting for <what>" when
-      # the time runs out: for a wait no check follows.
-      def wait_until!(what, timeout: 5, interval: 0.01, &)
-        wait_until(timeout: timeout, interval: interval, &) || raise("timed out after #{timeout} s waiting for #{what}")
+      # Polls the block every +interval+ s until it returns truthy and
+      # returns that value. After +timeout+ s (scaled) it raises
+      # Waiting::TimedOut naming +what+ ("timed out after 5 s waiting for
+      # <what>"); +seen+ (a callable) adds what it last saw, since a falsy
+      # block value tells nothing.
+      def wait_for(what, timeout: 5, interval: 0.01, seen: nil, &)
+        wait_until(timeout: timeout, interval: interval, &) ||
+          raise(TimedOut, Waiting.timed_out_message(what, timeout, seen: seen))
+      end
+
+      # Thread#join with a deadline (scaled): a thread that never ends fails
+      # the test with "<what> to finish" named, not after a suite-wide guard.
+      # Raises what the thread raised, as #join does.
+      # @return [Thread]
+      def join!(thread, what, timeout: 5)
+        thread.join(Waiting.scaled(timeout)) || raise(TimedOut, Waiting.timed_out_message("#{what} to finish", timeout))
+      end
+
+      # Queue#pop with a deadline (scaled), for a queue that never carries
+      # nil: nothing pushed in time fails with +what+ named. A gate a stubbed
+      # lib thread waits on while the test works takes a longer +timeout+
+      # (30): it only has to end a hang, never cut a slow test short.
+      # @return the value popped.
+      def pop!(queue, what, timeout: 5)
+        value = queue.pop(timeout: Waiting.scaled(timeout))
+        raise(TimedOut, Waiting.timed_out_message(what, timeout)) if value.nil?
+
+        value
       end
 
       # Whether no process +pid+ is left (a killed grandchild is reaped a
-      # moment later: wait_until { process_gone?(pid) }). One that isn't
-      # ours to signal is still there (ProcessGroup.alive?).
+      # moment later: wait_for("pid gone") { process_gone?(pid) }). One that
+      # isn't ours to signal is still there (ProcessGroup.alive?).
       def process_gone?(pid) = !ProcessGroup.alive?(pid)
     end
   end