samagotchi 0.103.0 → 0.105.0

data/lib/samagotchi/terminal_ui/remote_connector.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "socket"
+require_relative "../session_access"
+require_relative "../session_access/remote"
+require_relative "../session_access/http_transport"
+require_relative "../owner_lock"
+require_relative "../launch_mode"
+require_relative "attach_launcher"
+require_relative "remote_link"
+
+module Samagotchi
+  class TerminalUI
+    # AttachLauncher.run's connector for a session on another machine: POST
+    # …/wake through that machine's chi web (the live worker, else one
+    # started there) and the RemoteLink the attached loop runs on. Its
+    # failures are AttachLauncher::Error, in the words chi web sent (the
+    # exceptions' own: one condition, one message).
+    class RemoteConnector
+      # @param remote [SessionAccess::Remote]
+      # @param session_id [String] resolved there already
+      # @param hostname [String] this machine's, for #client_id
+      # @param err [IO] where the wait for a leaving worker is said
+      def initialize(remote, session_id:, hostname: Socket.gethostname, err: $stderr)
+        @remote = remote
+        @session_id = session_id
+        @hostname = hostname
+        @err = err
+      end
+
+      # A worker there still leaving (its recap on leave) is said here
+      # (LaunchMode::RECAP_WAIT, as a local attach says it) and waited for
+      # by the wake, whose read then allows for that wait too.
+      # @return [RemoteLink]
+      # @raise [AttachLauncher::Error]
+      def connect
+        woke = @remote.attach_wake(@session_id, wait: AttachLauncher::BRIDGE_WAIT, read_timeout: wake_read_timeout)
+        RemoteLink.new(remote: @remote, session_id: @session_id, woke: woke)
+      rescue SessionManager::OwnedByTUI, SessionAccess::Refused, SessionAccess::RemoteFailed, ArgumentError => e
+        raise AttachLauncher::Error, e.message
+      end
+
+      # Seconds past the transport's read timeout a wake may take beyond its
+      # leave and Bridge waits there.
+      WAKE_READ_SLACK = 15.0
+
+      # nil (the transport's) unless the worker there is leaving: then its
+      # leave's budget left, OwnerLock::LEAVE_MARGIN, the Bridge wait and
+      # some slack, at least the transport's.
+      def wake_read_timeout
+        state = @remote.attach_worker(@session_id)
+        return nil unless state.leaving
+
+        @err.puts(LaunchMode::RECAP_WAIT)
+        budget = state.leave_seconds.to_f + OwnerLock::LEAVE_MARGIN + AttachLauncher::BRIDGE_WAIT + WAKE_READ_SLACK
+        [budget, SessionAccess::HTTPTransport::READ_TIMEOUT].max
+      end
+
+      # tui:PID@HOST (P4-D6): an interactive, human client for the UIs
+      # there (ClientId), with the machine it types on (its short name).
+      def self.client_id(hostname = Socket.gethostname) = "tui:#{Process.pid}@#{hostname.to_s.split(".").first}"
+
+      def client_id = RemoteConnector.client_id(@hostname)
+    end
+  end
+end

data/lib/samagotchi/terminal_ui/remote_link.rb

@@ -0,0 +1,245 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require "fileutils"
+require_relative "attach_link"
+require_relative "../session_access"
+require_relative "../session_access/remote"
+require_relative "../session_access/web_bridge_client"
+require_relative "../children_status"
+require_relative "../image_store"
+require_relative "../delivery"
+require_relative "../session_inbox"
+require_relative "../log"
+
+module Samagotchi
+  class TerminalUI
+    # The AttachLink of a session on another machine (chi --remote NAME
+    # --attach ID), reached through that machine's chi web: one
+    # SessionAccess::WebBridgeClient for the whole attach (a new worker
+    # changes its worker_key, never the object), the worker's features from
+    # the wake and GET …/worker, and that machine's default model, newest
+    # chi and parent strategy. What stays this machine's: the prompt
+    # history, the run flags' baselines, the @path files (ingested here,
+    # uploaded there).
+    class RemoteLink
+      include AttachLink
+
+      # The registry ids (SessionCommands' local: true, :attached) the
+      # attached loop runs remotely as it does here: its own leaving, the
+      # deliveries, the dequeue route, and the exit, archive, stats and
+      # recap through chi web's routes.
+      HANDLED_COMMANDS = %i[detach cut queue unqueue exit archive stats recap].freeze
+
+      # About a minute of reconnects (P4-D5); a Bridge here keeps
+      # EventStream::REDISCOVER_DELAYS.
+      RECONNECT_DELAYS = [0.25, 0.5, 1.0, 2.0, 2.0, 3.0, 5.0, 5.0, 5.0, 5.0, 10.0, 10.0, 10.0].freeze
+      # How long chi web must answer "no live worker" (not stopped) before
+      # the reconnect wakes one (P4-D9): a restart's new worker comes
+      # sooner, so it isn't raced.
+      WAKE_AFTER = 3.0
+      # The wake's wait for a woken worker's Bridge (AttachLauncher's).
+      WAKE_WAIT = 10.0
+
+      attr_reader :client
+
+      # @param remote [SessionAccess::Remote]
+      # @param session_id [String]
+      # @param woke [SessionAccess::WakeReply] the launch's wake
+      # @param clock [#call] monotonic seconds
+      def initialize(remote:, session_id:, woke:, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+        @remote = remote
+        @woke = woke
+        @features = Array(woke.worker&.fetch("features", nil))
+        @client = remote.attach_client(session_id, worker: woke.worker || {})
+        @clock = clock
+        @no_worker_since = nil
+        @woken_once = false
+        @failure = nil
+        @default_model = woke.default_model
+        @last_records = nil
+      end
+
+      # GET …/worker, with the reason when there is none: [client, nil]
+      # (its key updated in place: EventStream sees a move when it
+      # changed), [nil, reason] to retry, [:gone, reason] to stop. A worker
+      # chi web says isn't live and isn't stopped is woken once per
+      # outage, after WAKE_AFTER (P4-D9).
+      def live_worker
+        state = @remote.attach_worker(session_id)
+        return [:gone, "stopped"] if state.stopped
+        return follow(state.worker) if state.live
+
+        wake_once || [nil, "worker_gone"]
+      rescue SessionAccess::AuthRefused, SessionAccess::RemoteTooOld => e
+        @failure = e.message
+        [:gone, e.is_a?(SessionAccess::AuthRefused) ? "auth_failed" : "too_old"]
+      rescue SessionAccess::RemoteFailed => e
+        @failure = e.message
+        [nil, "unreachable"]
+      rescue ArgumentError
+        [:gone, "unknown_session"]
+      rescue StandardError => e
+        Log.debug(:attached, "rediscover_failed", error: e.class.name, message: e.message)
+        @failure = e.message
+        [nil, "unreachable"]
+      end
+
+      # Still the same client (its key moved already: the stream's epoch
+      # watch, or #live_worker); the new worker's features read again.
+      def switched(event)
+        epoch = event[:epoch] || event["epoch"]
+        @client.worker_key = epoch if epoch
+        refresh_features
+        @client
+      end
+
+      def queues? = @features.include?(Delivery::QUEUE_FEATURE)
+
+      def unqueues? = @features.include?(SessionInbox::UNQUEUE_FEATURE)
+
+      # Each @path image ingested here (converted and downscaled as chi send
+      # does), then uploaded into the session there.
+      def ingest_images(paths)
+        refs = paths.map { |path| ImageStore.ingest(image_dir, path: path) }
+        @remote.upload_images(session_id, refs, from: image_dir)
+      rescue SessionAccess::Refused => e
+        raise ImageStore::Error, e.message
+      rescue SessionAccess::RemoteFailed => e
+        raise SessionAccess::WebBridgeClient::Unreachable, e.message
+      end
+
+      # A turn's saved records there (GET …/turns/:turn_id/records): [] and
+      # nil when they can't be read, as for an older session. A join asks
+      # for the tool records, then the turn record, of the same turn: one
+      # read gives both, so the second takes what the first read.
+      def saved_tool_records(turn_id)
+        @last_records = [turn_id, turn_records(turn_id)]
+        Array(@last_records[1]&.fetch("tool_records", nil))
+      end
+
+      def saved_turn_record(turn_id)
+        read = @last_records&.first == turn_id ? @last_records[1] : turn_records(turn_id)
+        @last_records = nil
+        record = read&.fetch("turn_record", nil)
+        record.is_a?(Hash) ? record : nil
+      end
+
+      # The delegates' counts there (GET …/children/counts); none when they
+      # can't be read (the status row keeps no children segment).
+      def children_counts
+        json = @client.children_counts
+        ChildrenStatus::Counts.new(running: json["running"].to_i, waiting: json["waiting"].to_i,
+                                   unreported: json["unreported"].to_i)
+      rescue StandardError => e
+        Log.debug(:attached, "children_counts_failed", error: e.class.name, message: e.message)
+        ChildrenStatus::Counts.none
+      end
+
+      # GET …/stats (nil when the worker didn't give them).
+      # @raise [SessionAccess::WebBridgeClient::Unreachable] chi web couldn't
+      #   be asked, or has no worker for it: the loop says why
+      def stats_metrics = @client.stats
+
+      # /exit --delete, once the worker let go: DELETE …?force=true there
+      # (SessionManager.delete_session's refusals, raised from chi web's
+      # fields).
+      def delete_session
+        @remote.delete(session_id, force: true)
+      rescue SessionAccess::RemoteFailed => e
+        raise SessionAccess::WebBridgeClient::Unreachable, e.message
+      end
+
+      # /archive: POST …/archive there ({discarded: […]}, as
+      # SessionManager.archive_session's).
+      def archive_session
+        @remote.archive(session_id)
+      rescue SessionAccess::RemoteFailed => e
+        raise SessionAccess::WebBridgeClient::Unreachable, e.message
+      end
+
+      # That machine's newest chi (the wake's).
+      def installed_version = @woke.installed
+
+      # That machine's config's default model: the wake's, then the newest
+      # worker read's (a restart may come with another config).
+      def default_model_name = @default_model
+
+      # Read again now (a parent's report at the end of a run), the wake's
+      # when chi web can't be asked.
+      def parent_strategy
+        @remote.attach_worker(session_id).strategy
+      rescue StandardError
+        @woke.strategy
+      end
+
+      # / commands and this machine's @path files (images are ingested here);
+      # no memory names: they would be this machine's (P4-D11).
+      def completions = Completions.new(paths: true, memories: false)
+
+      def reconnect_delays = RECONNECT_DELAYS
+
+      def via = @remote.invocation
+
+      # Why chi web couldn't be asked, the last time it couldn't (its
+      # message: the remote's label and the cause), for the closing line.
+      attr_reader :failure
+
+      # The images' scratch dir goes with the attach.
+      def close
+        FileUtils.rm_rf(@image_dir) if @image_dir
+        @image_dir = nil
+      end
+
+      private
+
+      def session_id = @client.session_id
+
+      def image_dir = @image_dir ||= Dir.mktmpdir("chi-attach-images")
+
+      def follow(worker)
+        @no_worker_since = nil
+        @woken_once = false
+        @features = Array(worker&.fetch("features", nil))
+        key = worker && (worker["epoch"] || worker["started_at"])
+        @client.worker_key = key if key
+        [@client, nil]
+      end
+
+      # nil while chi web hasn't said "no worker" for WAKE_AFTER yet, or
+      # once this outage's wake was made (or failed).
+      def wake_once
+        now = @clock.call
+        @no_worker_since ||= now
+        return nil if @woken_once || now - @no_worker_since < WAKE_AFTER
+
+        @woken_once = true
+        woke = @remote.attach_wake(session_id, wait: WAKE_WAIT)
+        follow(woke.worker)
+      rescue SessionAccess::Refused, SessionManager::OwnedByTUI => e
+        @failure = e.message
+        nil
+      end
+
+      # nil without a turn id (an older session's prompt) or when chi web
+      # can't give them (a join reads them within JOIN_READ_TIMEOUT).
+      def turn_records(turn_id)
+        return nil if turn_id.to_s.empty?
+
+        json = @client.turn_records(turn_id)
+        json.is_a?(Hash) ? json : nil
+      rescue StandardError => e
+        Log.debug(:attached, "turn_records_failed", error: e.class.name, message: e.message)
+        nil
+      end
+
+      def refresh_features
+        state = @remote.attach_worker(session_id)
+        @features = Array(state.worker&.fetch("features", nil)) if state.live
+        @default_model = state.default_model if state.default_model
+      rescue StandardError => e
+        Log.debug(:attached, "worker_features_failed", error: e.class.name, message: e.message)
+      end
+    end
+  end
+end

data/lib/samagotchi/terminal_ui/version_lines.rb

@@ -20,6 +20,7 @@ module Samagotchi
       # @return [String, nil] one "chi> ..." line, or nil when all is current
       def at_attach(worker:, installed:, session_id:, terminal: VERSION, via: ChiInvocation::HERE)
         return nil if worker.to_s.empty?
+        return across_machines(worker: worker, installed: installed, session_id: session_id, terminal: terminal, via: via) if via.remote?
 
         newest = [installed, terminal].select { |v| Gem::Version.correct?(v.to_s) && !v.to_s.empty? }
                                       .max_by { |v| Gem::Version.new(v) }
@@ -41,6 +42,28 @@ module Samagotchi
         end
       end
 
+      # A remote attach (P4-D10): the worker and +installed+ are that
+      # machine's, the terminal this one's, so each is advised where it
+      # runs: the worker older than that machine's newest moves with chi
+      # --remote NAME sessions restart; this terminal older than the newer
+      # of the two updates here.
+      # @return [String, nil]
+      def across_machines(worker:, installed:, session_id:, terminal:, via:)
+        there = [installed, worker].select { |v| Gem::Version.correct?(v.to_s) && !v.to_s.empty? }
+                                   .max_by { |v| Gem::Version.new(v) }
+        worker_old = InstalledVersions.newer?(installed, worker)
+        terminal_old = InstalledVersions.newer?(there, terminal)
+        return nil unless worker_old || terminal_old
+
+        lines = []
+        if worker_old
+          lines << "the session's worker runs chi #{worker}; #{via.remote} has #{installed} installed: " \
+                   "#{via.command("sessions restart #{session_id.to_s[0, 8]}")}"
+        end
+        lines << "#{worker_old ? "This" : "this"} terminal runs chi #{terminal} (#{via.remote} has #{there}): chi update here" if terminal_old
+        "chi> #{lines.join(". ")}"
+      end
+
       # After the stream moved to a new worker (a restart).
       def restarted(worker)
         worker.to_s.empty? ? "chi> the session's worker restarted" : "chi> the session's worker restarted on chi #{worker}"

data/lib/samagotchi/terminal_ui.rb

@@ -867,7 +867,10 @@ module Samagotchi
       route = ComposerRoute.call(input, running: false, registry: command_registry)
       return @surface.commit(STEER_CHOICE_USAGE) if route.kind == :usage
 
-      input = route.text if route.choice
+      if route.choice
+        input = route.text
+        @surface.commit(STEER_CHOICE_IDLE)
+      end
 
       # /model, /models, !rollback, !cmd, /continue (shared with workers);
       # an anytime command's cards print as it shows them (btw's

data/lib/samagotchi/testing/waiting.rb

@@ -5,12 +5,71 @@ require_relative "../process_group"
 module Samagotchi
   module Testing
     # Waiting with a deadline, for a test that waits on a thread or a
-    # process (Helpers has them).
+    # process (Helpers has them). Every deadline is the given +timeout+ ×
+    # Waiting.time_scale (SPEC_TIME_SCALE, CI sets 2): a slow runner gets
+    # more headroom, a fast one never waits longer than its condition takes.
     module Waiting
+      # A wait whose deadline passed: its message names what it waited for
+      # (Waiting.timed_out_message).
+      class TimedOut < StandardError; end
+
+      # The most of a +seen:+ value's inspect a message shows.
+      SEEN_LIMIT = 300
+
+      class << self
+        # SPEC_TIME_SCALE as a Float, at least 1 (unset or empty: 1). A value
+        # that isn't a number gives 1 and one warning. Read per call.
+        def time_scale
+          raw = ENV.fetch("SPEC_TIME_SCALE", "").strip
+          return 1.0 if raw.empty?
+
+          [Float(raw), 1.0].max
+        rescue ArgumentError
+          unless @warned_scale
+            @warned_scale = true
+            warn("SPEC_TIME_SCALE=#{raw.inspect} is not a number: waiting with scale 1")
+          end
+          1.0
+        end
+
+        # +timeout+ s scaled: the deadline a helper waits for.
+        def scaled(timeout, scale = time_scale) = timeout * scale
+
+        # How long a wait took to give up: "5 s", or with a scale
+        # "10.0 s (5 s × SPEC_TIME_SCALE 2)".
+        def waited(timeout, scale = time_scale)
+          return "#{timeout} s" if scale == 1
+
+          "#{scaled(timeout, scale).round(3).to_f} s (#{timeout} s × SPEC_TIME_SCALE #{number(scale)})"
+        end
+
+        # The one failure text of wait_for, join! and pop!:
+        # "timed out after 5 s waiting for <what>", then "; last saw: <value>"
+        # when +seen+ (a callable) is given, its value's inspect cut to
+        # SEEN_LIMIT chars.
+        def timed_out_message(what, timeout, scale: time_scale, seen: nil)
+          message = "timed out after #{waited(timeout, scale)} waiting for #{what}"
+          seen ? "#{message}; last saw: #{seen_text(seen)}" : message
+        end
+
+        private
+
+        def number(value) = value == value.to_i ? value.to_i.to_s : value.to_s
+
+        def seen_text(seen)
+          text = seen.call.inspect
+          text.length > SEEN_LIMIT ? "#{text[0, SEEN_LIMIT]}…" : text
+        rescue StandardError => e
+          "(raised #{e.class}: #{e.message})"
+        end
+      end
+
       # The block's value once truthy, polled every +interval+ s; after
-      # +timeout+ s its last (falsy) value.
+      # +timeout+ s (scaled) its last (falsy) value. For a wait whose
+      # timeout is fine (a bounded best-effort wait); a wait that must hold
+      # is #wait_for.
       def wait_until(timeout: 5, interval: 0.01)
-        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + Waiting.scaled(timeout)
         until (value = yield)
           return yield if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
 
@@ -19,15 +78,39 @@ module Samagotchi
         value
       end
 
-      # #wait_until, raising "timed out after N s waiting for <what>" when
-      # the time runs out: for a wait no check follows.
-      def wait_until!(what, timeout: 5, interval: 0.01, &)
-        wait_until(timeout: timeout, interval: interval, &) || raise("timed out after #{timeout} s waiting for #{what}")
+      # Polls the block every +interval+ s until it returns truthy and
+      # returns that value. After +timeout+ s (scaled) it raises
+      # Waiting::TimedOut naming +what+ ("timed out after 5 s waiting for
+      # <what>"); +seen+ (a callable) adds what it last saw, since a falsy
+      # block value tells nothing.
+      def wait_for(what, timeout: 5, interval: 0.01, seen: nil, &)
+        wait_until(timeout: timeout, interval: interval, &) ||
+          raise(TimedOut, Waiting.timed_out_message(what, timeout, seen: seen))
+      end
+
+      # Thread#join with a deadline (scaled): a thread that never ends fails
+      # the test with "<what> to finish" named, not after a suite-wide guard.
+      # Raises what the thread raised, as #join does.
+      # @return [Thread]
+      def join!(thread, what, timeout: 5)
+        thread.join(Waiting.scaled(timeout)) || raise(TimedOut, Waiting.timed_out_message("#{what} to finish", timeout))
+      end
+
+      # Queue#pop with a deadline (scaled), for a queue that never carries
+      # nil: nothing pushed in time fails with +what+ named. A gate a stubbed
+      # lib thread waits on while the test works takes a longer +timeout+
+      # (30): it only has to end a hang, never cut a slow test short.
+      # @return the value popped.
+      def pop!(queue, what, timeout: 5)
+        value = queue.pop(timeout: Waiting.scaled(timeout))
+        raise(TimedOut, Waiting.timed_out_message(what, timeout)) if value.nil?
+
+        value
       end
 
       # Whether no process +pid+ is left (a killed grandchild is reaped a
-      # moment later: wait_until { process_gone?(pid) }). One that isn't
-      # ours to signal is still there (ProcessGroup.alive?).
+      # moment later: wait_for("pid gone") { process_gone?(pid) }). One that
+      # isn't ours to signal is still there (ProcessGroup.alive?).
       def process_gone?(pid) = !ProcessGroup.alive?(pid)
     end
   end

data/lib/samagotchi/tools/task_runtime.rb

@@ -99,9 +99,10 @@ module Samagotchi
       # status. Only while this chi runs. A record a refresh finished in
       # between (failed, no exit code) is corrected; a stopped task (its
       # stop flag on disk) is left as task_stop records it.
-      # The thread is named EXIT_STATUS_THREAD (a spec joins them before it
-      # removes the tasks' folder) here, before it returns: named from
-      # inside, a thread not yet scheduled was missed by that join.
+      # The thread is named EXIT_STATUS_THREAD (a spec joins them with
+      # #quiesce! before it removes the tasks' folder) here, before it
+      # returns: named from inside, a thread not yet scheduled was missed by
+      # that join.
       # @return [Thread]
       def record_exit_status(waiter, path, task_id:)
         # Absolute now: safe against a later chdir.
@@ -132,6 +133,19 @@ module Samagotchi
       # record_exit_status's threads' name.
       EXIT_STATUS_THREAD = "task_exit_status"
 
+      # Joins every live EXIT_STATUS_THREAD, for up to +timeout+ s in all: a
+      # spec calls it (with_task_dir) after stopping its tasks and before it
+      # removes their folder, which a recorder may still write into. chi
+      # itself doesn't call it.
+      # @return [Array<Thread>] the ones still alive after +timeout+ (empty
+      #   when quiet)
+      def quiesce!(timeout: 5)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+        threads = Thread.list.select { |thread| thread.name == EXIT_STATUS_THREAD && thread != Thread.current }
+        threads.each { |thread| thread.join([deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max) }
+        threads.select(&:alive?)
+      end
+
       # Reasons a refresh finishes a record with when it read no exit code.
       MISSING_CODE_REASONS = ["process_ended_without_exit_code", NOT_CHIS_PROCESS].freeze
 

data/lib/samagotchi/turn_checkpoint.rb

@@ -159,13 +159,16 @@ module Samagotchi
     # Every mark of the turn carries +enqueued_id+ (InterruptedTurn's).
     # @param prompt [String, nil]
     # @param enqueued_id [String, nil] the input the turn runs
-    def begin_turn(prompt: nil, enqueued_id: nil)
+    # @param author [Hash] a participant's prompt's author fields
+    #   (Engine#author_fields), so a kill before the first boundary leaves
+    #   it theirs; {} for the owner's
+    def begin_turn(prompt: nil, enqueued_id: nil, author: {})
       @boundaries = 0
       @enqueued_id = enqueued_id.to_s.empty? ? nil : enqueued_id.to_s
       @written_at = nil
       @started_at = Time.now.iso8601(3)
       messages = Array(@session.messages).dup
-      messages << { role: "user", content: prompt } unless prompt.to_s.empty?
+      messages << { role: "user", content: prompt, **author } unless prompt.to_s.empty?
       @session.turn_checkpoint = { messages: messages, marker: InterruptedTurn.new(started_at: @started_at, at: @started_at, iteration: 0,
                                                                                    enqueued_id: @enqueued_id) }
     end

data/lib/samagotchi/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Samagotchi
-  VERSION = "0.103.0"
+  VERSION = "0.105.0"
   # Sent as the User-Agent of every HTTP request chi makes.
   USER_AGENT = "chi/#{VERSION}"
 end