samagotchi 0.155.0 → 0.156.0

data/lib/samagotchi/session_manager/spawn.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+require_relative "../worker_link"
+
+module Samagotchi
+  class SessionManager
+    # How long after a spawn a resume trusts that worker to take the
+    # session, while it runs (and the session wasn't stopped since).
+    WORKER_START_GRACE = WorkerLink::START_LIMIT
+
+    @spawns = {}
+    @spawns_mutex = Mutex.new
+
+    # Spawn: deciding to start a session's worker (a new session, a resume, a
+    # delegate's report waking its parent) and the spawns this process
+    # remembers. Calls WorkerCommandPart's spawn_options and worker_command and
+    # the facade's ownership checks. Meant to be loaded through samagotchi/session_manager.
+    module SpawnPart
+      # Spawn a new background session that processes the given prompt (or,
+      # with none, waits idle for input).
+      #
+      # Returns the session object with its ID. Every worker always starts its
+      # per-session Bridge (the single live client transport), so external
+      # clients can reach it once the sidecar is published.
+      # @param memories [Array<String>] --memory: preloaded into the worker's prompt
+      # @param muted_memories [Array<String>] --mute: hidden from the session
+      #   (both are session fields, so a respawn keeps them)
+      # @param parent_id [String, nil] the session that delegates this one (the
+      #   `delegate` tool) or was forked from; a session field too
+      # @param messages [Array<Hash>] a conversation to start from (a plugin's
+      #   ctx.sessions.fork); its image refs are copied from +images_from+
+      # @param images_from [String, nil] the session dir the seed's images are in
+      # @param preview [String, nil] what the lists show before the first turn
+      #   (the prompt's preview by default, else the seed's first user message)
+      # @param title [String, nil] the session's own name (Session#title; a
+      #   continue's link inherits its previous link's), saved before the
+      #   worker starts: the worker owns the file from then on
+      # @param setup [SessionSetup] the session's own settings (chi
+      #   --llm-context; a fork and a continue copy their session's; a delegate
+      #   child the toolkit its delegate call named); without a toolkit the
+      #   session is stored on config's toolkit.default, a delegate child on
+      #   toolkit.child (Toolkit.configured)
+      # @param continues [String, nil] the session this one continues (a
+      #   session field; SessionContinue)
+      # @param note [String, nil] a context note the session starts with,
+      #   from +note_source+: queued before its worker spawns, so its first
+      #   turn sees it
+      # @param review [SessionReview, nil] a review session's (ReviewStart; a
+      #   continue keeps its previous link's)
+      # @param verify_fresh [VerifyFreshOrigin, nil] a Verify fresh child's
+      #   requester, rules and claim (VerifyFresh.start)
+      # @param before_spawn [#call, nil] (session) runs once the session and
+      #   its note are saved, before its worker spawns
+      # @param scratch [Boolean] a `chi scratch` session (its terminal's
+      #   AttachLauncher): its worker runs a scratch Engine and deletes it
+      #   once its last client is gone
+      # @return [Session] with #seed_images_dropped: refs whose file was gone,
+      #   and #model_warning: an id its host's saved list doesn't have
+      def spawn_session(prompt:, mode: "assist", working_directory: nil, model_name: nil, state_dir: nil,
+                        memories: [], muted_memories: [], parent_id: nil, messages: [], images_from: nil,
+                        preview: nil, title: nil, delegate: false, setup: SessionSetup.new, continues: nil, note: nil,
+                        note_source: "chi", before_spawn: nil, review: nil, verify_fresh: nil, scratch: false)
+        sd = state_dir || Session.default_state_dir
+        # The resolved ref is stored (a resumed session keeps its model when an
+        # alias is retargeted), with the name as typed beside it. A wrong host
+        # refuses here, before anything is saved or spawned; a model id the
+        # host's saved list doesn't have (ModelListStore) only warns
+        # (#model_warning on the session), as some hosts serve ids they don't
+        # list.
+        typed = Samagotchi::ModelProfile.check_host!(Samagotchi::ModelProfile.required_model_name(model_name))
+        model_warning = Samagotchi::ModelProfile.model_warning(typed)
+        ref = HostConfig.model_ref(typed).ref
+        session = Session.new_session(
+          mode: mode,
+          model_name: ref,
+          model_typed: typed == ref ? nil : typed,
+          working_directory: working_directory || Dir.pwd,
+          preloaded_memory_names: memories,
+          muted_memory_names: muted_memories,
+          parent_id: parent_id,
+          messages: messages,
+          delegate: delegate,
+          setup: setup,
+          continues: continues,
+          review: review,
+          scratch: scratch
+        )
+        session.toolkit = Toolkit.configured(delegate: delegate) if session.toolkit.nil?
+        session.toolkit_origin = setup.toolkit_origin || "default"
+        # Saved before the worker starts: its first turn is already the requester's.
+        session.verify_fresh = verify_fresh
+        session.model_warning = model_warning
+        # With no prompt there is no first turn to run (an attaching UI sends
+        # the prompts), so the session starts idle.
+        session.status = prompt.to_s.strip.empty? ? Session::STATUS_IDLE : Session::STATUS_RUNNING
+        session.last_prompt = prompt
+        # The first prompt's turn gets an id like a sent message's, so a wait
+        # for it can name it (chi send --new --wait, chi web's create): the
+        # worker runs it with that origin, and its reply file and last_turn
+        # carry the id.
+        session.first_turn_id = SecureRandom.uuid unless prompt.to_s.strip.empty?
+        # The worker takes last_prompt and clears it, and messages are saved at
+        # the turn's end: until then this is the only preview a list has.
+        session.first_preview = Session.preview_of(preview.to_s.strip.empty? ? prompt : preview)
+        session.title = title
+        session_dir = Session.session_dir(session.id, state_dir: sd)
+        unless session.messages.empty?
+          session.messages, dropped = ImageStore.copy_refs(session.messages, from: images_from, to: session_dir)
+          session.seed_images_dropped = dropped
+        end
+        setup_session_directory(session_dir, session, state_dir: sd)
+        SessionInbox.write_note(session.id, text: note, source: note_source, state_dir: sd) if note
+        before_spawn&.call(session)
+        spawn_worker_for_session(session, state_dir: sd)
+        session
+      end
+
+      # Ensure an existing session has a live worker process.
+      # Returns the loaded session after state reconciliation.
+      # @raise [OwnedByTUI] when the interactive TUI owns the session
+      def resume_session(session_id, state_dir: nil)
+        sd = state_dir || Session.default_state_dir
+        session = Session.load(session_id, state_dir: sd)
+        # Its terminal's worker came from spawn_session; nothing wakes another.
+        raise ScratchSession, session.id if session.scratch
+        return session if refuse_tui!(session.id, state_dir: sd)
+
+        # status is turn state, not liveness: a new worker runs no turn yet,
+        # and must not find the session stopped (it would exit): neither the
+        # marker nor a status field from before it.
+        # One this process just spawned may not hold the lock yet (a slow
+        # start, deliver_turn's second resume): it would only lose it.
+        return session if still_starting?(session.id, state_dir: sd)
+        # A continue moving delegates to it spawns its worker once they are
+        # moved; one spawned now (a moved child's ring) would race the move.
+        return session if StartingMarker.fresh?(session.id, state_dir: sd)
+
+        Session.clear_stopped(session.id, state_dir: sd)
+        session.status = Session::STATUS_IDLE
+        session.save(state_dir: sd)
+        spawn_worker_for_session(session, state_dir: sd)
+        session
+      end
+
+      # A delegate child rang a parent that has no worker (ChildRing): start
+      # one, which runs a turn for the report. Not for a parent that is gone,
+      # stopped, archived, a scratch session, a one-turn run's, or a
+      # continue's new link still taking its delegates (its own spawn comes):
+      # the ring waits on disk for whoever runs it next (#resume_session
+      # itself would clear the stop). A stop landing between the check and
+      # the resume is a small window, left as is. An archived parent with a
+      # later link in its chain wakes that link instead: a ring a moved child
+      # wrote just as the continue moved it reaches the new link that way
+      # (ChildReports reads its predecessor's rings).
+      # @return [Symbol] :woken, or why not (:gone, :stopped, :archived,
+      #   :scratch, :owned, :starting)
+      def wake_for_report(session_id, state_dir: nil, forward: true)
+        sd = state_dir || Session.default_state_dir
+        return :gone unless Session.exist?(session_id, state_dir: sd)
+        return :owned if session_owner(session_id, state_dir: sd)
+        return :stopped if Session.stopped_marker?(session_id, state_dir: sd)
+
+        if ArchiveStore.archived?(Session.session_dir(session_id, state_dir: sd))
+          following = forward && SessionChain.next_of(session_id, state_dir: sd)
+          return following ? wake_for_report(following, state_dir: sd, forward: false) : :archived
+        end
+        return :starting if StartingMarker.fresh?(session_id, state_dir: sd)
+
+        session = Session.load(session_id, state_dir: sd)
+        return :stopped if session.status == Session::STATUS_STOPPED
+        return :scratch if session.scratch
+
+        resume_session(session_id, state_dir: sd)
+        Log.info(:worker, "delegate_woke_parent", parent: session_id[0, 8])
+        :woken
+      rescue OwnedByTUI
+        :owned
+      end
+
+      # A worker this process spawned for the session within
+      # WORKER_START_GRACE is still running, and no stop came since.
+      private def still_starting?(session_id, state_dir:)
+        return false if Session.stopped_marker?(session_id, state_dir: state_dir)
+
+        at, waiter = @spawns_mutex.synchronize { @spawns[[state_dir, session_id]] }
+        return false unless at && waiter.respond_to?(:alive?) && waiter.alive?
+
+        Process.clock_gettime(Process::CLOCK_MONOTONIC) - at < WORKER_START_GRACE
+      end
+
+      private def note_spawn(session_id, state_dir:, waiter:)
+        @spawns_mutex.synchronize do
+          @spawns.delete_if { |_, (_, w)| !(w.respond_to?(:alive?) && w.alive?) }
+          @spawns[[state_dir, session_id]] = [Process.clock_gettime(Process::CLOCK_MONOTONIC), waiter]
+        end
+      end
+
+      # Forget the spawns #still_starting? remembers (specs).
+      def forget_spawns
+        @spawns_mutex.synchronize { @spawns.clear }
+      end
+
+      # Starter processes are children of chi web, unlike the detached workers
+      # they launch; the web restart adopts these pids before replacing itself.
+      def live_spawn_pids
+        @spawns_mutex.synchronize do
+          @spawns.delete_if { |_, (_, waiter)| !(waiter.respond_to?(:alive?) && waiter.alive?) }
+          @spawns.values.filter_map do |_, waiter|
+            waiter.pid if waiter.respond_to?(:pid) && waiter.alive?
+          end.uniq.sort
+        end
+      end
+
+      private def setup_session_directory(session_dir, session, state_dir:)
+        FileUtils.mkdir_p(session_dir)
+        FileUtils.mkdir_p(File.join(session_dir, SessionInbox::INPUT_DIR))
+        FileUtils.mkdir_p(File.join(session_dir, SessionInbox::OUTPUT_DIR))
+        session.save(state_dir: state_dir)
+      end
+
+      private def spawn_worker_for_session(session, state_dir:)
+        session_dir = Session.session_dir(session.id, state_dir: state_dir)
+        FileUtils.mkdir_p(session_dir)
+        FileUtils.mkdir_p(File.join(session_dir, SessionInbox::INPUT_DIR))
+        FileUtils.mkdir_p(File.join(session_dir, SessionInbox::OUTPUT_DIR))
+
+        opts = spawn_options(session)
+        env = opts.delete(:env)
+        command = worker_command(session.id, state_dir: state_dir)
+        pid = env ? Process.spawn(env, *command, **opts) : Process.spawn(*command, **opts)
+        # Reaped when it exits: a long-lived spawner (chi web) keeps no zombies.
+        # +pid+ is the worker's starter, which exits once the worker it forked
+        # holds the lock (WorkerBoot.detach_from_spawner): the lock's pid is the worker's.
+        note_spawn(session.id, state_dir: state_dir, waiter: Process.detach(pid))
+        Log.info(:worker, "spawn", sid: session.id, child_pid: pid)
+        pid
+      end
+    end
+
+    extend SpawnPart
+  end
+end

data/lib/samagotchi/session_manager/worker_command.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Samagotchi
+  class SessionManager
+    # CROSS-VERSION CONTRACT: this boot string, the entry it names
+    # (run_session_loop(id, state_dir:), the forwarder below), the env keys
+    # spawn_options sets and the input/ file
+    # format are read by a NEWER chi than the one writing them. Change them
+    # only additively (spec/worker_contract_spec.rb).
+    BOOT_FALLBACK_ENV = "SAMAGOTCHI_BOOT_FALLBACK"
+    # "1" in a worker spawn_options started: it traps signals (WorkerSignals).
+    # WorkerBoot.run_session_loop deletes it from ENV as it reads it.
+    WORKER_PROCESS_ENV = "SAMAGOTCHI_WORKER_PROCESS"
+    # "1" in a worker spawn_options started: WorkerBoot.run_session_loop
+    # leaves the spawner's process tree first (detach_from_spawner); deleted
+    # from ENV as it is read, so an rspec run by the worker's execute never
+    # forks.
+    WORKER_DETACH_ENV = "SAMAGOTCHI_WORKER_DETACH"
+
+    # The boot string's rescue under a chi home: one line in the spawner's
+    # log file (SAMAGOTCHI_LOG_FILE, set by spawn_options), LogLine's shape,
+    # then exit. Single-quoted: the #{} run in the worker. SID is replaced.
+    # rubocop:disable-next Lint/InterpolationCheck -- the worker interpolates these, not this process
+    HOME_BOOT_FAILED = 'rescue ScriptError, StandardError => e; f = ENV["SAMAGOTCHI_LOG_FILE"].to_s; begin; ' \
+                       'File.open(f, "a") { |io| io.puts(Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.%LZ") + ' \
+                       '" ERROR worker pid=#{Process.pid} sid=SID boot_failed ruby=#{RbConfig.ruby.inspect} error=" + ' \
+                       "e.message.lines.first.to_s.strip.inspect) } unless f.empty?; rescue SystemCallError; end; exit 1"
+
+    # WorkerCommand: the worker's command line and env (CROSS-VERSION
+    # CONTRACT: spec/worker_contract_spec.rb); the worker process itself is
+    # WorkerBoot (worker_boot.rb), which only the forwarder (run_session_loop)
+    # loads, as a worker boots. Calls no other part. Meant to be loaded
+    # through samagotchi/session_manager.
+    module WorkerCommandPart
+      # Build the opts hash passed to Process.spawn for a forked worker. The
+      # child inherits this process's ENV and reads config.yml itself (as it
+      # is when the worker starts); opts[:env] adds to it (merged, not
+      # replaced) what it can't read there: this `chi`'s CLI settings
+      # (Config.cli_env) and the absolute log file. It unsets
+      # what ChildEnv lists (SAMAGOTCHI_GUARDRAILS_*, the session markers).
+      # XDG_CONFIG_HOME passes: whoever starts or
+      # wakes the worker picks which config.yml it reads (docs/guardrails.md).
+      # XDG_STATE_HOME and XDG_CONFIG_HOME, when set, pass made absolute
+      # (Paths): the worker starts in the session's directory.
+      private def spawn_options(session)
+        # Own process group: workers outlive `chi web`, and a Ctrl-C in its
+        # terminal must not reach them (the worker then leaves the process
+        # tree too: WORKER_DETACH_ENV).
+        opts = { out: File::NULL, err: File::NULL, pgroup: true }
+        # The worker's tools (and `!cmd`) run in the session's directory, not
+        # in the cwd of whoever woke it (`chi web`, another terminal).
+        dir = session.working_directory.to_s
+        if !dir.empty? && File.directory?(dir)
+          opts[:chdir] = dir
+        else
+          Log.warn(:worker, "cwd_gone", sid: session.id, dir: dir, cwd: Dir.pwd)
+        end
+        # (A chi a session's tool runs took that session's project env out at
+        # its start: bin/chi, ProjectEnv.take_orig!.)
+        child_env = {}
+        # A worker gets no CLI args: --model, --thinking, --log-level and the
+        # other flags this chi was started with.
+        child_env.merge!(Config.cli_env)
+        child_env[ModelProfile::MODEL_FROM_CLI_ENV] = "1" if child_env.key?(ModelProfile::MODEL_ENV)
+        # And the spawner's debug log, absolute: a relative log.file would
+        # otherwise land in the worker's (the session's) directory.
+        log_path = begin LogPath.resolve rescue nil end # degrade-ok: no log to say it in; the worker's is off (below)
+        if log_path
+          child_env["SAMAGOTCHI_LOG_FILE"] = log_path
+        else
+          child_env["SAMAGOTCHI_LOG_DISABLE"] = "true"
+        end
+        # And the spawner's XDG dirs, absolute: a relative XDG_STATE_HOME or
+        # XDG_CONFIG_HOME would otherwise be read from the session's directory.
+        child_env.merge!(Paths.absolute_xdg)
+        # Unset (Process.spawn unsets a key mapped to nil) what no child chi
+        # takes from its spawner (ChildEnv): the spawner's guardrails settings
+        # (a parent agent starting or waking a session; config.yml no longer
+        # reads them, this keeps an older or future env-exposed one out as
+        # well); the session markers, a session's own (a worker started from
+        # its commands, chi send, is another session: its execute and tasks
+        # set its own; nothing else it runs has one); and a smoke's fake
+        # "newest installed", for this process's notices only.
+        child_env.merge!(ChildEnv.unset)
+        # A worker process: WorkerBoot.run_session_loop traps signals, and
+        # leaves this process's tree.
+        child_env[WORKER_PROCESS_ENV] = "1"
+        child_env[WORKER_DETACH_ENV] = "1"
+        # A gem-mode worker activates the newest installed chi itself: under
+        # `bundle exec` (a Gemfile that has samagotchi) Bundler's RUBYOPT would
+        # hold it to the bundle's version, and its badge would never clear.
+        child_env.merge!(ChildEnv.unbundled)
+        opts[:env] = child_env unless child_env.empty?
+        opts
+      end
+
+      # The worker's entry, the one every boot string names (CROSS-VERSION
+      # CONTRACT: spec/worker_contract_spec.rb): it only forwards to
+      # WorkerBoot.run_session_loop, which runs the worker process. Looked up
+      # by name on purpose: a constant reference here would put SessionManager
+      # back into one cycle with Worker, Engine and Bridge (the class map).
+      # REMOVE one release after the boot string switches (see
+      # ~/.claude/plans/arch-worker-boot.md).
+      def run_session_loop(session_id, state_dir: nil, **)
+        require_relative "../worker_boot"
+        Samagotchi.const_get(:WorkerBoot).run_session_loop(session_id, state_dir: state_dir, **)
+      end
+
+      # The worker's command line. Run from an installed gem, the worker
+      # activates the newest installed samagotchi (`>= 0.a`, prereleases
+      # count), whoever spawns it: an old chi web, chi send or --attach then
+      # starts new workers on the newest chi rather than on its own. Activating
+      # the gem also makes its dependencies resolve as its gemspec pins them
+      # (reline ~> 0.6.3). Should that fail (a broken newest install), the
+      # worker falls back to the spawner's own version and logs why.
+      #
+      # A source checkout (bin/chi, bundle exec; InstalledGem.spec is nil
+      # there) keeps the plain -I lib: a worktree's bin/chi spawns its own code.
+      def worker_command(session_id, state_dir:, gem_spec: InstalledGem.spec,
+                         home: ChiHome.current(gem_spec: gem_spec))
+        boot = "require 'samagotchi/session_manager'; " \
+               "Samagotchi::SessionManager.run_session_loop('#{session_id}', state_dir: #{state_dir.inspect})"
+        return [RbConfig.ruby, "-I", File.expand_path("../..", __dir__), "-e", boot] unless gem_spec
+        return home_worker_command(session_id, state_dir: state_dir, gem_spec: gem_spec, home: home) if home
+
+        activate = "begin; gem 'samagotchi', '>= 0.a'; rescue LoadError => e; " \
+                   "ENV['#{BOOT_FALLBACK_ENV}'] = e.message; gem 'samagotchi', '= #{gem_spec.version}'; end"
+        [RbConfig.ruby, "-e", "#{activate}; #{boot}"]
+      end
+
+      # With a chi home (ChiHome.current, docs/architecture.md "How chi
+      # launches itself"): the home's ruby, `Gem.paths` set to its gems in the
+      # boot string (no GEM_* in the worker's env), the newest chi there, else
+      # the fallback pin (ChiHome.fallback_version). A home that still can't
+      # load chi (a broken install) logs HOME_BOOT_FAILED and exits: never a
+      # silent spawn timeout.
+      private def home_worker_command(session_id, state_dir:, gem_spec:, home:)
+        newest = home.newest_version
+        if newest && newest < Gem::Version.new(gem_spec.version.to_s)
+          ChiHome.warn_once(:older_than_spawner, home: newest.to_s, spawner: gem_spec.version.to_s)
+        end
+        fallback = ChiHome.fallback_version(home, gem_spec)
+        activate = if fallback
+                     "begin; gem 'samagotchi', '>= 0.a'; rescue LoadError => e; " \
+                       "ENV['#{BOOT_FALLBACK_ENV}'] = e.message; gem 'samagotchi', '= #{fallback}'; end"
+                   else
+                     "gem 'samagotchi', '>= 0.a'"
+                   end
+        home.boot_argv(run: "begin; #{activate}; require 'samagotchi/session_manager'; " \
+                             "#{HOME_BOOT_FAILED.sub("SID", session_id)}; end; " \
+                             "Samagotchi::SessionManager.run_session_loop('#{session_id}', state_dir: #{state_dir.inspect})")
+      end
+    end
+
+    extend WorkerCommandPart
+  end
+end