samagotchi 0.160.0 → 0.161.0

data/lib/samagotchi/session_manager/removal.rb

@@ -6,12 +6,20 @@ module Samagotchi
     # reason: :worker_running (not asked to stop it) or :still_stopping
     # (stopped, but the worker outlived the wait).
     class DeleteRefused < StandardError
-      attr_reader :session_id, :reason
+      attr_reader :session_id, :reason, :schedule_id
 
-      def initialize(session_id, reason)
+      def initialize(session_id, reason, message: nil, schedule_id: nil)
         @session_id = session_id
         @reason = reason
-        super(reason == :still_stopping ? "session #{session_id}'s worker is still shutting down" : "session #{session_id}'s worker is running")
+        @schedule_id = schedule_id
+        message ||= case reason
+                    when :still_stopping then "session #{session_id}'s worker is still shutting down"
+                    when :scheduled
+                      "this session is the target of an enabled schedule" \
+                        "#{"; cancel it with chi schedule cancel #{schedule_id}" if schedule_id}"
+                    else "session #{session_id}'s worker is running"
+                    end
+        super(message)
       end
     end
 
@@ -37,6 +45,8 @@ module Samagotchi
         when :queued_child then "its delegate #{@busy_id[0, 8]} has a prompt queued, which a resume would run; let it run first"
         when :offer then "the step-limit question waits for an answer; answer Continue or Stop first"
         when :offer_child then "its delegate #{@busy_id[0, 8]} waits for an answer to its step-limit question; answer it first"
+        when :starting then "its worker is starting; try again in a moment"
+        when :starting_child then "its delegate #{@busy_id[0, 8]}'s worker is starting; try again in a moment"
         else "a turn is running; wait for it or cancel it first"
         end
       end
@@ -47,6 +57,12 @@ module Samagotchi
     # workers stopped first; discarded: empty sessions its stopping worker deleted.
     ArchiveResult = Data.define(:id, :archived, :stopped, :discarded)
 
+    # Every store with session-scoped state. Deletion, retention and failed
+    # continuation cleanup all pass through this registry.
+    SESSION_STATE_STORES = %i[
+      session_files plugin_session_state context_sources participants system_one_runs context_candidates_log
+    ].freeze
+
     # Removal: archive, unarchive and delete a session (an archive takes its
     # delegates too), and the retention sweep's entry. Calls LifecyclePart's
     # stop_session, ListingPart's descendant_ids and the facade's ownership
@@ -87,23 +103,29 @@ module Samagotchi
         raise ArchiveRefused.new(id, :scratch) if Session.load(id, state_dir: state_dir).scratch
 
         tree = [id, *descendant_ids(id, state_dir, except: except)]
-        owners = tree.to_h { |sid| [sid, session_owner(sid, state_dir: state_dir)] }
+        presences = tree.to_h do |sid|
+          [sid, WorkerPresence.read(sid, state_dir: state_dir, owner: session_owner(sid, state_dir: state_dir))]
+        end
         tree.each do |sid|
-          refusal = archive_refusal(sid, owners[sid], state_dir)
+          refusal = archive_refusal(sid, presences[sid], state_dir)
           raise ArchiveRefused.new(id, sid == id ? refusal : :"#{refusal}_child", busy_id: sid) if refusal
         end
-        [tree, owners]
+        [tree, presences.transform_values(&:owner)]
       end
 
-      # What keeps +sid+ (in an archive's tree) from being archived: :busy (a
+      # What keeps +sid+ (in an archive's tree) from being archived:
+      # :starting (a worker is being started for it, holding no lock yet: a
+      # continue's new link, a resume), :busy (a
       # turn runs), :queued (input/ holds a prompt, which outlives the stop and
       # runs on a resume, live worker or not), :offer (a live worker's continue
       # offer, which the stop would drop), or nil.
+      # The status is read only while someone owns it (+presence+'s owner).
       # @raise [OwnedByTUI] a one-turn chi run owns it
-      private def archive_refusal(sid, owner, state_dir)
-        raise OwnedByTUI, sid if owner&.tui?
+      private def archive_refusal(sid, presence, state_dir)
+        raise OwnedByTUI, sid if presence.tui_owned?
+        return :starting if presence.starting? && !presence.owned?
 
-        session = owner && Session.load(sid, state_dir: state_dir)
+        session = presence.owned? ? Session.load(sid, state_dir: state_dir) : nil
         return :busy if session&.status == Session::STATUS_RUNNING
         return :queued unless SessionInbox.find_new_input_files(Session.session_dir(sid, state_dir: state_dir)).empty?
 
@@ -135,12 +157,14 @@ module Samagotchi
       # @return [Hash] {id:, removed: [paths], stopped: whether a worker was stopped}
       # @raise [ArgumentError] unknown id (Session::AmbiguousId for a prefix of several)
       # @raise [OwnedByTUI] a one-turn chi run owns it (never stopped from here)
-      # @raise [DeleteRefused] a worker owns it and +stop+ is false, or it
-      #   outlived the wait
+      # @raise [DeleteRefused] a worker owns it or is being started for it
+      #   (the starting marker) and +stop+ is false, or it outlived the wait
       # @param purge_runs [Boolean] also delete System 1's saved runs of it and
       #   its candidate-log lines; a session a worker left empty has none worth
       #   the read
-      def delete_session(id_or_prefix, state_dir: nil, stop: false, wait: STOP_WAIT, purge_runs: true)
+      # @param allow_scheduled [Boolean] internal rollback of a failed continue
+      def delete_session(id_or_prefix, state_dir: nil, stop: false, wait: STOP_WAIT, purge_runs: true,
+                         allow_scheduled: false)
         sd = state_dir || Session.default_state_dir
         given = id_or_prefix.to_s
         # Only a plain id: anything else could name a path outside the state dir.
@@ -152,12 +176,81 @@ module Samagotchi
         # A folder with no file left is still deleted (not resolve!).
         raise Session::NotFound, given unless File.exist?(path) || Dir.exist?(dir)
 
-        owner = refuse_tui!(id, state_dir: sd)
-        if owner
+        unless allow_scheduled
+          require_relative "../schedules"
+          schedule = Schedules.new(state_dir: sd).protecting(id)
+          if schedule
+            raise DeleteRefused.new(id, :scheduled, schedule_id: schedule.id)
+          end
+        end
+
+        presence = WorkerPresence.read(id, state_dir: sd, owner: session_owner(id, state_dir: sd))
+        raise OwnedByTUI, id if presence.tui_owned?
+
+        held = presence.owned? || presence.starting?
+        if held
           raise DeleteRefused.new(id, :worker_running) unless stop
-          raise DeleteRefused.new(id, :still_stopping) unless stop_session(id, state_dir: sd, wait: wait)
+          raise DeleteRefused.new(id, :still_stopping) unless stop_held(id, state_dir: sd, wait: wait)
         end
 
+        removed = remove_files(id, path, dir, state_dir: sd, purge_runs: purge_runs)
+        { id: id, removed: removed, stopped: held }
+      end
+
+      # Remove every store with state scoped to +id+. The file and sidecar
+      # directory keep the deleting owner lock; the other stores have their
+      # own locking where they need it.
+      # @return [Array<String>] the session file, sidecar and context paths removed
+      def remove_files(id, path, dir, state_dir:, purge_runs: true)
+        removed = []
+        SESSION_STATE_STORES.each do |store|
+          case store
+          when :session_files
+            removed.concat(remove_session_files(id, path, dir, state_dir: state_dir))
+          when :plugin_session_state
+            PluginSessionState.remove(id)
+          when :context_sources
+            removed.concat(ContextSources.remove_session(id, state_dir: state_dir))
+          when :participants
+            Participants.end_sessions([id], by: "delete", state_dir: state_dir)
+          when :system_one_runs
+            next
+          when :context_candidates_log
+            next
+          else
+            raise ArgumentError, "no cleanup for session store #{store.inspect}"
+          end
+        end
+        purge_removal_stores([id], state_dir: state_dir) if purge_runs
+        removed
+      end
+
+      # The session-scoped logs are expensive to scan; callers removing a
+      # batch purge them once after the session files are gone.
+      def purge_removal_stores(ids, state_dir:)
+        return if ids.empty?
+
+        purge_session_runs(ids, state_dir)
+        ContextCandidatesLog.purge_sessions(ids, state_dir: state_dir)
+      end
+
+      # The session's file and directory, removed while this process holds
+      # its owner lock (kind deleting, which a stop never signals), so no
+      # worker takes the session in the middle; the starting marker is read
+      # again under it (a resume that claimed the start since the caller's
+      # check keeps the session; one after it sees the lock held). Taken
+      # without waiting: one that took it since the check keeps the session.
+      # The lock file goes with the directory; a worker starting after that
+      # makes a new one, and finds no session (Session.load). The delete and
+      # the retention sweep and failed-continue cleanup use #remove_files.
+      # @return [Array<String>] the paths removed
+      # @raise [DeleteRefused] a worker took the lock, or a start was
+      #   claimed, after the caller's check
+      def remove_session_files(id, path, dir, state_dir:)
+        lock = Dir.exist?(dir) ? OwnerLock.acquire(dir, kind: OwnerLock::DELETING, wait: 0) : nil
+        raise DeleteRefused.new(id, :worker_running) if Dir.exist?(dir) && lock.nil?
+        raise DeleteRefused.new(id, :worker_running) if StartingMarker.fresh?(id, state_dir: state_dir)
+
         removed = []
         if File.exist?(path)
           FileUtils.rm_f(path)
@@ -167,21 +260,32 @@ module Samagotchi
           FileUtils.rm_rf(dir)
           removed << dir
         end
-        PluginSessionState.remove(id)
-        removed.concat(ContextSources.remove_session(id, state_dir: sd))
-        Participants.end_sessions([id], by: "delete", state_dir: sd)
-        if purge_runs
-          purge_session_runs(id, sd)
-          ContextCandidatesLog.purge_session(id, state_dir: sd)
+        removed
+      ensure
+        lock&.release
+      end
+
+      # Stop the worker that owns +id+ or is being started for it (the stop
+      # marker: one that takes the lock leaves at once), and wait for both to
+      # be gone.
+      # @return [Boolean] whether no owner and no start was left within +wait+
+      private def stop_held(id, state_dir:, wait:)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + wait.to_f
+        return false unless stop_session(id, state_dir: state_dir, wait: wait)
+
+        while StartingMarker.fresh?(id, state_dir: state_dir) || session_owner(id, state_dir: state_dir)
+          return false if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
+          sleep(OwnerLock::RETRY_INTERVAL)
         end
-        { id: id, removed: removed, stopped: !owner.nil? }
+        true
       end
 
       # Never a reason a delete fails: one warn, and the sweep ages them out.
-      private def purge_session_runs(id, state_dir)
-        SystemOne::Runs.purge_session(id, sessions_dir: state_dir)
+      private def purge_session_runs(ids, state_dir)
+        SystemOne::Runs.purge_sessions(ids, sessions_dir: state_dir)
       rescue StandardError => e
-        Log.warn(:worker, "session_runs_purge_failed", sid: id, error: e.class.name, msg: e.message.to_s[0, 200])
+        Log.warn(:worker, "session_runs_purge_failed", sid: ids.first, error: e.class.name, msg: e.message.to_s[0, 200])
       end
     end
 

data/lib/samagotchi/session_manager/spawn.rb

@@ -1,19 +1,24 @@
 # frozen_string_literal: true
 
 require_relative "../worker_link"
+require_relative "../starting_marker"
 
 module Samagotchi
   class SessionManager
-    # How long after a spawn a resume trusts that worker to take the
-    # session, while it runs (and the session wasn't stopped since).
+    # How long a worker's starter waits for it to take the session
+    # (WorkerBoot.detach_from_spawner): the starting marker naming the
+    # starter holds that long at most.
     WORKER_START_GRACE = WorkerLink::START_LIMIT
 
-    @spawns = {}
-    @spawns_mutex = Mutex.new
+    # The starters this process spawned (their Process.detach waiters), for
+    # chi web's restart to adopt (#live_spawn_pids). Decides nothing: the
+    # starting marker is what holds a second spawn off.
+    @starters = []
+    @starters_mutex = Mutex.new
 
     # Spawn: deciding to start a session's worker (a new session, a resume, a
-    # delegate's report waking its parent) and the spawns this process
-    # remembers. Calls WorkerCommandPart's spawn_options and worker_command and
+    # delegate's report waking its parent) and the starters this process
+    # spawned. Calls WorkerCommandPart's spawn_options and worker_command and
     # the facade's ownership checks. Meant to be loaded through samagotchi/session_manager.
     module SpawnPart
       # Spawn a new background session that processes the given prompt (or,
@@ -22,9 +27,6 @@ module Samagotchi
       # Returns the session object with its ID. Every worker always starts its
       # per-session Bridge (the single live client transport), so external
       # clients can reach it once the sidecar is published.
-      # @param memories [Array<String>] --memory: preloaded into the worker's prompt
-      # @param muted_memories [Array<String>] --mute: hidden from the session
-      #   (both are session fields, so a respawn keeps them)
       # @param parent_id [String, nil] the session that delegates this one (the
       #   `delegate` tool) or was forked from; a session field too
       # @param messages [Array<Hash>] a conversation to start from (a plugin's
@@ -35,9 +37,9 @@ module Samagotchi
       # @param title [String, nil] the session's own name (Session#title; a
       #   continue's link inherits its previous link's), saved before the
       #   worker starts: the worker owns the file from then on
-      # @param setup [SessionSetup] the session's own settings (chi
-      #   --llm-context; a fork and a continue copy their session's; a delegate
-      #   child the toolkit its delegate call named); without a toolkit the
+      # @param setup [SessionSetup] the session's own settings (chi --memory,
+      #   --mute, --llm-context; a fork and a continue copy their session's;
+      #   a delegate child the memory and toolkit its delegate call named); without a toolkit the
       #   session is stored on config's toolkit.default, a delegate child on
       #   toolkit.child (Toolkit.configured)
       # @param continues [String, nil] the session this one continues (a
@@ -54,17 +56,20 @@ module Samagotchi
       # @param scratch [Boolean] a `chi scratch` session (its terminal's
       #   AttachLauncher): its worker runs a scratch Engine and deletes it
       #   once its last client is gone
-      # @param client_id [String, nil] who sent +prompt+ when no person did
-      #   (a ClientId: a delegating parent, a schedule's chain link): its
-      #   first turn's origin (Session#first_turn_client_id)
+      # @param client_id [String, nil] who sent +prompt+ (a ClientId); required
+      #   even when nil is intentional for a session with no initial prompt
+      # @param session_id [String, nil] an id reserved by a durable operation
       # @return [Session] with #seed_images_dropped: refs whose file was gone,
       #   and #model_warning: an id its host's saved list doesn't have
-      def spawn_session(prompt:, mode: "assist", working_directory: nil, model_name: nil, state_dir: nil,
-                        memories: [], muted_memories: [], parent_id: nil, messages: [], images_from: nil,
-                        preview: nil, title: nil, delegate: false, setup: SessionSetup.new, continues: nil, note: nil,
-                        note_source: "chi", before_spawn: nil, review: nil, verify_fresh: nil, scratch: false,
-                        client_id: nil)
+      def spawn_session(prompt:, client_id:, mode: "assist", working_directory: nil, model_name: nil, state_dir: nil,
+                        parent_id: nil, messages: [], images_from: nil, preview: nil, title: nil, delegate: false,
+                        setup: SessionSetup.new, continues: nil, note: nil, note_source: "chi", before_spawn: nil,
+                        review: nil, verify_fresh: nil, scratch: false, session_id: nil)
         sd = state_dir || Session.default_state_dir
+        if session_id && (!Session.valid_id?(session_id) || Session.exist?(session_id, state_dir: sd))
+          raise ArgumentError, "invalid or existing session id #{session_id.inspect}"
+        end
+
         # The resolved ref is stored (a resumed session keeps its model when an
         # alias is retargeted), with the name as typed beside it. A wrong host
         # refuses here, before anything is saved or spawned; a model id the
@@ -79,8 +84,6 @@ module Samagotchi
           model_name: ref,
           model_typed: typed == ref ? nil : typed,
           working_directory: working_directory || Dir.pwd,
-          preloaded_memory_names: memories,
-          muted_memory_names: muted_memories,
           parent_id: parent_id,
           messages: messages,
           delegate: delegate,
@@ -89,6 +92,7 @@ module Samagotchi
           review: review,
           scratch: scratch
         )
+        session.id = session_id if session_id
         session.toolkit = Toolkit.configured(delegate: delegate) if session.toolkit.nil?
         session.toolkit_origin = setup.toolkit_origin || "default"
         # Saved before the worker starts: its first turn is already the requester's.
@@ -104,93 +108,115 @@ module Samagotchi
         # carry the id.
         unless prompt.to_s.strip.empty?
           session.first_turn_id = SecureRandom.uuid
-          session.first_turn_client_id = client_id
+          session.first_turn_client_id = client_id unless ClientId.human?(client_id)
         end
         # The worker takes last_prompt and clears it, and messages are saved at
         # the turn's end: until then this is the only preview a list has.
         session.first_preview = Session.preview_of(preview.to_s.strip.empty? ? prompt : preview)
         session.title = title
         session_dir = Session.session_dir(session.id, state_dir: sd)
-        unless session.messages.empty?
-          session.messages, dropped = ImageStore.copy_refs(session.messages, from: images_from, to: session_dir)
-          session.seed_images_dropped = dropped
+        # Starting before its file exists: the file is never there unowned and
+        # unmarked (a sweep would take a scratch session from under its
+        # worker). spawn_worker_for_session hands it to the starter.
+        StartingMarker.mark(session.id, state_dir: sd)
+        begin
+          unless session.messages.empty?
+            session.messages, dropped = ImageStore.copy_refs(session.messages, from: images_from, to: session_dir)
+            session.seed_images_dropped = dropped
+          end
+          setup_session_directory(session_dir, session, state_dir: sd)
+          SessionInbox.write_note(session.id, text: note, source: note_source, state_dir: sd) if note
+          before_spawn&.call(session)
+          spawn_worker_for_session(session, state_dir: sd)
+        rescue StandardError
+          # Nothing is starting: a caller's cleanup (a continue's failed link) may delete it.
+          StartingMarker.clear(session.id, state_dir: sd)
+          raise
         end
-        setup_session_directory(session_dir, session, state_dir: sd)
-        SessionInbox.write_note(session.id, text: note, source: note_source, state_dir: sd) if note
-        before_spawn&.call(session)
-        spawn_worker_for_session(session, state_dir: sd)
         session
       end
 
       # Ensure an existing session has a live worker process.
       # Returns the loaded session after state reconciliation.
       # @raise [OwnedByTUI] when the interactive TUI owns the session
+      # @yieldparam outcome [Symbol] what it did, when a block is given:
+      #   :woken (it spawned the worker), :live (a worker holds the session,
+      #   or took it since the first read) or :starting (another start is
+      #   pending, or won the claim)
       def resume_session(session_id, state_dir: nil)
+        session, outcome = resume(session_id, state_dir: state_dir)
+        yield outcome if block_given?
+        session
+      end
+
+      # resume_session's work, with its outcome.
+      # @return [Array(Session, Symbol)]
+      private def resume(session_id, state_dir:)
         sd = state_dir || Session.default_state_dir
         session = Session.load(session_id, state_dir: sd)
         # Its terminal's worker came from spawn_session; nothing wakes another.
         raise ScratchSession, session.id if session.scratch
-        return session if refuse_tui!(session.id, state_dir: sd)
 
-        # status is turn state, not liveness: a new worker runs no turn yet,
-        # and must not find the session stopped (it would exit): neither the
-        # marker nor a status field from before it.
-        # One this process just spawned may not hold the lock yet (a slow
-        # start, deliver_turn's second resume): it would only lose it.
-        return session if still_starting?(session.id, state_dir: sd)
-        # A continue moving delegates to it spawns its worker once they are
-        # moved; one spawned now (a moved child's ring) would race the move.
-        return session if StartingMarker.fresh?(session.id, state_dir: sd)
+        presence = WorkerPresence.read(session.id, state_dir: sd, owner: session_owner(session.id, state_dir: sd))
+        raise OwnedByTUI, session.id if presence.tui_owned?
+        return [session, :live] if presence.owned?
 
-        Session.clear_stopped(session.id, state_dir: sd)
-        session.status = Session::STATUS_IDLE
-        session.save(state_dir: sd)
-        spawn_worker_for_session(session, state_dir: sd)
-        session
-      end
+        # A worker being started for it, by this process or another (chi web,
+        # a `chi send`, a child's ring; a continue moving delegates to its new
+        # link), may not hold the lock yet: a second one would only lose it,
+        # or race the move (#resume_held_off?).
+        return [session, :starting] if presence.resume_held_off?
 
-      # A delegate child rang a parent that has no worker (ChildRing): start
-      # one, which runs a turn for the report. Not for a parent that is gone,
-      # stopped, archived, a scratch session, a one-turn run's, or a
-      # continue's new link still taking its delegates (its own spawn comes):
-      # the ring waits on disk for whoever runs it next (#resume_session
-      # itself would clear the stop). A stop landing between the check and
-      # the resume is a small window, left as is. An archived parent with a
-      # later link in its chain wakes that link instead: a ring a moved child
-      # wrote just as the continue moved it reaches the new link that way
-      # (ChildReports reads its predecessor's rings).
-      # @return [Symbol] :woken, or why not (:gone, :stopped, :archived,
-      #   :scratch, :owned, :starting)
-      def wake_for_report(session_id, state_dir: nil, forward: true)
-        wake_existing_session(session_id, state_dir: state_dir || Session.default_state_dir, forward: forward,
-                              event: :delegate_woke_parent)
+        claimed = claim_start(session.id, state_dir: sd, stopped: presence.stop_marked?)
+        return [session, claimed] unless claimed == :claimed
+
+        # The new worker must not find the session stopped (it would exit).
+        # Nothing else is written: the file is the worker's, which turns a
+        # status from before it idle under the lock (Worker#settle_stale_status).
+        begin
+          Session.clear_stopped(session.id, state_dir: sd)
+          spawn_worker_for_session(session, state_dir: sd)
+        rescue StandardError
+          # Its claim holds nothing now (a command or the options failed
+          # before Process.spawn): no 60 s hold-off.
+          StartingMarker.clear(session.id, state_dir: sd)
+          raise
+        end
+        [session, :woken]
       end
 
-      # A pushed context source asked for a wake: start an idle-exited
-      # session, but never clear a stop or forward an archived link.
+      # A ring waits in a parent's children/ (a failed continue's previous
+      # link got rings back): wake it as the ring's own delivery would
+      # (SessionInbox.wake :ring, its WakePolicy). Not for a parent that is
+      # gone, stopped (the stop marker; the file's status never decides),
+      # archived, a scratch session, a one-turn run's or owned, or a
+      # continue's new link still taking its delegates (its own spawn
+      # comes): the ring waits on disk for whoever runs it next. A stop
+      # landing between the check and the resume is a small window, left as
+      # is. An archived parent with a later link wakes that link (one hop:
+      # ChildReports reads its predecessor's rings, no further).
       # @return [Symbol] :woken, or why not (:gone, :stopped, :archived,
       #   :scratch, :owned, :starting)
-      def wake_for_context(session_id, state_dir: nil)
-        wake_existing_session(session_id, state_dir: state_dir || Session.default_state_dir, forward: false,
-                              event: :context_woke_session)
+      def wake_for_report(session_id, state_dir: nil)
+        sd = state_dir || Session.default_state_dir
+        outcome = SessionInbox.wake(:ring, session_id, state_dir: sd, owner: ->(id) { session_owner(id, state_dir: sd) }).wake
+        outcome == :live ? :owned : outcome
       end
 
-      private def wake_existing_session(session_id, state_dir:, forward:, event:)
-        return :gone unless Session.exist?(session_id, state_dir: state_dir)
-        return :owned if session_owner(session_id, state_dir: state_dir)
-        return :stopped if Session.stopped_marker?(session_id, state_dir: state_dir)
-
-        if ArchiveStore.archived?(Session.session_dir(session_id, state_dir: state_dir))
-          following = forward && SessionChain.next_of(session_id, state_dir: state_dir)
-          return following ? wake_existing_session(following, state_dir: state_dir, forward: false, event: event) : :archived
-        end
-        return :starting if StartingMarker.fresh?(session_id, state_dir: state_dir)
+      # Start a worker for a delivery whose WakePolicy decided :start
+      # (SessionInbox.deliver): resume_session, which keeps its own checks
+      # (a claim on the starting marker: of starts racing, one spawns),
+      # logged as +event+.
+      # @param event [Symbol] :delegate_woke_parent (a ring) or :context_woke_session (a push)
+      # @return [Symbol] :woken (this start spawned it), :starting (another
+      #   start won the claim, or is pending), :live (a worker took it since)
+      #   or :owned (a one-turn chi run or its terminal's scratch session did)
+      def start_for_inbox(session_id, state_dir:, event:)
+        # A stand-in resume_session that yields nothing counts as a start.
+        outcome = :woken
+        resume_session(session_id, state_dir: state_dir) { |done| outcome = done }
+        return outcome unless outcome == :woken
 
-        session = Session.load(session_id, state_dir: state_dir)
-        return :stopped if session.status == Session::STATUS_STOPPED
-        return :scratch if session.scratch
-
-        resume_session(session_id, state_dir: state_dir)
         details = event == :delegate_woke_parent ? { parent: session_id[0, 8] } : { sid: session_id[0, 8] }
         Log.info(:worker, event, **details)
         :woken
@@ -198,37 +224,46 @@ module Samagotchi
         :owned
       end
 
-      # A worker this process spawned for the session within
-      # WORKER_START_GRACE is still running, and no stop came since.
-      private def still_starting?(session_id, state_dir:)
-        return false if Session.stopped_marker?(session_id, state_dir: state_dir)
-
-        at, waiter = @spawns_mutex.synchronize { @spawns[[state_dir, session_id]] }
-        return false unless at && waiter.respond_to?(:alive?) && waiter.alive?
+      # Take the start for resume_session: the marker claimed (of resumes
+      # racing from any processes, one claims it), then the owner read
+      # again: a worker that took the lock since the first read cleared an
+      # earlier start's marker, and keeps the session. After a stop, a start
+      # marked before it is taken over (a stop cancels the hold-off), one a
+      # resume made after it holds (two sends after a stop start one worker).
+      # @return [Symbol] :claimed (this resume spawns), :starting (another
+      #   start holds the claim) or :live (a worker took the lock since)
+      private def claim_start(session_id, state_dir:, stopped:)
+        over_before = stopped ? stopped_at(session_id, state_dir) : nil
+        return :starting unless StartingMarker.claim(session_id, state_dir: state_dir, over_before: over_before)
+        return :claimed unless session_owner(session_id, state_dir: state_dir)
 
-        Process.clock_gettime(Process::CLOCK_MONOTONIC) - at < WORKER_START_GRACE
+        StartingMarker.clear(session_id, state_dir: state_dir)
+        :live
       end
 
-      private def note_spawn(session_id, state_dir:, waiter:)
-        @spawns_mutex.synchronize do
-          @spawns.delete_if { |_, (_, w)| !(w.respond_to?(:alive?) && w.alive?) }
-          @spawns[[state_dir, session_id]] = [Process.clock_gettime(Process::CLOCK_MONOTONIC), waiter]
+      # When the stop marker was written (Session.stopped_at); nil when it
+      # is gone (another resume cleared it: its start came after the stop,
+      # and holds).
+      private def stopped_at(session_id, state_dir) = Session.stopped_at(session_id, state_dir: state_dir)
+
+      private def note_starter(waiter)
+        @starters_mutex.synchronize do
+          @starters.select! { |w| w.respond_to?(:alive?) && w.alive? }
+          @starters << waiter
         end
       end
 
-      # Forget the spawns #still_starting? remembers (specs).
+      # Forget the starters this process spawned (specs).
       def forget_spawns
-        @spawns_mutex.synchronize { @spawns.clear }
+        @starters_mutex.synchronize { @starters.clear }
       end
 
       # Starter processes are children of chi web, unlike the detached workers
       # they launch; the web restart adopts these pids before replacing itself.
       def live_spawn_pids
-        @spawns_mutex.synchronize do
-          @spawns.delete_if { |_, (_, waiter)| !(waiter.respond_to?(:alive?) && waiter.alive?) }
-          @spawns.values.filter_map do |_, waiter|
-            waiter.pid if waiter.respond_to?(:pid) && waiter.alive?
-          end.uniq.sort
+        @starters_mutex.synchronize do
+          @starters.select! { |waiter| waiter.respond_to?(:alive?) && waiter.alive? }
+          @starters.filter_map { |waiter| waiter.pid if waiter.respond_to?(:pid) }.uniq.sort
         end
       end
 
@@ -248,14 +283,39 @@ module Samagotchi
         opts = spawn_options(session)
         env = opts.delete(:env)
         command = worker_command(session.id, state_dir: state_dir)
-        pid = env ? Process.spawn(env, *command, **opts) : Process.spawn(*command, **opts)
-        # Reaped when it exits: a long-lived spawner (chi web) keeps no zombies.
+        # Starting from here (WorkerPresence#starting?), for as long as this
+        # process runs, until the spawn below names its starter.
+        StartingMarker.mark(session.id, state_dir: state_dir)
+        begin
+          pid = env ? Process.spawn(env, *command, **opts) : Process.spawn(*command, **opts)
+        rescue StandardError
+          StartingMarker.clear(session.id, state_dir: state_dir)
+          raise
+        end
         # +pid+ is the worker's starter, which exits once the worker it forked
-        # holds the lock (WorkerBoot.detach_from_spawner): the lock's pid is the worker's.
-        note_spawn(session.id, state_dir: state_dir, waiter: Process.detach(pid))
+        # holds the lock (WorkerBoot.detach_from_spawner): the lock's pid is
+        # the worker's, and the marker holds exactly while the starter runs
+        # (most spawners exit right after this). The worker removes it once it
+        # holds the lock (Worker#start).
+        if starter_running?(pid)
+          StartingMarker.mark(session.id, state_dir: state_dir, pid: pid)
+          # Reaped when it exits: a long-lived spawner (chi web) keeps no zombies.
+          note_starter(Process.detach(pid))
+        else
+          # It failed to exec or boot: nothing is starting.
+          StartingMarker.clear(session.id, state_dir: state_dir)
+        end
         Log.info(:worker, "spawn", sid: session.id, child_pid: pid)
         pid
       end
+
+      # Whether the starter just spawned still runs (reaped here if it
+      # already exited). Not a child of this process: nothing to wait for.
+      private def starter_running?(pid)
+        Process.waitpid(pid, Process::WNOHANG).nil?
+      rescue Errno::ECHILD
+        false
+      end
     end
 
     extend SpawnPart

data/lib/samagotchi/session_manager.rb

@@ -59,6 +59,7 @@ require_relative "context_candidates_log"
 require_relative "session_chain"
 require_relative "child_move"
 require_relative "starting_marker"
+require_relative "worker_presence"
 require_relative "tools/task_runtime"
 
 module Samagotchi