samagotchi 0.159.0 → 0.161.0

data/lib/samagotchi/session_manager/delivery.rb

@@ -37,6 +37,17 @@ module Samagotchi
 
       def detail = ack["detail"]
 
+      # How a delivery that did not go in reads, as one line: the word for it
+      # (refused: a worker older than queueing would steer the running turn,
+      # or a remote refused it as chi send refuses it here; failed: it simply
+      # did not get in) and its reason. chi send prints it, and the delegate
+      # tool's follow-up says it after "Error: session <id8>".
+      # @return [String]
+      def refusal_text
+        refused = error == Delivery::QUEUE_UNAVAILABLE["error"] || DeliverResult::REMOTE_REFUSALS.include?(error)
+        "#{refused ? "refused" : "failed"}: #{detail || "could not queue it"}"
+      end
+
       # What the worker applied (a Bridge ack only): the delivery asked for,
       # or next_step for a --queue that found no turn running; nil from the
       # input file or a worker older than delivery.
@@ -49,6 +60,13 @@ module Samagotchi
       def cut_after = ack["cut_after"]
     end
 
+    # A remote's refusals of what this machine checks before sending (a
+    # parent's /continue yes, a loosening /parent-answers, images to a
+    # model that can't see them): its turn route's error codes. Named on
+    # DeliverResult, not inside its Data.define block, where a constant
+    # would land on SessionManager.
+    DeliverResult::REMOTE_REFUSALS = %w[parent_answers_refused parent_continue_refused no_vision].freeze
+
     # The unclaimed input files +author+ wrote.
     PARTICIPANT_QUEUE_LOCK = "participants.lock"
 
@@ -118,7 +136,10 @@ module Samagotchi
           link = WorkerLink.await(session_id, session_dir: session_dir, timeout: TURN_BRIDGE_WAIT)
           link.live? ? link.client(session_id) : nil
         end
-        manager.resume_session(session_id, state_dir: state_dir)
+        # Its refusals (no session, a one-turn run's, a scratch one) are
+        # resume_session's to raise, in their one wording.
+        start_for_send(manager, session_id, state_dir, before: true)
+        enqueued_id = SessionInbox.new_enqueued_id
         if (client = bridge.call)
           if (missing = anchors.present.find { |anchor| !worker_has?(client, anchor.class::FEATURE) })
             return DeliverResult.new(status: :refused, code: 409, ack: missing.class::UNAVAILABLE)
@@ -130,7 +151,7 @@ module Samagotchi
           return review_unavailable if Steer.participant_author?(author) && !worker_has?(client, Delivery::REVIEW_FEATURE)
 
           begin
-            options = { prompt: prompt, client_id: client_id }
+            options = { prompt: prompt, client_id: client_id, enqueued_id: enqueued_id }
             options[:images] = images unless images.empty?
             # An old worker ignores the field: it cuts anyway (its own default).
             options[:delivery] = delivery unless Delivery.next_step?(delivery)
@@ -164,12 +185,12 @@ module Samagotchi
         # still holding the session with its Bridge gone (exiting) may be an
         # older one, which would run it without the review boundary.
         # (A one-turn chi run holding it is refused as for anyone, below.)
-        if Steer.participant_author?(author) && (holder = delivery_owner(manager, session_id, state_dir)) && !holder.tui?
+        if Steer.participant_author?(author) && (holder = delivery_presence(manager, session_id, state_dir)).owned? &&
+           !holder.tui_owned?
           return DeliverResult.new(status: :timeout, via: :file,
                                    ack: { "error" => "worker_timeout",
                                           "detail" => "the session's worker is exiting, so the message was not sent; send it again" })
         end
-        enqueued_id = SecureRandom.uuid
         input = { prompt: prompt, client_id: client_id, enqueued_id: enqueued_id, state_dir: state_dir }
         input[:images] = images unless images.empty?
         input[:delivery] = delivery unless Delivery.next_step?(delivery)
@@ -189,21 +210,51 @@ module Samagotchi
         end
         return DeliverResult.new(status: :failed) unless path
 
-        owner = delivery_owner(manager, session_id, state_dir)
         # A TUI that took the session between the resume and the write never
         # reads input files; a later worker would replay this one.
-        if owner&.tui?
-          FileUtils.rm_f(path) if path.is_a?(String)
-          raise OwnedByTUI, session_id
-        end
         # A worker that idle-exited since the resume never reads it either:
         # wake a new one. (The exiting worker also looks for input it left;
         # one the first resume spawned that is still starting is left be.)
-        manager.resume_session(session_id, state_dir: state_dir) if owner.nil?
+        start_for_send(manager, session_id, state_dir, before: false) do
+          FileUtils.rm_f(path) if path.is_a?(String)
+          raise OwnedByTUI, session_id
+        end
         DeliverResult.new(status: :accepted, via: :file,
                           ack: { "status" => "accepted", "enqueued_id" => enqueued_id, "session_id" => session_id })
       end
 
+      # deliver_turn's two liveness checks, on the session inbox's send row
+      # (SessionInbox::WakePolicy: send un-stops and starts an archived
+      # link; it is pinned to the id it names): resume_session unless a
+      # worker holds the session or one is being started. Its refusals (no
+      # session, a one-turn run's, a scratch one) are resume_session's to
+      # raise, in their one wording: +before+ the write, a session it would
+      # not resume is checked for them (WakePolicy#refusal: the file read
+      # only then, resume_session reads it otherwise); after it, a one-turn
+      # run that took the session since runs the block, and a worker that
+      # is leaving (OwnerLock::Owner#leaving?: its Bridge closed, the lock
+      # held for its last work) has its input served by its own respawn
+      # (WorkerBoot's leave): a stop that landed before this send is
+      # cleared, as resume_session clears it (a send un-stops); one that
+      # lands after it still wins.
+      private def start_for_send(manager, session_id, state_dir, before:)
+        presence = delivery_presence(manager, session_id, state_dir)
+        policy = SessionInbox::WakePolicy.for(:send)
+        outcome = policy.decide(presence)
+        yield if outcome == :owned && block_given?
+        if !before && outcome == :live && presence.stop_marked? && presence.owner&.leaving?
+          Session.clear_stopped(session_id, state_dir: state_dir || Session.default_state_dir)
+          Log.info(:worker, "send_unstopped_leaving", sid: session_id[0, 8])
+        end
+        if %i[live starting].include?(outcome)
+          return unless before
+
+          load = -> { Session.load(session_id, state_dir: state_dir || Session.default_state_dir) }
+          return unless policy.refusal(presence, load: load)
+        end
+        manager.resume_session(session_id, state_dir: state_dir)
+      end
+
       # Whether the worker +client+ talks to names +feature+: the features of
       # the sidecar the client was made from (the Live link WorkerLink.await
       # found), not a second read of bridge.json.
@@ -244,8 +295,11 @@ module Samagotchi
                                  "detail" => "the session's worker did not answer, so the message was not sent" })
       end
 
-      private def delivery_owner(manager, session_id, state_dir)
-        Degrade.call(:worker, "owner_unreadable", sid: session_id) { manager.session_owner(session_id, state_dir: state_dir) }
+      # The session's WorkerPresence, its owner read through +manager+ (a
+      # stand-in's #session_owner is optional: then none).
+      private def delivery_presence(manager, session_id, state_dir)
+        owner = Degrade.call(:worker, "owner_unreadable", sid: session_id) { manager.session_owner(session_id, state_dir: state_dir) }
+        WorkerPresence.read(session_id, state_dir: state_dir, owner: owner)
       end
     end
 

data/lib/samagotchi/session_manager/emptiness.rb

@@ -3,6 +3,7 @@
 require_relative "../analytics_file"
 require_relative "../session_inbox"
 require_relative "../worker_link"
+require_relative "../starting_marker"
 
 module Samagotchi
   class SessionManager
@@ -12,7 +13,9 @@ module Samagotchi
     # nothing writes or reads it now, but a folder from before still has one
     # and is still empty. A turn's drawing (Bridge::TurnDrawings::FILE)
     # exists only after a turn that kept messages: never a skeleton file.
-    EMPTY_SKELETON_FILES = ["pid", "owner.lock", WorkerLink::FILE, AnalyticsFile::NAME].freeze
+    # The starting marker is one too: a start left one behind; a fresh one
+    # keeps the session from the deleters itself.
+    EMPTY_SKELETON_FILES = ["pid", "owner.lock", WorkerLink::FILE, AnalyticsFile::NAME, StartingMarker::FILE].freeze
     EMPTY_DIRS = [SessionInbox::INPUT_DIR, SessionInbox::NOTES_DIR, "images"].freeze
     EMPTY_SKELETON_DIRS = (EMPTY_DIRS + [SessionInbox::OUTPUT_DIR]).freeze
 
@@ -94,7 +97,9 @@ module Samagotchi
           elsif name == "cards.json"
             cards_empty?(path)
           else
-            EMPTY_SKELETON_FILES.include?(name)
+            # A starting claim's temp files a kill left (StartingMarker.claim:
+            # starting.<pid>.<hex>, starting.old.<pid>.<hex>) are the marker's.
+            EMPTY_SKELETON_FILES.include?(name) || name.start_with?("#{StartingMarker::FILE}.")
           end
         end
       end

data/lib/samagotchi/session_manager/lifecycle.rb

@@ -220,7 +220,8 @@ module Samagotchi
         # A stop drops the reminders: a restart's hand-off no successor took
         # yet must not bring them back on the next resume.
         ReminderCarry.remove(Session.session_dir(session_id, state_dir: sd))
-        pid = owner&.pid.to_i
+        # A deleter holding the lock (chi web, the CLI) is no worker to stop.
+        pid = owner&.deleting? ? 0 : owner&.pid.to_i
         begin
           Process.kill("TERM", pid) if pid && pid > 0
         rescue Errno::ESRCH

data/lib/samagotchi/session_manager/listing.rb

@@ -11,7 +11,9 @@ module Samagotchi
       # +include_archived+: archived sessions too (Session#archived says which).
       def list_sessions(state_dir: nil, sort: "updated_at", order: "desc", limit: nil, offset: 0, project_root: nil,
                         include_archived: false)
-        Session.list(state_dir: state_dir || Session.default_state_dir, sort: sort, order: order, limit: limit,
+        sd = state_dir || Session.default_state_dir
+        SessionContinue.repair_interrupted(state_dir: sd)
+        Session.list(state_dir: sd, sort: sort, order: order, limit: limit,
                      offset: offset, project_root: project_root, include_archived: include_archived)
       end
 
@@ -34,6 +36,7 @@ module Samagotchi
       def session_rows(live: false, cwd: nil, limit: nil, include_tests: true, exclude: nil, state_dir: nil,
                        project_root: nil, include_archived: false, sort: nil, order: nil)
         sd = state_dir || Session.default_state_dir
+        SessionContinue.repair_interrupted(state_dir: sd)
         root = cwd && folder_path(cwd)
         roots = {}
         # One registry per listing, built only when a row has a saved context

data/lib/samagotchi/session_manager/removal.rb

@@ -6,12 +6,20 @@ module Samagotchi
     # reason: :worker_running (not asked to stop it) or :still_stopping
     # (stopped, but the worker outlived the wait).
     class DeleteRefused < StandardError
-      attr_reader :session_id, :reason
+      attr_reader :session_id, :reason, :schedule_id
 
-      def initialize(session_id, reason)
+      def initialize(session_id, reason, message: nil, schedule_id: nil)
         @session_id = session_id
         @reason = reason
-        super(reason == :still_stopping ? "session #{session_id}'s worker is still shutting down" : "session #{session_id}'s worker is running")
+        @schedule_id = schedule_id
+        message ||= case reason
+                    when :still_stopping then "session #{session_id}'s worker is still shutting down"
+                    when :scheduled
+                      "this session is the target of an enabled schedule" \
+                        "#{"; cancel it with chi schedule cancel #{schedule_id}" if schedule_id}"
+                    else "session #{session_id}'s worker is running"
+                    end
+        super(message)
       end
     end
 
@@ -37,6 +45,8 @@ module Samagotchi
         when :queued_child then "its delegate #{@busy_id[0, 8]} has a prompt queued, which a resume would run; let it run first"
         when :offer then "the step-limit question waits for an answer; answer Continue or Stop first"
         when :offer_child then "its delegate #{@busy_id[0, 8]} waits for an answer to its step-limit question; answer it first"
+        when :starting then "its worker is starting; try again in a moment"
+        when :starting_child then "its delegate #{@busy_id[0, 8]}'s worker is starting; try again in a moment"
         else "a turn is running; wait for it or cancel it first"
         end
       end
@@ -47,6 +57,12 @@ module Samagotchi
     # workers stopped first; discarded: empty sessions its stopping worker deleted.
     ArchiveResult = Data.define(:id, :archived, :stopped, :discarded)
 
+    # Every store with session-scoped state. Deletion, retention and failed
+    # continuation cleanup all pass through this registry.
+    SESSION_STATE_STORES = %i[
+      session_files plugin_session_state context_sources participants system_one_runs context_candidates_log
+    ].freeze
+
     # Removal: archive, unarchive and delete a session (an archive takes its
     # delegates too), and the retention sweep's entry. Calls LifecyclePart's
     # stop_session, ListingPart's descendant_ids and the facade's ownership
@@ -87,23 +103,29 @@ module Samagotchi
         raise ArchiveRefused.new(id, :scratch) if Session.load(id, state_dir: state_dir).scratch
 
         tree = [id, *descendant_ids(id, state_dir, except: except)]
-        owners = tree.to_h { |sid| [sid, session_owner(sid, state_dir: state_dir)] }
+        presences = tree.to_h do |sid|
+          [sid, WorkerPresence.read(sid, state_dir: state_dir, owner: session_owner(sid, state_dir: state_dir))]
+        end
         tree.each do |sid|
-          refusal = archive_refusal(sid, owners[sid], state_dir)
+          refusal = archive_refusal(sid, presences[sid], state_dir)
           raise ArchiveRefused.new(id, sid == id ? refusal : :"#{refusal}_child", busy_id: sid) if refusal
         end
-        [tree, owners]
+        [tree, presences.transform_values(&:owner)]
       end
 
-      # What keeps +sid+ (in an archive's tree) from being archived: :busy (a
+      # What keeps +sid+ (in an archive's tree) from being archived:
+      # :starting (a worker is being started for it, holding no lock yet: a
+      # continue's new link, a resume), :busy (a
       # turn runs), :queued (input/ holds a prompt, which outlives the stop and
       # runs on a resume, live worker or not), :offer (a live worker's continue
       # offer, which the stop would drop), or nil.
+      # The status is read only while someone owns it (+presence+'s owner).
       # @raise [OwnedByTUI] a one-turn chi run owns it
-      private def archive_refusal(sid, owner, state_dir)
-        raise OwnedByTUI, sid if owner&.tui?
+      private def archive_refusal(sid, presence, state_dir)
+        raise OwnedByTUI, sid if presence.tui_owned?
+        return :starting if presence.starting? && !presence.owned?
 
-        session = owner && Session.load(sid, state_dir: state_dir)
+        session = presence.owned? ? Session.load(sid, state_dir: state_dir) : nil
         return :busy if session&.status == Session::STATUS_RUNNING
         return :queued unless SessionInbox.find_new_input_files(Session.session_dir(sid, state_dir: state_dir)).empty?
 
@@ -135,12 +157,14 @@ module Samagotchi
       # @return [Hash] {id:, removed: [paths], stopped: whether a worker was stopped}
       # @raise [ArgumentError] unknown id (Session::AmbiguousId for a prefix of several)
       # @raise [OwnedByTUI] a one-turn chi run owns it (never stopped from here)
-      # @raise [DeleteRefused] a worker owns it and +stop+ is false, or it
-      #   outlived the wait
+      # @raise [DeleteRefused] a worker owns it or is being started for it
+      #   (the starting marker) and +stop+ is false, or it outlived the wait
       # @param purge_runs [Boolean] also delete System 1's saved runs of it and
       #   its candidate-log lines; a session a worker left empty has none worth
       #   the read
-      def delete_session(id_or_prefix, state_dir: nil, stop: false, wait: STOP_WAIT, purge_runs: true)
+      # @param allow_scheduled [Boolean] internal rollback of a failed continue
+      def delete_session(id_or_prefix, state_dir: nil, stop: false, wait: STOP_WAIT, purge_runs: true,
+                         allow_scheduled: false)
         sd = state_dir || Session.default_state_dir
         given = id_or_prefix.to_s
         # Only a plain id: anything else could name a path outside the state dir.
@@ -152,12 +176,81 @@ module Samagotchi
         # A folder with no file left is still deleted (not resolve!).
         raise Session::NotFound, given unless File.exist?(path) || Dir.exist?(dir)
 
-        owner = refuse_tui!(id, state_dir: sd)
-        if owner
+        unless allow_scheduled
+          require_relative "../schedules"
+          schedule = Schedules.new(state_dir: sd).protecting(id)
+          if schedule
+            raise DeleteRefused.new(id, :scheduled, schedule_id: schedule.id)
+          end
+        end
+
+        presence = WorkerPresence.read(id, state_dir: sd, owner: session_owner(id, state_dir: sd))
+        raise OwnedByTUI, id if presence.tui_owned?
+
+        held = presence.owned? || presence.starting?
+        if held
           raise DeleteRefused.new(id, :worker_running) unless stop
-          raise DeleteRefused.new(id, :still_stopping) unless stop_session(id, state_dir: sd, wait: wait)
+          raise DeleteRefused.new(id, :still_stopping) unless stop_held(id, state_dir: sd, wait: wait)
         end
 
+        removed = remove_files(id, path, dir, state_dir: sd, purge_runs: purge_runs)
+        { id: id, removed: removed, stopped: held }
+      end
+
+      # Remove every store with state scoped to +id+. The file and sidecar
+      # directory keep the deleting owner lock; the other stores have their
+      # own locking where they need it.
+      # @return [Array<String>] the session file, sidecar and context paths removed
+      def remove_files(id, path, dir, state_dir:, purge_runs: true)
+        removed = []
+        SESSION_STATE_STORES.each do |store|
+          case store
+          when :session_files
+            removed.concat(remove_session_files(id, path, dir, state_dir: state_dir))
+          when :plugin_session_state
+            PluginSessionState.remove(id)
+          when :context_sources
+            removed.concat(ContextSources.remove_session(id, state_dir: state_dir))
+          when :participants
+            Participants.end_sessions([id], by: "delete", state_dir: state_dir)
+          when :system_one_runs
+            next
+          when :context_candidates_log
+            next
+          else
+            raise ArgumentError, "no cleanup for session store #{store.inspect}"
+          end
+        end
+        purge_removal_stores([id], state_dir: state_dir) if purge_runs
+        removed
+      end
+
+      # The session-scoped logs are expensive to scan; callers removing a
+      # batch purge them once after the session files are gone.
+      def purge_removal_stores(ids, state_dir:)
+        return if ids.empty?
+
+        purge_session_runs(ids, state_dir)
+        ContextCandidatesLog.purge_sessions(ids, state_dir: state_dir)
+      end
+
+      # The session's file and directory, removed while this process holds
+      # its owner lock (kind deleting, which a stop never signals), so no
+      # worker takes the session in the middle; the starting marker is read
+      # again under it (a resume that claimed the start since the caller's
+      # check keeps the session; one after it sees the lock held). Taken
+      # without waiting: one that took it since the check keeps the session.
+      # The lock file goes with the directory; a worker starting after that
+      # makes a new one, and finds no session (Session.load). The delete and
+      # the retention sweep and failed-continue cleanup use #remove_files.
+      # @return [Array<String>] the paths removed
+      # @raise [DeleteRefused] a worker took the lock, or a start was
+      #   claimed, after the caller's check
+      def remove_session_files(id, path, dir, state_dir:)
+        lock = Dir.exist?(dir) ? OwnerLock.acquire(dir, kind: OwnerLock::DELETING, wait: 0) : nil
+        raise DeleteRefused.new(id, :worker_running) if Dir.exist?(dir) && lock.nil?
+        raise DeleteRefused.new(id, :worker_running) if StartingMarker.fresh?(id, state_dir: state_dir)
+
         removed = []
         if File.exist?(path)
           FileUtils.rm_f(path)
@@ -167,21 +260,32 @@ module Samagotchi
           FileUtils.rm_rf(dir)
           removed << dir
         end
-        PluginSessionState.remove(id)
-        removed.concat(ContextSources.remove_session(id, state_dir: sd))
-        Participants.end_sessions([id], by: "delete", state_dir: sd)
-        if purge_runs
-          purge_session_runs(id, sd)
-          ContextCandidatesLog.purge_session(id, state_dir: sd)
+        removed
+      ensure
+        lock&.release
+      end
+
+      # Stop the worker that owns +id+ or is being started for it (the stop
+      # marker: one that takes the lock leaves at once), and wait for both to
+      # be gone.
+      # @return [Boolean] whether no owner and no start was left within +wait+
+      private def stop_held(id, state_dir:, wait:)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + wait.to_f
+        return false unless stop_session(id, state_dir: state_dir, wait: wait)
+
+        while StartingMarker.fresh?(id, state_dir: state_dir) || session_owner(id, state_dir: state_dir)
+          return false if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
+          sleep(OwnerLock::RETRY_INTERVAL)
         end
-        { id: id, removed: removed, stopped: !owner.nil? }
+        true
       end
 
       # Never a reason a delete fails: one warn, and the sweep ages them out.
-      private def purge_session_runs(id, state_dir)
-        SystemOne::Runs.purge_session(id, sessions_dir: state_dir)
+      private def purge_session_runs(ids, state_dir)
+        SystemOne::Runs.purge_sessions(ids, sessions_dir: state_dir)
       rescue StandardError => e
-        Log.warn(:worker, "session_runs_purge_failed", sid: id, error: e.class.name, msg: e.message.to_s[0, 200])
+        Log.warn(:worker, "session_runs_purge_failed", sid: ids.first, error: e.class.name, msg: e.message.to_s[0, 200])
       end
     end