samagotchi 0.60.0 → 0.62.0

data/lib/samagotchi/session_manager.rb

@@ -22,6 +22,7 @@ require_relative "turn_note"
 require_relative "owner_lock"
 require_relative "bridge_client"
 require_relative "worker_sidecar"
+require_relative "worker_signals"
 require_relative "log"
 require_relative "log_path"
 require_relative "model_profile"
@@ -210,7 +211,8 @@ module Samagotchi
     # wakes the worker picks which config.yml it reads (docs/guardrails.md).
     private_class_method def self.spawn_options(session)
       # Own process group: workers outlive `chi web`, and a Ctrl-C in its
-      # terminal must not reach them.
+      # terminal must not reach them (the worker then leaves the process
+      # tree too: WORKER_DETACH_ENV).
       opts = { out: File::NULL, err: File::NULL, pgroup: true }
       # The worker's tools (and `!cmd`) run in the session's directory, not
       # in the cwd of whoever woke it (`chi web`, another terminal).
@@ -253,6 +255,10 @@ module Samagotchi
       SESSION_MARKER_ENV.each { |key| child_env[key] = nil }
       # A smoke's fake "newest installed" is for this process's notices only.
       child_env[InstalledVersions::ENV_KEY] = nil
+      # A worker process: run_session_loop traps signals, and leaves this
+      # process's tree.
+      child_env[WORKER_PROCESS_ENV] = "1"
+      child_env[WORKER_DETACH_ENV] = "1"
       # A gem-mode worker activates the newest installed chi itself: under
       # `bundle exec` (a Gemfile that has samagotchi) Bundler's RUBYOPT would
       # hold it to the bundle's version, and its badge would never clear.
@@ -688,8 +694,8 @@ module Samagotchi
     # so a resume right after spawns a fresh worker instead of finding the
     # dying one.
     # A turn still running is canceled first (STOP_CANCEL_WAIT), as the
-    # web's Cancel does: its Engine saves the prompt, which the TERM alone
-    # would lose with the worker.
+    # web's Cancel does: its worker saves it. The TERM stops (and saves)
+    # what still runs (WorkerSignals), with a short grace after the marker.
     # @return [Boolean, nil] with +wait+: whether the owner was gone in time
     # @raise [OwnedByTUI] when the interactive TUI owns the session
     def self.stop_session(session_id, state_dir: nil, wait: nil)
@@ -930,7 +936,8 @@ module Samagotchi
     # @param idle_exit_minutes [Numeric, nil] nil: session.idle_exit_minutes
     # @param poll_interval [Numeric, nil] seconds between the loop's fallback
     #   ticks (nil: Worker::FALLBACK_TICK_SECONDS); queued turns wake it at once
-    # A stopped session's worker exits 0 and a crashed one 1, there.
+    # A stopped session's worker exits 0 (a signalled one too) and a
+    # crashed one 1, there.
     # @return [Symbol] :idle_exit, :exit_requested or :restart
     def self.run_session_loop(session_id, state_dir: nil, owner_wait: OwnerLock::DEFAULT_WAIT,
                               idle_exit_minutes: nil, poll_interval: nil)
@@ -942,6 +949,12 @@ module Samagotchi
       Log.configure(stderr: false)
       Log.session_id = session_id
       session_dir = Session.session_dir(session_id, state_dir: sd)
+      # Read (and taken out of ENV, so nothing the worker runs inherits it)
+      # before anything else: set by spawn_options for a spawned worker.
+      spawned = ENV.delete(WORKER_PROCESS_ENV) == "1"
+      # Out of the spawner's process tree before the lock (it records this
+      # process's pid) and before any thread starts.
+      detach_from_spawner(session_dir) if ENV.delete(WORKER_DETACH_ENV) == "1"
       # Kept in a class ivar so the lock's File lives as long as the worker.
       @owner_lock = OwnerLock.acquire(session_dir, kind: "worker", wait: owner_wait)
       exit(0) unless @owner_lock
@@ -949,8 +962,11 @@ module Samagotchi
       fallback = ENV.delete(BOOT_FALLBACK_ENV)
       Log.info(:worker, "start", cwd: Dir.pwd, version: VERSION)
       Log.warn(:worker, "boot_fallback", version: VERSION, error: fallback) if fallback
+      # A spawned worker stops its turn on TERM/HUP/INT and saves it; a
+      # loop run in-process (specs) traps nothing.
+      signals = spawned ? WorkerSignals.new.install! : WorkerSignals::NONE
       worker = Worker.new(session_id: session_id, state_dir: sd, session_dir: session_dir,
-                          idle_exit_minutes: idle_exit_minutes, poll_interval: poll_interval)
+                          idle_exit_minutes: idle_exit_minutes, poll_interval: poll_interval, signals: signals)
       result = begin
         worker.run
       rescue StandardError, ScriptError => e
@@ -959,11 +975,19 @@ module Samagotchi
         raise
       ensure
         @owner_lock.release
+        # A signal from here on ends the process as Ruby's default does.
+        signals.restore!
       end
       Log.info(:worker, "stop", reason: result)
       # A stopped session is left as it is: no resume for input that came
-      # in, no discard.
+      # in, no discard. One whose worker a signal stopped isn't discarded
+      # either; input that came in (queued behind the turn it stopped)
+      # wakes a new worker, as after an idle exit.
       exit(0) if result == :stopped
+      if result == :signal
+        resume_session(session_id, state_dir: sd) unless SessionInbox.find_new_input_files(session_dir).empty?
+        exit(0)
+      end
       exit(1) if result == :crashed
       # Only after the release: a writer that still saw this worker as the
       # owner may have queued input since the last check. Either it finds no
@@ -981,6 +1005,31 @@ module Samagotchi
       result
     end
 
+    # A spawned worker leaves its spawner's process tree: a killer that
+    # walks the tree by parent pid (a terminal closing, a tool's teardown
+    # of a `chi send --wait` it ran in the background) no longer reaches it.
+    # This process (the spawner's child, which its Process.detach waits on)
+    # forks the worker, which starts a session of its own (setsid: no
+    # controlling terminal, no SIGHUP from one) and returns to run the loop;
+    # this one waits until the worker holds the session's OwnerLock, or has
+    # exited, or WORKER_START_GRACE passed, and exits: the spawner's waiter
+    # stays alive exactly while the worker starts (#still_starting?). The
+    # worker is reparented to init (launchd), which reaps it.
+    private_class_method def self.detach_from_spawner(session_dir)
+      worker = Process.fork
+      unless worker
+        Process.setsid
+        return
+      end
+
+      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + WORKER_START_GRACE
+      until Process.waitpid(worker, Process::WNOHANG) || OwnerLock.owner(session_dir)&.pid == worker ||
+            Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+        sleep(OwnerLock::RETRY_INTERVAL)
+      end
+      Process.exit!(0)
+    end
+
     # Delete a session its worker left empty. Checked again now the lock is
     # free: a note or input may have come in since the worker looked, and a
     # worker woken meanwhile owns it (delete_session refuses). A `chi send`
@@ -1024,6 +1073,13 @@ module Samagotchi
     # format are read by a NEWER chi than the one writing them. Change them
     # only additively (spec/worker_contract_spec.rb).
     BOOT_FALLBACK_ENV = "SAMAGOTCHI_BOOT_FALLBACK"
+    # "1" in a worker spawn_options started: it traps signals (WorkerSignals).
+    # run_session_loop deletes it from ENV as it reads it.
+    WORKER_PROCESS_ENV = "SAMAGOTCHI_WORKER_PROCESS"
+    # "1" in a worker spawn_options started: run_session_loop leaves the
+    # spawner's process tree first (detach_from_spawner); deleted from ENV
+    # as it is read, so an rspec run by the worker's execute never forks.
+    WORKER_DETACH_ENV = "SAMAGOTCHI_WORKER_DETACH"
 
     def self.worker_command(session_id, state_dir:, gem_spec: InstalledGem.spec)
       boot = "require 'samagotchi/session_manager'; " \
@@ -1046,6 +1102,8 @@ module Samagotchi
       command = worker_command(session.id, state_dir: state_dir)
       pid = env ? Process.spawn(env, *command, **opts) : Process.spawn(*command, **opts)
       # Reaped when it exits: a long-lived spawner (chi web) keeps no zombies.
+      # +pid+ is the worker's starter, which exits once the worker it forked
+      # holds the lock (detach_from_spawner): the lock's pid is the worker's.
       note_spawn(session.id, state_dir: state_dir, waiter: Process.detach(pid))
       Log.info(:worker, "spawn", sid: session.id, child_pid: pid)
       pid

data/lib/samagotchi/session_metrics.rb

@@ -416,10 +416,20 @@ module Samagotchi
     # analytics.json in the session directory. Atomic write; failures are
     # swallowed (analytics must never break the running session).
     # @param state_dir [String, nil]
+    # @param running_turn [Boolean] also save the running turn's record so
+    #   far (a worker's turn checkpoint): as the turn a worker killed now
+    #   would leave, canceled with reason worker_stopped (TurnNote's), up to
+    #   its completed_at. The next collector loads it with the others; the
+    #   turn's own end writes over it. Only the file has it.
     # @return [Boolean] true on success
-    def persist(state_dir: nil)
+    def persist(state_dir: nil, running_turn: false)
       sid, data, turn_count = @mutex.synchronize do
-        [@session_id, build_snapshot(:all).merge(active_turn: nil, active_tools: []), @turn_records.size]
+        data = build_snapshot(:all).merge(active_turn: nil, active_tools: [])
+        if running_turn && @turn
+          data[:turn_records] << running_turn_record
+          data[:cancellations] += 1
+        end
+        [@session_id, data, @turn_records.size]
       end
       return false if sid.nil? || sid.to_s.empty?
 
@@ -611,6 +621,18 @@ module Samagotchi
       end
     end
 
+    # The running turn's record so far, as end_turn would make it for a
+    # turn a worker died in (#persist's running_turn). Its open generation
+    # (none at an iteration boundary) and running tool calls are left out.
+    # Caller holds the mutex.
+    def running_turn_record
+      {
+        id: @turn.id, status: "canceled", started_at: @turn.started_at, completed_at: now.iso8601(3),
+        duration_ms: elapsed_ms(@turn.started_monotonic), cancellation_reason: "worker_stopped",
+        continue: @turn.continue, resumed: @turn.resumed
+      }.merge(turn_token_fields(@turn))
+    end
+
     # What turn_failed's line is made of (format.js failedTurnText): a
     # provider error's summary, else the message (capped: a server's error
     # page can be long) and class, and the steps that stayed (kept_steps).

data/lib/samagotchi/terminal_ui/formatting.rb

@@ -363,7 +363,7 @@ module Samagotchi
 
       # A cancel's reason as the UIs name it (anything else as it is);
       # spec/shared/labels_matrix.json pins it for both.
-      CANCEL_REASONS = { "ctrl_c" => "Ctrl-C", "user" => "stopped", "hook" => "by a hook" }.freeze
+      CANCEL_REASONS = { "ctrl_c" => "Ctrl-C", "user" => "stopped", "hook" => "by a hook", "worker_stopped" => "worker stopped" }.freeze
       # A canceled turn's mark: a stop someone chose (Ctrl-C, Stop, a hook),
       # not an error's ✕ (timing.js STOP_MARK; the labels matrix pins it).
       STOP_MARK = "■"

data/lib/samagotchi/terminal_ui.rb

@@ -9,6 +9,7 @@ require_relative "client_id"
 require_relative "iteration_limit"
 require_relative "model_profile"
 require_relative "turn_note"
+require_relative "turn_checkpoint"
 require_relative "config"
 require_relative "context_note"
 require_relative "cancellation_controller"
@@ -277,6 +278,9 @@ module Samagotchi
         session.save if @resume_session
       end
       claim_session!(session.id) unless @owner_lock
+      # A turn a worker died in (TurnCheckpoint): ended as canceled, with its
+      # note, before this REPL's first turn appends after it.
+      session.save if @resume_session && TurnCheckpoint.settle(session)
       keep_cards(session)
       # Saved at once, so a process killed before its first turn leaves a
       # file the sweep knows to delete.
@@ -881,12 +885,35 @@ module Samagotchi
       end
 
       # The REPL keeps the kernel's conversation as-is (no [No response]
-      # placeholder), so /continue resumes from the tool results.
-      session.messages = result.conversation
+      # placeholder), so /continue resumes from the tool results; the
+      # answer keeps what the after_turn hooks stored on it.
+      session.messages = with_answer_display(result.conversation, session.messages)
       @turn_flow.after_turn(result, continue: continue)
       save_session(session)
     end
 
+    # The answer's display keys (Engine#store_answer_display put them on the
+    # session's last message) carried onto +conversation+'s last message,
+    # when that is the same answer; +conversation+ as it is otherwise.
+    ANSWER_DISPLAY_KEYS = [AnswerDisplay::KEY, AnswerRefs::KEY].freeze
+    private_constant :ANSWER_DISPLAY_KEYS
+
+    def with_answer_display(conversation, stored)
+      answer = Array(conversation).last
+      kept = Array(stored).last
+      return conversation unless answer.is_a?(Hash) && kept.is_a?(Hash) && message_field(answer, :role).to_s == "model"
+      return conversation unless message_field(kept, :role).to_s == "model" &&
+                                 message_field(kept, :content) == message_field(answer, :content)
+
+      extra = ANSWER_DISPLAY_KEYS.each_with_object({}) do |key, found|
+        value = message_field(kept, key)
+        found[key] = value unless value.nil?
+      end
+      extra.empty? ? conversation : conversation[0...-1] + [answer.merge(extra)]
+    end
+
+    def message_field(message, key) = message.key?(key) ? message[key] : message[key.to_s]
+
     def save_session(session)
       @engine.store_model!(session)
       session.save
@@ -966,7 +993,7 @@ module Samagotchi
       end
       canceled = result&.canceled?
       if result && !canceled
-        session.messages = result.conversation if result.conversation.is_a?(Array)
+        session.messages = with_answer_display(result.conversation, session.messages) if result.conversation.is_a?(Array)
         @engine.store_model!(session)
         session.save
       end

data/lib/samagotchi/tool_activity.rb

@@ -211,12 +211,16 @@ module Samagotchi
     # "stopped" for a
     # task_wait the user's Stop ended (the task itself runs on), for one
     # whose task was stopped (the stop-task button, or the model's task_stop),
-    # and for a command the user's Stop killed or kept from starting: a
-    # cancel, not a failure (its text still starts "Error:" for the model).
+    # and for a command the user's Stop (or a signal to chi) killed or kept
+    # from starting: a cancel, not a failure (its text still starts "Error:"
+    # for the model).
     def tool_activity_status(result, tool_name = nil, command: nil)
       text = result.to_s
-      return "stopped" if text == Tools::Execute::NOT_RUN_ON_STOP
-      return "stopped" if tool_name == Tools::Execute::NAME && text.start_with?(Tools::Execute::STOPPED_BY_USER)
+      return "stopped" if [Tools::Execute::NOT_RUN_ON_STOP, Tools::Execute::NOT_RUN_ON_SIGNAL].include?(text)
+      if tool_name == Tools::Execute::NAME &&
+         text.start_with?(Tools::Execute::STOPPED_BY_USER, Tools::Execute::STOPPED_BY_SIGNAL)
+        return "stopped"
+      end
       return "error" if text.start_with?("Error:")
       # Still running, as a task: its output tail may hold any "exit: N".
       return "ok" if handed_over?(tool_name, text)

data/lib/samagotchi/tool_response.rb

@@ -9,11 +9,11 @@ module Samagotchi
   # (max_tool_output_chars).
   module ToolResponse
     SEPARATOR = "\n\n---\n\n"
-    # Saved with a result for the web's reload (a plugin tool's params line
-    # and label, an edit/write's diff) and for the LLMContextView (each
+    # Saved with a result for the web's reload (a plugin tool's params line,
+    # label and row refs, an edit/write's diff) and for the LLMContextView (each
     # run's chi-owned id, ToolIds, and a strategy's edits by id,
     # LLMContextEdit); the prompt never renders them.
-    SAVED_KEYS = %i[tool_params tool_labels tool_diffs tool_ids edits].freeze
+    SAVED_KEYS = %i[tool_params tool_labels tool_diffs tool_refs tool_ids edits].freeze
     # A run's "[name]" lead: the name, then a newline, a space or the end.
     LEAD = /\A\[([^\]\s]+)\](?:\n| |\z)/
 
@@ -60,7 +60,7 @@ module Samagotchi
         entry[:image_counts] = runs.map { |run| Array(run[:images]).size }
       end
       saved = { tool_params: runs.map { |run| run[:shown_params] }, tool_labels: runs.map { |run| run[:shown_label] },
-                tool_diffs: runs.map { |run| run[:diff] } }
+                tool_diffs: runs.map { |run| run[:diff] }, tool_refs: runs.map { |run| run[:refs] } }
       saved.each { |key, values| entry[key] = values if values.any? }
       entry[:tool_ids] = ids
       entry
@@ -75,6 +75,7 @@ module Samagotchi
       entry[:tool_params] = run[:shown_params] if run[:shown_params]
       entry[:tool_labels] = run[:shown_label] if run[:shown_label]
       entry[:tool_diffs] = run[:diff] if run[:diff]
+      entry[:tool_refs] = run[:refs] if run[:refs]
       entry[:tool_ids] = ids
       entry
     end

data/lib/samagotchi/tool_row_fields.rb

@@ -16,13 +16,15 @@ module Samagotchi
   #   tool_call_completed (ToolActivity.tool_activity_event builds that
   #   activity itself, for the TUI and the metrics);
   # * EVENT_KEYS: top-level on both, so a UI that missed the start (a
-  #   replay gap) builds its row from the completed event.
+  #   replay gap) builds its row from the completed event. +refs+ (a
+  #   plugin tool's, RowRefs) is one, but ToolRunner adds it: #for has no
+  #   plugins, so a reload reads the saved tool_refs instead.
   #
   # params and label stay where they are: their sources differ per path
   # (the registry's preview, a plugin's label, the saved shown params).
   module ToolRowFields
     ACTIVITY_KEYS = %i[title called_as].freeze
-    EVENT_KEYS = %i[view ref].freeze
+    EVENT_KEYS = %i[view ref refs].freeze
     KEYS = (ACTIVITY_KEYS + EVENT_KEYS).freeze
 
     module_function

data/lib/samagotchi/tool_runner.rb

@@ -2,6 +2,7 @@
 
 require_relative "tool_activity"
 require_relative "tool_row_fields"
+require_relative "row_refs"
 require_relative "guardrails"
 require_relative "vision_context"
 require_relative "log"
@@ -41,7 +42,8 @@ module Samagotchi
     #   shown_params: the params line the live row showed, and shown_label:
     #   its label ("chrome: screenshot"), only for a tool that isn't built in
     #   (the loops save them with the result, so a reload without the plugin
-    #   shows the same row), and diff: what an edit/write changed in its
+    #   shows the same row), refs: its row's refs (a plugin tool's refs:,
+    #   RowRefs; saved likewise), and diff: what an edit/write changed in its
     #   file (EditPreview.change; also on tool_call_completed, never in the
     #   model's output)
     def run(call, iteration:, call_index:, call_count:, on_stream_event:, max_tool_output_chars:)
@@ -55,9 +57,12 @@ module Samagotchi
       started = { type: :tool_call_started, iteration: iteration, call_count: call_count, call_index: call_index,
                   tool: call[:name], call: call.dup, params: params }
       started[:label] = label if label
-      # Its row fields (ToolRowFields: title, called_as, view, ref); the worker runs in
-      # its session's working directory.
+      # Its row fields (ToolRowFields: title, called_as, view, ref, and a
+      # plugin tool's refs); the worker runs in its session's working
+      # directory.
       fields = ToolRowFields.for(call[:name], call, cwd: Dir.pwd)
+      refs = RowRefs.for(tools && !call[:name].nil? ? tools[call[:name]]&.refs : nil, call, cwd: session_cwd)
+      fields[:refs] = refs unless refs.empty?
       started.merge!(fields)
       emit(on_stream_event, started)
 
@@ -121,6 +126,7 @@ module Samagotchi
       run[:images] = images if images&.any?
       run[:shown_params] = params if params && plugin_tool?(call[:name])
       run[:shown_label] = label if label
+      run[:refs] = refs unless refs.empty?
       run[:diff] = diff if diff
       run
     end
@@ -249,6 +255,13 @@ module Samagotchi
       [result[:output], *notes].join("\n")
     end
 
+    # The session's folder (the Engine's, KernelLoop#session_cwd), else the
+    # process's: a bare kernel (specs) has none.
+    def session_cwd
+      cwd = @kernel.respond_to?(:session_cwd) && @kernel.session_cwd&.call
+      cwd.to_s.empty? ? Dir.pwd : cwd
+    end
+
     # The kernel's tools, for the activity line of a tool that isn't built in.
     def tools = @kernel.tools
 

data/lib/samagotchi/tools/builtins.rb

@@ -77,7 +77,7 @@ module Samagotchi
                     start_line: call[:start_line], end_line: call[:end_line])
         end,
         TaskCreate::NAME => lambda do |call, kctx|
-          next Execute::NOT_RUN_ON_STOP if cancelled_proc(kctx).call
+          next Execute.not_run(stop_reason_proc(kctx)) if cancelled_proc(kctx).call
 
           TaskCreate.call(call[:content], cwd: call[:cwd], env: call[:env],
                           marker_env: task_marker_env(kctx, call[:env]))
@@ -87,7 +87,8 @@ module Samagotchi
                                         done_pattern: call[:done_pattern], cancelled: cancelled_proc(kctx))
         end,
         Execute::NAME => lambda do |call, kctx|
-          Execute.call(call[:content], cwd: call[:cwd], cancelled: cancelled_proc(kctx), env: parent_env(kctx))
+          Execute.call(call[:content], cwd: call[:cwd], cancelled: cancelled_proc(kctx), stop_reason: stop_reason_proc(kctx),
+                                       env: parent_env(kctx))
         end,
         RegisterReminder::NAME => lambda do |call, kctx|
           RegisterReminder.call(call[:content], reminder_store: kctx.reminder_store, description: call[:description],
@@ -163,6 +164,12 @@ module Samagotchi
         -> { !!peers&.cancelled? }
       end
 
+      # The running turn's stop reason (:signal, :manual, …), nil without one.
+      def stop_reason_proc(kctx)
+        peers = kctx.peers
+        -> { peers.respond_to?(:cancel_reason) ? peers.cancel_reason : nil }
+      end
+
       # @param command_description [Boolean] false: execute declares no
       #   description parameter (execute.description off)
       # @param policy [String, nil] llm_context.policy, the sentence

data/lib/samagotchi/tools/execute.rb

@@ -27,6 +27,10 @@ module Samagotchi
       NOT_RUN_ON_STOP = "Error: not run, the user stopped the turn"
       # The first words of a command the user's Stop killed.
       STOPPED_BY_USER = "Error: command stopped by the user"
+      # The same two when a signal to chi (a worker's TERM, HUP or INT)
+      # stopped the turn: its cancel reason is :signal.
+      NOT_RUN_ON_SIGNAL = "Error: not run, a signal to chi stopped the turn"
+      STOPPED_BY_SIGNAL = "Error: command stopped by a signal to chi"
       # The first words of a command killed at the limit in kill mode (hooks
       # match on it).
       TIMEOUT_PREFIX = "Error: command timed out after"
@@ -73,11 +77,13 @@ module Samagotchi
 
       # @param cancelled [#call] true once the user stopped the turn: the
       #   command is killed (or not started) and its output so far returned
+      # @param stop_reason [#call] the stop's reason (:signal: a signal to
+      #   chi, which the result then names instead of the user)
       # @param env [Hash, nil] added to the command's environment
       # @param on_timeout [Symbol, String, nil] :task or :kill (ON_TIMEOUT);
       #   nil reads execute.on_timeout
-      def self.call(command, cwd: nil, cancelled: -> { false }, env: nil, on_timeout: nil)
-        return NOT_RUN_ON_STOP if cancelled.call
+      def self.call(command, cwd: nil, cancelled: -> { false }, stop_reason: -> {}, env: nil, on_timeout: nil)
+        return not_run(stop_reason) if cancelled.call
 
         command = command.strip
         resolved_cwd = resolve_cwd(cwd)
@@ -109,11 +115,17 @@ module Samagotchi
         # a long command, and return whatever it printed before it was killed.
         stopped_result("#{TIMEOUT_PREFIX} #{timeout_sec}s\n#{TIMEOUT_HINT}", e)
       rescue CommandCancelled => e
-        stopped_result("#{STOPPED_BY_USER} after #{e.elapsed.round}s (killed; rerun it if still needed)", e)
+        first = signal?(stop_reason) ? STOPPED_BY_SIGNAL : STOPPED_BY_USER
+        stopped_result("#{first} after #{e.elapsed.round}s (killed; rerun it if still needed)", e)
       rescue StandardError => e
         "Error: #{e.message}"
       end
 
+      # What a command the turn's stop kept from starting returns.
+      def self.not_run(stop_reason) = signal?(stop_reason) ? NOT_RUN_ON_SIGNAL : NOT_RUN_ON_STOP
+
+      def self.signal?(stop_reason) = stop_reason.call == :signal
+
       # execute.on_timeout, or the caller's: :task (the default) or :kill.
       def self.on_timeout_mode(value)
         value = Samagotchi::Config.get("execute.on_timeout") if value.nil? || value.to_s.strip.empty?

data/lib/samagotchi/tools/peers.rb

@@ -10,11 +10,12 @@ module Samagotchi
     # can't host one), and the model the session runs on (model_ref: the
     # resolved ref, live after /model), and the session's own thinking level
     # (thinking: nil without one) and reasoning_history level
-    # (reasoning_history: nil without one). Engine hands KernelLoop one that follows its current
+    # (reasoning_history: nil without one), and why the turn was canceled
+    # (cancel_reason: :signal, :manual, …; nil while it runs). Engine hands KernelLoop one that follows its current
     # session (Engine::PeerView); this plain one is for specs and a kernel
     # without an Engine.
     Peers = Struct.new(:session_id, :cwd, :state_dir, :cancelled, :relay, :model_ref, :thinking, :reasoning_history,
-                       keyword_init: true) do
+                       :cancel_reason, keyword_init: true) do
       # @return [Boolean] the turn was canceled (:cancelled is a proc or a value)
       def cancelled?
         cancelled.respond_to?(:call) ? !!cancelled.call : !!cancelled

data/lib/samagotchi/tools/registry.rb

@@ -20,11 +20,13 @@ module Samagotchi
       #   tool" when nil); ToolActivity knows the built-ins' own
       # @!attribute preview [#call, nil] call → the activity line's params
       # @!attribute targets [#call, nil] call → what guardrail rules match
+      # @!attribute refs [#call, nil] call → the row's refs (a plugin's
+      #   refs:, RowRefs checks what it gives)
       # @!attribute source [String] "core", or the bundle that added it
       # @!attribute layer [Symbol, nil] the LLM context layer the tool
       #   belongs to (LLMContextStrategy): offered only in a turn running
       #   under it (forget_outputs, :forget); nil: always
-      Entry = Struct.new(:name, :schema, :handler, :label, :preview, :targets, :source, :layer, keyword_init: true) do
+      Entry = Struct.new(:name, :schema, :handler, :label, :preview, :targets, :refs, :source, :layer, keyword_init: true) do
         def core? = source == "core"
 
         # Offered in a turn under +layers+ (the turn's LLM context layers).
@@ -36,11 +38,11 @@ module Samagotchi
       end
 
       # @return [Entry]
-      def register(name, schema:, handler:, label: nil, preview: nil, targets: nil, source: "core", layer: nil)
+      def register(name, schema:, handler:, label: nil, preview: nil, targets: nil, refs: nil, source: "core", layer: nil)
         raise ArgumentError, "tool #{name} is already registered" if @entries.key?(name)
 
         @entries[name] = Entry.new(name: name, schema: schema, handler: handler, label: label, preview: preview,
-                                   targets: targets, source: source, layer: layer)
+                                   targets: targets, refs: refs, source: source, layer: layer)
       end
 
       # Remove a tool (a plugin's tool set changed after load).

data/lib/samagotchi/turn_checkpoint.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "time"
+require_relative "interrupted_turn"
+require_relative "log"
+require_relative "session"
+require_relative "steer"
+require_relative "turn_note"
+
+module Samagotchi
+  # A worker's running turn, kept on disk at the loop's iteration
+  # boundaries, so a worker killed mid-turn (SIGKILL, a crash of the
+  # process) loses at most INTERVAL seconds of it instead of all of it.
+  #
+  # At each boundary (Worker#pending_input_drain, on the turn's thread) the
+  # turn's live conversation (Engine#live_conversation) is copied, with the
+  # lines that boundary merges (their input files are deleted as they are
+  # claimed), into the session's overlay (Session#turn_checkpoint), marked
+  # with an InterruptedTurn. Every save while the turn runs writes the
+  # overlay instead of the session's messages: this one's, written at the
+  # first boundary (the prompt reaches the disk) and then at most every
+  # INTERVAL (or when the boundary merged input), and any other save in the
+  # turn (a question asked). The session's in-memory messages never change
+  # (a Bridge snapshot shows the turn in progress or in the messages, never
+  # both). The turn's end saves the real messages without a marker.
+  #
+  # The copy is what a Stop at that boundary would keep, minus its note, so
+  # the history before the turn is never rewritten (prompt caching); later
+  # steps of the same turn may still edit their own entries in place.
+  class TurnCheckpoint
+    INTERVAL = 10.0
+
+    # A session loaded with an InterruptedTurn mark (its worker died in the
+    # turn): the turn ends now, as canceled. Its note goes at the tail
+    # (append-only: the model reads on its next turn that the previous one
+    # was cut, after how long), the status goes idle and the mark goes. The
+    # dead turn is not run again. The caller saves.
+    #
+    # last_turn is left as it was: a new last_turn saved idle here, before
+    # the turn a `chi send --wait` started this worker for, would end that
+    # wait at once as the sent turn's (ReplyWait compares last_turn), with
+    # the dead turn's outcome.
+    # @return [InterruptedTurn, nil] the mark settled, nil when there was none
+    def self.settle(session)
+      mark = session.interrupted_turn
+      return nil unless mark
+
+      messages = Array(session.messages)
+      # How long it ran up to its last checkpoint (it died later); none
+      # when that was its first boundary ("after 0s" would mislead).
+      seconds = mark.seconds
+      seconds = nil if seconds && seconds < 1
+      note = TurnNote.cancelled(:worker_stopped, seconds: seconds, shown: TurnNote.interrupted_tail?(messages))
+      session.messages = messages + [note]
+      session.status = Session::STATUS_IDLE if session.status == Session::STATUS_RUNNING
+      session.interrupted_turn = nil
+      mark
+    end
+
+    # @param session [Session] the worker's session (saved as a whole)
+    # @param clock [#call] monotonic seconds (specs)
+    def initialize(session:, state_dir:, interval: INTERVAL, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+      @session = session
+      @state_dir = state_dir
+      @interval = interval
+      @clock = clock
+      @boundaries = 0
+      @written_at = nil
+      @started_at = nil
+    end
+
+    # A turn begins (before Engine#run_turn and the worker's running save):
+    # the overlay starts as the history plus +prompt+ (text only: no
+    # images, no turn id), marked at iteration 0, so a worker killed before
+    # the loop's first boundary leaves the prompt and a mark the next one
+    # settles. A continue turn (no prompt) is the history, marked.
+    # @param prompt [String, nil]
+    def begin_turn(prompt: nil)
+      @boundaries = 0
+      @written_at = nil
+      @started_at = Time.now.iso8601(3)
+      messages = Array(@session.messages).dup
+      messages << { role: "user", content: prompt } unless prompt.to_s.empty?
+      @session.turn_checkpoint = { messages: messages, marker: InterruptedTurn.new(started_at: @started_at, at: @started_at, iteration: 0) }
+    end
+
+    # The turn ended (or never began): the session saves its own messages.
+    def end_turn
+      @session.turn_checkpoint = nil
+    end
+
+    # An iteration boundary of the running turn.
+    # @param conversation [Array<Hash>, nil] the loop's live conversation;
+    #   nil (no turn): nothing
+    # @param merging [Array<Steer::Line>] the lines this boundary merges
+    #   (appended after the conversation, as Steer.inject! will)
+    # @param turn_id [String, nil]
+    # @return [Boolean] whether the session file was written
+    def boundary(conversation, merging: [], turn_id: nil)
+      return false unless conversation
+
+      @boundaries += 1
+      merged = merging.empty? ? [] : Steer.merge(merging).messages
+      # One level deep: the loop may edit an entry in place later (a
+      # forget stub) while a save on another thread serializes this one.
+      messages = conversation.map { |message| message.is_a?(Hash) ? message.dup : message } + merged
+      marker = InterruptedTurn.new(id: turn_id, started_at: @started_at, at: Time.now.iso8601(3), iteration: @boundaries)
+      @session.turn_checkpoint = { messages: messages, marker: marker }
+      return false unless due? || !merged.empty?
+
+      write
+    end
+
+    private
+
+    def due? = @written_at.nil? || @clock.call - @written_at >= @interval
+
+    def write
+      @session.save(state_dir: @state_dir)
+      @written_at = @clock.call
+      true
+    rescue StandardError => e
+      # The turn goes on; the next boundary tries again.
+      Log.warn(:worker, "checkpoint_failed", error: e.class.name, msg: e.message)
+      false
+    end
+  end
+end

data/lib/samagotchi/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Samagotchi
-  VERSION = "0.60.0"
+  VERSION = "0.62.0"
   # Sent as the User-Agent of every HTTP request chi makes.
   USER_AGENT = "chi/#{VERSION}"
 end