samagotchi 0.60.0 → 0.62.0

data/lib/samagotchi/web/public/turn_html.js

@@ -7,7 +7,7 @@ import { commandBlockHtml, rowHover } from "./command_view.js";
 import { copySourceAttr } from "./copy.js";
 import { activityDiffHtml } from "./diff_view.js";
 import { escapeHtml, userBodyHtml } from "./format.js";
-import { refAttrs } from "./refs.js";
+import { refAttrs, rowTitleRef } from "./refs.js";
 import { readLinesHtml } from "./read_view.js";
 import { formatDuration } from "./timing.js";
 import { emptyAnswerLine } from "./turn_events.js";
@@ -84,9 +84,10 @@ export function toolRowHtml(row, { thumbs = () => "" } = {}) {
 export function toolRowInnerHtml(row, { thumbs = () => "" } = {}) {
   const shown = row.title || row.params;
   const hover = rowHover(row) ? ` title="${escapeHtml(rowHover(row))}"` : "";
-  // A row with a ref (a file tool's file) carries it on its title, which
-  // opens it, and its diff lines open it at their line (refs.js).
-  const ref = shown ? refAttrs(row.ref) : "";
+  // A row with a ref (a file tool's file, else a plugin tool's first ref)
+  // carries it on its title, which opens it, and its diff lines open it at
+  // their line (refs.js; bindRowRefs binds a plugin's data to the title).
+  const ref = shown ? refAttrs(rowTitleRef(row)) : "";
   const params = shown ? `<span class="activity-params${ref ? " ref" : ""}"${hover}${ref}>${escapeHtml(shown)}</span>` : "";
   // The name the model called the tool by when it was an alias's, dim.
   const alias = row.called_as ? `<span class="activity-alias">(called as ${escapeHtml(row.called_as)})</span>` : "";

data/lib/samagotchi/web/public/turn_model.js

@@ -152,6 +152,20 @@ export function notice(turn, { hook, text, level } = {}) {
   return { gen };
 }
 
+// The whole text of the reply that asked the user (a question or an
+// approval blocks its call): the last gen's text, trimmed, while one of its
+// calls runs and the turn goes on; else "". Only that step's text: a hook's
+// question (asked before its call's row, or between steps) has no running
+// row there, so it gets none rather than another step's (or the answer's)
+// text. The stage shows it while the card waits; the card keeps it once
+// answered.
+export function askingText(turn) {
+  if (!turn || turn.ended) return "";
+  const gen = turn.gens[turn.gens.length - 1];
+  if (!gen?.text || !gen.tools.some((row) => row.status === "running")) return "";
+  return gen.text.trim();
+}
+
 // The turn is over (completed, canceled, failed, the worker gone).
 export function turnEnded(turn) {
   const closed = closeCurrent(turn);
@@ -215,6 +229,41 @@ export function genLabelParts(gen) {
   return { head: "thinking", calls: "", repeats: false };
 }
 
+// A step that said something (its text is not the turn's answer): a row of
+// its own in a finished block's outline.
+export function isNarrated(gen) {
+  return !gen.answer && !!String(gen.text || "").trim();
+}
+
+// A finished block's outline (+gens+: its steps in order, as the block holds
+// them): each narrated step a row, each run of the others between (tool-only,
+// thinking-only) one fold ({fold: [indices]}), so every mid-turn text's first
+// line reads without a click and the steps that said nothing take one line.
+// [] when no step said anything: the block stays closed, as before.
+// A last run of one thinking-only step stays a row (no "· 1 step" at the end).
+export function outlineRuns(gens) {
+  if (!gens.some(isNarrated)) return [];
+  const runs = [];
+  gens.forEach((gen, i) => {
+    if (isNarrated(gen)) return runs.push({ step: i });
+    const last = runs[runs.length - 1];
+    if (last?.fold) last.fold.push(i);
+    else runs.push({ fold: [i] });
+  });
+  // A turn doesn't end on a "1 step" fold for a lone thinking-only step
+  // (the answer's own thinking): it stays a row, its "thinking" label.
+  const last = runs[runs.length - 1];
+  const lone = last.fold?.length === 1 && gens[last.fold[0]];
+  if (lone && lone.tools.length === 0) runs[runs.length - 1] = { step: last.fold[0] };
+  return runs;
+}
+
+// A finished turn's steps as its block holds them: the answer's step left it
+// when the answer was all it had (turn_view.js promote).
+export function blockGens(turn) {
+  return turn.gens.filter((gen) => !(gen.answer && !gen.thinking && gen.tools.length === 0));
+}
+
 // The block's summary line: "working · 3 steps · 8 tool calls · execute ×7 · …"
 // while it runs, without the prefix after; the tally (tally.js) from 3 calls.
 export function blockSummary(turn, { running = false } = {}) {

data/lib/samagotchi/web/public/turn_view.js

@@ -21,6 +21,8 @@
 // chrome (CSS :has), so a plain answer streams like today's bubble, and is
 // removed at the end: the answer bubble takes its place.
 import { html, nothing, render } from "./vendor/lit-html.js";
+import { setAnswerRefs } from "./answer_refs.js";
+import { bindRowRefs } from "./refs.js";
 import { leavesBlock } from "./card.js";
 import { copySourceAttr } from "./copy.js";
 import { escapeHtml, failedTurnText, messageBodyHtml, noteHtml, normalize, sourceLabel, steerRowHtml } from "./format.js";
@@ -115,6 +117,67 @@ function rawNodes(markup) {
   return t.content;
 }
 
+// ---- A finished block's outline ----
+
+// A finished block as its outline (turn_model.js outlineRuns of +gens+, its
+// steps in order): open, each narrated step a one-line row as before, each
+// run of steps that said nothing hidden behind one "N steps" fold that
+// shows them. The live turn's end and a reload call it alike. The folds sit
+// between the steps, not in them (`:scope > .gen` still counts steps), each
+// before its run's first step and before that step's lit marker comment, so
+// a later render of the step leaves it alone. False when there is nothing
+// to outline (no step said anything): the block stays as it is.
+export function outlineBlock(blockEl, gens) {
+  const runs = model.outlineRuns(gens);
+  const steps = blockEl ? [...blockEl.querySelectorAll(":scope > .gen")] : [];
+  if (!runs.length || steps.length !== gens.length) return false;
+  blockEl.classList.add("outline");
+  // A folded step → its fold's opener: a ref's "show in turn" (refs.js
+  // showInTurn dispatches ref:reveal on the row before it scrolls) unfolds
+  // the run its row is in, or the row stays hidden.
+  const unfold = new Map();
+  for (const { fold } of runs) {
+    if (!fold) continue;
+    const folded = fold.map((i) => steps[i]);
+    folded.forEach((el) => el.classList.add("folded"));
+    const marker = document.createElement("div");
+    marker.className = "outline-fold";
+    marker.setAttribute("role", "button");
+    marker.tabIndex = 0;
+    marker.setAttribute("aria-expanded", "false");
+    marker.textContent = `${fold.length} step${fold.length === 1 ? "" : "s"}`;
+    const toggle = () => {
+      const open = marker.classList.toggle("open");
+      marker.setAttribute("aria-expanded", String(open));
+      folded.forEach((el) => el.classList.toggle("unfolded", open));
+    };
+    folded.forEach((el) => unfold.set(el, () => { if (!marker.classList.contains("open")) toggle(); }));
+    marker.addEventListener("click", toggle);
+    marker.addEventListener("keydown", (e) => {
+      if (e.key !== "Enter" && e.key !== " ") return;
+      e.preventDefault();
+      toggle();
+    });
+    const first = folded[0];
+    const anchor = first.previousSibling?.nodeType === 8 ? first.previousSibling : first;
+    anchor.before(marker);
+  }
+  blockEl.addEventListener("ref:reveal", (e) => {
+    const step = e.target.closest?.(".gen.folded");
+    if (step?.parentElement === blockEl) unfold.get(step)?.();
+  });
+  blockEl.open = true;
+  return true;
+}
+
+// Reloaded tool rows (turn_html.js toolRowHtml) as nodes, each row's
+// plugin refs bound to its title (refs.js bindRowRefs).
+function rowNodes(rows, thumbs) {
+  const nodes = rawNodes(rows.map((r) => toolRowHtml(r, { thumbs })).join(""));
+  [...nodes.children].forEach((el, i) => bindRowRefs(el, rows[i]));
+  return nodes;
+}
+
 // ---- The live view ----
 
 // The parts of a drawn tool row a user may have changed (the command's
@@ -444,12 +507,16 @@ export function createTurnView({
     const inner = toolRowInnerHtml(r.row, { thumbs: rowThumbs });
     if (!el) {
       el = htmlElement(toolRowHtml(r.row, { thumbs: rowThumbs }));
+      bindRowRefs(el, r.row);
       rowHtml.set(el, inner);
       entry.rows.set(r.row.key, el);
       addItem(entry, el);
     } else {
       if (!r.gen.text) entry.placeholder = false;
-      if (rowHtml.get(el) !== inner) redrawRow(el, inner);
+      if (rowHtml.get(el) !== inner) {
+        redrawRow(el, inner);
+        bindRowRefs(el, r.row);
+      }
       rowHtml.set(el, inner);
       // A late result in a closed step: its label and failed mark.
       draw(entry);
@@ -602,7 +669,11 @@ export function createTurnView({
     if (blockEl) {
       updateBlockSummary();
       blockEl.classList.add("done");
-      if (!blockToggled) blockEl.open = false;
+      // The outline (turn_model.js outlineRuns): open on its narrated steps'
+      // rows, as a reload draws it; a block with nothing said closes.
+      if (newestFirst) chronological();
+      const outlined = outlineBlock(blockEl, model.blockGens(turn));
+      if (!blockToggled) blockEl.open = outlined;
       for (const el of blockEl.querySelectorAll(".plugin-card")) {
         if (leavesBlock({ warn: el.classList.contains("warn"), kind })) kept.push(el);
       }
@@ -614,6 +685,20 @@ export function createTurnView({
     return kept;
   }
 
+  // The steps back in chronological order after newestFirst (a no-op
+  // otherwise); a fold (outlineBlock) stays before its run's first step.
+  function chronological() {
+    if (!blockEl || !turn) return;
+    for (const gen of turn.gens) {
+      const entry = genEls.get(gen);
+      if (entry?.el.parentNode !== blockEl) continue;
+      const nodes = stepNodes(entry);
+      const fold = nodes[0].previousSibling;
+      if (fold?.classList?.contains("outline-fold")) nodes.unshift(fold);
+      blockEl.append(...nodes);
+    }
+  }
+
   function reset() {
     hold.finish();
     turn = null;
@@ -711,17 +796,14 @@ export function createTurnView({
     // block, a step's elements by index, and the steps back in
     // chronological order after newestFirst.
     current() { return { turn, blockEl }; },
+    // The running turn's asking reply (turn_model.js askingText): what a
+    // question or approval card keeps once answered.
+    askingText: () => model.askingText(turn),
     step(i) {
       const entry = turn && genEls.get(turn.gens[i]);
       return entry ? { el: entry.el, summary: entry.el.querySelector(":scope > summary"), thinkingWrap: thinkingWrap(entry) } : null;
     },
-    chronological() {
-      if (!blockEl || !turn) return;
-      for (const gen of turn.gens) {
-        const entry = genEls.get(gen);
-        if (entry?.el.parentNode === blockEl) blockEl.append(...stepNodes(entry));
-      }
-    },
+    chronological,
   };
 }
 
@@ -736,6 +818,14 @@ function defaultPlaceAnswer(appendToHistory) {
 
 // ---- A reload ----
 
+// The text of a reloaded block's step +iteration+ (1-based, as the snapshot's
+// cards count), "" when it has none: a reloaded question card's asking
+// reply.
+export function stepText(block, iteration) {
+  const step = block?.querySelectorAll(":scope > .gen")[Number(iteration) - 1];
+  return step?.querySelector(":scope > .gen-text")?.textContent || "";
+}
+
 // A re-rendered failed turn's line (a .bubble.cancel, as live's
 // addEndBubble), in turn_failed's words (failedTurnText) from what the turn
 // record kept of it (SessionMetrics' failure: summary, message,
@@ -779,10 +869,12 @@ export function turnHistoryNodes(items, timing, { thumbs }) {
       : "";
   };
   // The timing line after the bubble, not in it: where the live view
-  // leaves it at the turn's end.
-  const answerHtml = (m, timing) => {
+  // leaves it at the turn's end. +at+: the message's index, for its refs
+  // (data-answer-at, gone before the nodes are returned).
+  const answerHtml = (at, timing) => {
+    const m = items[at];
     const { body, renderedMarkdown } = messageBodyHtml(m);
-    return `<div class="bubble output${renderedMarkdown ? " markdown" : ""}"${copySourceAttr(m, renderedMarkdown)}>${body}</div>${timing}`;
+    return `<div class="bubble output${renderedMarkdown ? " markdown" : ""}" data-answer-at="${at}"${copySourceAttr(m, renderedMarkdown)}>${body}</div>${timing}`;
   };
   // A user's line, labelled when chi brought it in (a delegate report). A
   // `!cmd`'s saved message (the worker's "!(<command>)\n<output>") renders
@@ -799,14 +891,16 @@ export function turnHistoryNodes(items, timing, { thumbs }) {
     const j = items.findIndex((m, k) => k > i && m?.role === "assistant" && String(m.content ?? "").trim());
     if (j < 0 || j === group.answer || items.slice(i + 1, j).some((m) => m?.role === "user" && !m.merged)) return "";
     const { body, renderedMarkdown } = messageBodyHtml(items[j]);
-    return `<div class="bubble output report-reply${renderedMarkdown ? " markdown" : ""}"${copySourceAttr(items[j], renderedMarkdown)}>${body}</div>`;
+    return `<div class="bubble output report-reply${renderedMarkdown ? " markdown" : ""}" data-answer-at="${j}"${copySourceAttr(items[j], renderedMarkdown)}>${body}</div>`;
   };
   const steerHtml = (i, bubble) => steerRowHtml({ source: items[i].source, text: items[i].content }, { bubble });
   const parts = [];
+  // Each block's steps, in order: its outline once the nodes exist.
+  const outlines = [];
   for (const group of turnGroups(items, timing)) {
     if (group.kind === "note") { parts.push(`<div class="bubble note">${noteHtml(items[group.i])}</div>`); continue; }
     if (group.kind === "steer") { parts.push(steerHtml(group.i, true)); continue; }
-    if (group.kind === "answer") { parts.push(answerHtml(items[group.answer], "")); continue; }
+    if (group.kind === "answer") { parts.push(answerHtml(group.answer, "")); continue; }
     const prompt = items[group.user];
     // A context wake turn starts with its note.
     parts.push(prompt.role === "note" ? `<div class="bubble note">${noteHtml(prompt)}</div>` : userHtml(prompt));
@@ -824,9 +918,10 @@ export function turnHistoryNodes(items, timing, { thumbs }) {
       const steps = gens.map((gen, k) => stepTemplate(gen, {
         // The steers the step answered first, where the live rows went.
         steers: (group.steps[k].steers || []).map((i) => rawNodes(steerHtml(i, false))),
-        items: gen.tools.length ? [rawNodes(gen.tools.map((r) => toolRowHtml(r, { thumbs })).join(""))] : [],
+        items: gen.tools.length ? [rowNodes(gen.tools, thumbs)] : [],
       }));
       parts.push(html`<details class="turn-work done"><summary>${model.blockSummary({ gens, ended: true })}</summary>${steps}</details>`);
+      outlines.push(gens);
     }
     // Steers with no step to sit in, or in a turn that shows no block: on
     // their own after the prompt.
@@ -846,11 +941,20 @@ export function turnHistoryNodes(items, timing, { thumbs }) {
     // A turn with no answer (canceled, or running: no record yet) shows its
     // timing after its prompt and steps, where the live view leaves it.
     else if (group.answer === null) parts.push(timingHtml(group));
-    else parts.push(answerHtml(items[group.answer], timingHtml(group)));
+    else parts.push(answerHtml(group.answer, timingHtml(group)));
     // A canceled turn ends with its cancel line, a failed one whose steps
     // stayed with its failure line, as the live view left them.
     parts.push(cancelLineHtml(group.record, escapeHtml));
     parts.push(failedLineHtml(group.record));
   }
-  return fragmentOf(parts);
+  const out = fragmentOf(parts);
+  // A finished turn's block as the live end leaves it: its outline, open.
+  [...out.children].filter((el) => el.matches(".turn-work")).forEach((block, i) => outlineBlock(block, outlines[i] || []));
+  // Each answer bubble knows its message's refs (answer_refs.js; the
+  // page pairs them after the render), the index stays out of the DOM.
+  for (const bubble of out.querySelectorAll("[data-answer-at]")) {
+    setAnswerRefs(bubble, items[Number(bubble.dataset.answerAt)]?.display_refs);
+    bubble.removeAttribute("data-answer-at");
+  }
+  return out;
 }

data/lib/samagotchi/worker.rb

@@ -12,10 +12,12 @@ require_relative "steer"
 require_relative "worker_idle_exit"
 require_relative "worker_wakes"
 require_relative "worker_inbound"
+require_relative "worker_signals"
 require_relative "session_manager"
 require_relative "archive_store"
 require_relative "log"
 require_relative "turn_flow"
+require_relative "turn_checkpoint"
 require_relative "llm/errors"
 require_relative "continue_offer"
 require_relative "iteration_limit"
@@ -52,6 +54,13 @@ module Samagotchi
   # /continue) run on the loop between turns, before any queued prompt. One
   # taken while a turn runs (at an iteration boundary, or right after the
   # turn) is refused as busy.
+  #
+  # A TERM, HUP or INT (WorkerSignals, trapped in a spawned worker only)
+  # stops the running turn as a Stop does (reason :signal), the turn's
+  # work is saved as after any Stop, and the worker leaves (:signal) taking
+  # no new work. A turn that hasn't ended SIGNAL_GRACE seconds after the
+  # signal is cut: the session is saved as it is and the signal raised in
+  # the loop's thread, as Ruby's default would (the teardown still runs).
   class Worker
     FALLBACK_TICK_SECONDS = 5
     # A worker just started runs no wake turn for delegate reports this
@@ -64,6 +73,18 @@ module Samagotchi
     BUSY_OUTPUT = "busy: wait for the turn to end"
     # !rollback mid-turn would mean "the turn I'm in".
     ROLLBACK_BUSY_OUTPUT = "busy: Ctrl-C the turn first, then !rollback"
+    # How long a signalled worker's turn has to end (it is canceled at once,
+    # and again every SIGNAL_RETRY until it ends: a signal that lands just
+    # before the turn begins finds none to cancel).
+    SIGNAL_GRACE = 3.0
+    # A second signal, or a TERM after `chi sessions stop` marked the
+    # session stopped (that stop already canceled over the Bridge and waits
+    # only seconds for the lock): this much.
+    SIGNAL_SHORT_GRACE = 0.5
+    SIGNAL_RETRY = 0.1
+    # After the grace ran out and the signal was raised: the process ends
+    # this much later at the latest (a teardown that hangs).
+    SIGNAL_HARD_EXIT = 10.0
 
     # Wakes the worker loop. Whoever queues work writes it first and wakes
     # after, and #wait drains every wake before the loop looks for work: a
@@ -90,7 +111,12 @@ module Samagotchi
 
     # @param idle_exit_minutes [Numeric, nil] nil: session.idle_exit_minutes
     # @param poll_interval [Numeric, nil] seconds between fallback ticks
-    def initialize(session_id:, state_dir:, session_dir:, idle_exit_minutes: nil, poll_interval: nil)
+    # @param signals [WorkerSignals, WorkerSignals::None] the process's
+    #   trapped signals (SessionManager.run_session_loop); none by default
+    # @param signal_grace [Array(Numeric, Numeric, Numeric)] the grace, the
+    #   short grace and the hard exit's delay (specs shorten them)
+    def initialize(session_id:, state_dir:, session_dir:, idle_exit_minutes: nil, poll_interval: nil,
+                   signals: WorkerSignals::NONE, signal_grace: [SIGNAL_GRACE, SIGNAL_SHORT_GRACE, SIGNAL_HARD_EXIT])
       @session_id = session_id
       @state_dir = state_dir
       @session_dir = session_dir
@@ -116,6 +142,20 @@ module Samagotchi
       @asked_parent = false
       @owes_parent = false
       @initial_turn = false
+      @signals = signals
+      @signal_grace, @signal_short_grace, @signal_hard_exit = signal_grace
+      # Set as the loop starts, and as it returns (or raises): a signal's
+      # grace runs out only in between.
+      @loop_started = false
+      @left_loop = false
+      @signal_mutex = Mutex.new
+      @grace_expired = false
+      # The running turn has ended (its end event) and its after_turn hooks
+      # run: a signal's grace waits for them (#signalled).
+      @turn_ending = false
+      # The worker's own saves and a signal's (#signal_grace_expired), one
+      # at a time.
+      @save_mutex = Mutex.new
     end
 
     # Whether the session was empty as the worker left it, so the caller
@@ -135,30 +175,46 @@ module Samagotchi
     # @return [Symbol] :idle_exit, :exit_requested when a client asked it
     #   to exit (Bridge POST /exit), :restart when it asked for a restart
     #   (the caller starts a new worker), :stopped when the session was stopped
-    #   (chi stop), :crashed when the loop raised (the session is marked
-    #   errored); SessionManager.run_session_loop turns it into the exit
+    #   (chi stop), :signal when a signal stopped it (WorkerSignals),
+    #   :crashed when the loop raised (the session is marked errored);
+    #   SessionManager.run_session_loop turns it into the exit
     def run
+      # The thread a signal's expired grace raises in (the main thread in a
+      # worker process).
+      @run_thread = Thread.current
+      @signals.on_signal { |name, count| signalled(name, count) }
       start
       serve
-      begin
-        # A session stopped before this worker took the lock (e.g. a stop
-        # right after create) must not run its initial prompt.
-        return :stopped if stopped_on_disk?
-
-        start_background
-        loop do
-          leaving = pass
-          return leaving if leaving
-        end
+      # That raise lands in the loop only (#signal_grace_expired arms once
+      # it started, and never after it left): the teardown always runs
+      # whole, a raise already on its way waits until it is done.
+      Thread.handle_interrupt(SignalException => :never) do
+        Thread.handle_interrupt(SignalException => :immediate) { run_loop }
       rescue StandardError => e
         crashed(e)
       ensure
+        @signal_mutex.synchronize { @left_loop = true }
         teardown
       end
     end
 
     private
 
+    # @return [Symbol] what #run returns
+    def run_loop
+      @signal_mutex.synchronize { @loop_started = true }
+      # A session stopped before this worker took the lock (e.g. a stop
+      # right after create) must not run its initial prompt.
+      return :stopped if stopped_on_disk?
+      return signal_leave if signalled?
+
+      start_background
+      loop do
+        leaving = pass
+        return leaving if leaving
+      end
+    end
+
     # The session, its Engine and what runs its turns, before anyone can
     # reach it.
     def start
@@ -171,7 +227,10 @@ module Samagotchi
       ChildMove.adopt(@session_id, state_dir: @state_dir)
       @session = Session.load(@session_id, state_dir: @state_dir)
       drop_dead_question
+      # The running turn, on disk at its iteration boundaries.
+      @checkpoint = TurnCheckpoint.new(session: @session, state_dir: @state_dir)
       @engine = build_engine
+      settle_interrupted_turn
       # Before the Bridge serves anything: a UI joining a resumed worker's
       # stream gets the session's history and status in its snapshot, not
       # an empty session until the first turn.
@@ -200,8 +259,12 @@ module Samagotchi
         if Events::TURN_END.include?(event[:type])
           @turn_end_seq = event[:event_seq]
           @turn_end_type = event[:type]
+          @turn_ending = true
+        end
+        if event[:type] == :turn_started
+          @turn_id = event[:turn_id]
+          @turn_ending = false
         end
-        @turn_id = event[:turn_id] if event[:type] == :turn_started
         ring_question(event) if event[:type] == :question_requested
       })
       # This session's own delegate children's news (rings in children/).
@@ -268,6 +331,9 @@ module Samagotchi
     # next pass looks again; with none, #idle_pass.
     # @return [Symbol, nil] what #run returns as the worker leaves, nil to go on
     def pass
+      # A signalled worker takes no new work.
+      return signal_leave if signalled?
+
       # Check if the session was externally marked as stopped. Not
       # stopped_on_disk?, which reads a vanished file as "not stopped":
       # a session file deleted under a running worker crashes it here
@@ -316,8 +382,8 @@ module Samagotchi
     # @return [nil]
     def run_input_files(input_files)
       input_files.sort.each do |input_file|
-        # A stop between two queued turns leaves the rest queued.
-        break if stopped_on_disk?
+        # A stop (or a signal) between two queued turns leaves the rest queued.
+        break if signalled? || stopped_on_disk?
 
         # Commands that came before this file (D1: a queued /model X sent
         # before this prompt, a command an earlier file carried) run first.
@@ -385,6 +451,17 @@ module Samagotchi
       engine
     end
 
+    # A turn an earlier worker died in (TurnCheckpoint's mark in the file):
+    # ended now as canceled, with its note, and saved, before the Bridge
+    # serves the session (TurnCheckpoint.settle).
+    def settle_interrupted_turn
+      mark = TurnCheckpoint.settle(@session)
+      return unless mark
+
+      Log.info(:worker, "interrupted_turn_settled", turn_id: mark.id, iteration: mark.iteration, at: mark.at)
+      save_or_log(:interrupted_turn) { @session.save(state_dir: @state_dir) }
+    end
+
     # A question saved by a worker that died while it waited (an
     # ask_user_question, an approval, a hook's question): the turn that
     # asked is gone, so nobody can answer it, and left in the file it
@@ -572,7 +649,8 @@ module Samagotchi
     # @return [Boolean] whether any command ran
     def run_queued_commands
       ran = false
-      while (command = next_ready_command)
+      # A signalled worker runs no more (a /continue would start a turn).
+      while !signalled? && (command = next_ready_command)
         run_command(command)
         ran = true
       end
@@ -822,6 +900,7 @@ module Samagotchi
       @reporting_turn = reports_to_parent?(turn_args[:origin])
       restores = @turn_flow.restores
       begin
+        @checkpoint.begin_turn(prompt: prompt)
         @session.status = Session::STATUS_RUNNING
         # A continue that moved this session to the chain's new link wrote
         # the new parent beside it (Session::PARENT_FILE): this turn's
@@ -840,6 +919,8 @@ module Samagotchi
           Log.warn(:worker, "turn_not_begun", error: e.class.name, msg: e.message)
         end
       ensure
+        # Ended (or never begun): the saves write the session's own messages.
+        @checkpoint.end_turn
         refuse_queued_commands
       end
       yield result, error
@@ -995,6 +1076,10 @@ module Samagotchi
                 reports.map(&:line)
         # A wake turn starts with its first line, whoever sent it.
         lines[0] = lines[0].with(mark: mark) if mark && lines.any?
+        # The turn so far, with what this boundary merges (its input files
+        # are gone now), on disk at most every TurnCheckpoint::INTERVAL;
+        # its metrics so far with it, so a turn a KILL ends keeps them.
+        @engine.checkpoint_metrics if @checkpoint.boundary(@engine.live_conversation, merging: lines, turn_id: @turn_id)
         lines
       end
     end
@@ -1127,11 +1212,74 @@ module Samagotchi
       SessionManager.stopped_on_disk?(@session_id, state_dir: @state_dir)
     end
 
-    # Save the session unless it was stopped meanwhile: the stop (chi stop,
-    # from another process) saved its status, and this save would write
-    # the worker's over it.
+    # Save the session, a stopped one too: a stop is the marker file
+    # (Session.mark_stopped), which every load reads over the saved status,
+    # so this save can't hide it, and the turn the stop canceled is kept.
     def save_session
-      @session.save(state_dir: @state_dir) unless stopped_on_disk?
+      @save_mutex.synchronize { @session.save(state_dir: @state_dir) }
+    end
+
+    def signalled? = !@signals.signal.nil?
+
+    # @return [Symbol] :stopped when the session was stopped too (chi
+    #   sessions stop TERMs its worker), else :signal
+    def signal_leave
+      Session.stopped_marker?(@session_id, state_dir: @state_dir) ? :stopped : :signal
+    end
+
+    # A signal (WorkerSignals#fire), on a thread of its own: the loop wakes
+    # (an idle worker leaves at once), the running turn is canceled as a
+    # Stop (again every SIGNAL_RETRY, until the loop has left), and when
+    # the grace runs out first, #signal_grace_expired. A turn that has
+    # ended and runs its after_turn hooks (which store the answer's display
+    # and refs) is no longer canceled, and gets until SIGNAL_HARD_EXIT
+    # past the grace, the worker's own end of the turn (its saves, after
+    # the turn stopped running) too: the loop leaves next. A grace that runs out before the loop started waits
+    # for it (the loop's first look leaves). A signal after the loop left
+    # (the teardown, the caller's last steps) only brings the hard exit.
+    def signalled(name, count)
+      Log.info(:worker, "signal", signal: name, count: count, turn_running: @engine&.turn_running? || false)
+      return signal_hard_exit if @left_loop
+
+      @waker.wake
+      short = count > 1 || Session.stopped_marker?(@session_id, state_dir: @state_dir)
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      deadline = now + (short ? @signal_short_grace : @signal_grace)
+      hooks_deadline = deadline + @signal_hard_exit
+      ended = false
+      until @left_loop
+        running = @engine&.turn_running?
+        ended ||= running && @turn_ending
+        @engine.cancel_current_turn!(:signal) if running && !ended
+        break if @loop_started && Process.clock_gettime(Process::CLOCK_MONOTONIC) >= (ended ? hooks_deadline : deadline)
+
+        sleep(SIGNAL_RETRY)
+      end
+      signal_grace_expired(name) unless @left_loop
+    rescue StandardError => e
+      Log.warn(:worker, "signal_failed", signal: name, error: e.class.name, msg: e.message)
+    end
+
+    # The process ends SIGNAL_HARD_EXIT from now at the latest.
+    def signal_hard_exit
+      sleep(@signal_hard_exit)
+      @signals.hard_exit
+    end
+
+    # The turn didn't end in time (a tool that ignores the cancel, slow
+    # after_turn hooks): the session is saved as it is now (a turn that has
+    # ended: with it), and the signal raised in the loop's thread, so
+    # Worker#run's teardown and the lock's release still run. The process
+    # ends SIGNAL_HARD_EXIT later at the latest. Once, whatever the signals.
+    def signal_grace_expired(name)
+      expired = @signal_mutex.synchronize { !@grace_expired && !@left_loop && (@grace_expired = true) }
+      return unless expired
+
+      Log.warn(:worker, "signal_grace_expired", signal: name, turn_running: @engine&.turn_running? || false)
+      save_or_log(:signal) { save_session }
+      # Not once the loop has left (it may have, during the save).
+      @signal_mutex.synchronize { @run_thread&.raise(SignalException.new(name)) unless @left_loop }
+      signal_hard_exit
     end
   end
 end

data/lib/samagotchi/worker_inbound.rb

@@ -201,9 +201,11 @@ module Samagotchi
 
     private
 
-    # +unless_stopped+: not when the session was stopped meanwhile: the stop
-    # (chi stop, from another process) saved its status, and this save
-    # would write the worker's over it.
+    # +unless_stopped+: skipped when the session was stopped meanwhile (chi
+    # sessions stop, from another process: its marker file,
+    # Session.mark_stopped, which a load reads over the saved status). The
+    # save couldn't hide the stop; it is left out so the file stays as the
+    # stop found it, the worker's idle status unwritten.
     def save(at, unless_stopped: false)
       self.class.save_or_log(at) do
         @session.save(state_dir: @state_dir) unless unless_stopped && @stopped.call