samagotchi 0.60.0 → 0.61.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +41 -1
  3. data/docs/architecture.md +7 -4
  4. data/docs/hooks.md +72 -1
  5. data/docs/internals/engine.md +34 -5
  6. data/docs/internals/prompt-caching.md +3 -1
  7. data/docs/plugins.md +7 -3
  8. data/docs/sessions.md +7 -3
  9. data/lib/samagotchi/answer_display.rb +4 -3
  10. data/lib/samagotchi/answer_refs.rb +210 -0
  11. data/lib/samagotchi/bridge/card_store.rb +6 -1
  12. data/lib/samagotchi/bundles/check-in/manifest.yml +2 -2
  13. data/lib/samagotchi/bundles/check-in/plugin.rb +27 -3
  14. data/lib/samagotchi/bundles/system/manifest.yml +1 -1
  15. data/lib/samagotchi/engine.rb +38 -9
  16. data/lib/samagotchi/hooks/registry.rb +3 -1
  17. data/lib/samagotchi/interrupted_turn.rb +39 -0
  18. data/lib/samagotchi/kernel_loop.rb +6 -4
  19. data/lib/samagotchi/llm/chat_loop.rb +5 -2
  20. data/lib/samagotchi/log_subscriber.rb +6 -3
  21. data/lib/samagotchi/ref.rb +25 -8
  22. data/lib/samagotchi/reply_wait.rb +6 -2
  23. data/lib/samagotchi/session.rb +40 -4
  24. data/lib/samagotchi/session_manager.rb +64 -6
  25. data/lib/samagotchi/terminal_ui.rb +4 -0
  26. data/lib/samagotchi/turn_checkpoint.rb +128 -0
  27. data/lib/samagotchi/version.rb +1 -1
  28. data/lib/samagotchi/web/app.rb +5 -0
  29. data/lib/samagotchi/web/public/answer_refs.js +217 -0
  30. data/lib/samagotchi/web/public/app.js +15 -18
  31. data/lib/samagotchi/web/public/index.html +5 -3
  32. data/lib/samagotchi/web/public/ref_menu.js +9 -1
  33. data/lib/samagotchi/web/public/ref_scan.js +5 -3
  34. data/lib/samagotchi/web/public/refs.js +28 -6
  35. data/lib/samagotchi/web/public/stream_handlers.js +5 -3
  36. data/lib/samagotchi/web/public/turn_view.js +17 -7
  37. data/lib/samagotchi/worker.rb +168 -21
  38. data/lib/samagotchi/worker_signals.rb +81 -0
  39. metadata +6 -1
@@ -46,7 +46,7 @@ import { createTaskStopper } from "./task_stop.js";
46
46
  import { appendQuote } from "./annotations.js";
47
47
  import { parsePresets } from "./annotate_presets.js";
48
48
  import { createAnnotateBar } from "./annotate_bar.js";
49
- import { COPY_SOURCE_ATTR, copySource, copyText, installCopy, showsAnswer } from "./copy.js";
49
+ import { copyText, installCopy } from "./copy.js";
50
50
  import { ctxBarText, ctxBarTip, lastSpeedText, speedText } from "./ctx.js";
51
51
  import { createQuestionCards } from "./question_cards.js";
52
52
  import { cardClass, cardInnerHtml, cardPlace, isCard, isNoticeCard, noticeInnerHtml, splitSnapshotCards, turnNoticePlace } from "./card.js";
@@ -65,6 +65,7 @@ import {
65
65
  appendAboveLiveTiming, formatDuration, mergeTiming, normalizeTiming, sessionDurationNow, STOPPED_LINE_CLASS,
66
66
  } from "./timing.js";
67
67
  import { createLiveTurn } from "./live_turn.js";
68
+ import { applyFinalAnswer, pairAnswerRefs } from "./answer_refs.js";
68
69
  import { installRefClicks } from "./refs.js";
69
70
  import { linkRefs, touchedFiles } from "./ref_scan.js";
70
71
  import { DONE_MS, applyInitEvent, dropDone, initRowHtml, newInitTasks, seedInitTasks } from "./init_row.js";
@@ -904,30 +905,26 @@ async function readTurnEnd() {
904
905
  } catch(_){}
905
906
  }
906
907
 
907
- // The turn's answer as its rendered markdown, once it is final (the
908
- // turn-end read, an answer_display re-read); then its refs.
908
+ // The turn's answer as its rendered markdown and with its plugin refs,
909
+ // once it is final (the turn-end read, an answer_display re-read:
910
+ // answer_refs.js applyFinalAnswer); then the refs.
909
911
  function applyFinalMarkdown(messages) {
910
- const message = [...messages].reverse().find(
911
- (item) => item?.role === "assistant" && typeof item.html === "string",
912
- );
913
- const bubble = message && findInTurnRoots(".bubble.output")
914
- .reverse()
915
- .find((item) => showsAnswer(item, message, normalize));
916
- if (bubble) {
917
- bubble.classList.add("markdown");
918
- bubble.setAttribute(COPY_SOURCE_ATTR, copySource(message));
919
- bubble.innerHTML = message.html;
920
- }
912
+ applyFinalAnswer(messages, findInTurnRoots(".bubble.output"), normalize);
921
913
  linkTextRefs();
922
914
  }
923
915
 
924
- // The `path:line` refs in the answers' and closed steps' texts that name a
925
- // file the session's rows (and cards) touched, on this page (ref_scan.js).
926
- // Not mid-stream: at a turn's end and after a reload's render. A ref
927
- // already made stays as it is.
916
+ // The refs on the answers' and closed steps' texts, on this page: first
917
+ // each answer's own (a plugin's, answer_refs.js), then the `path:line`
918
+ // refs that name a file the session's rows (and cards) touched
919
+ // (ref_scan.js), which leave a plugin's alone. Not mid-stream: at a
920
+ // turn's end and after a reload's render. A ref already made stays as it
921
+ // is.
928
922
  function linkTextRefs() {
929
923
  const roots = turnRoots();
930
924
  const touched = touchedFiles(roots);
925
+ for (const root of roots) {
926
+ for (const el of root.querySelectorAll(".bubble.output")) pairAnswerRefs(el);
927
+ }
931
928
  if (!touched.size) return;
932
929
  for (const root of roots) {
933
930
  for (const el of root.querySelectorAll(".bubble.output, .gen:not(.live) > .gen-text")) linkRefs(el, touched, current.dir);
@@ -382,16 +382,18 @@ button.task-stop:hover:not(:disabled){background:var(--danger-bg-dim);filter:non
382
382
  .diff code{display:block;min-width:max-content}
383
383
  .diff-line{display:block;padding:0 10px;white-space:pre;color:var(--text-2)}
384
384
  /* Refs (refs.js): text until hovered or focused, and only with a target for this viewer (refs.js marks it: data-ref-main, the main one). A link ref keeps its link look. */
385
- .ref[data-ref-main]:not(a):is(:hover,:focus-visible){color:var(--accent);text-decoration:underline dotted;text-underline-offset:3px;cursor:pointer}
386
- .ref[data-ref-main="editor"]:not(a):is(:hover,:focus-visible)::after{content:"\2197";margin-left:.3em;color:var(--text-faint);display:inline-block}
385
+ .ref[data-ref-main]:not(a):is(:hover,:focus-visible,[data-ref-more]){color:var(--accent);text-decoration:underline dotted;text-underline-offset:3px;cursor:pointer}
386
+ .ref[data-ref-main="editor"]:not(a):is(:hover,:focus-visible,[data-ref-more])::after{content:"\2197";margin-left:.3em;color:var(--text-faint);display:inline-block}
387
387
  .ref[data-ref-kind]:focus-visible{outline:2px solid var(--accent);outline-offset:2px;border-radius:3px}
388
388
  :is(.activity-row,.bubble.approval) .diff-line[data-ref-main]:hover{cursor:pointer;box-shadow:inset 2px 0 var(--accent)}
389
389
  :is(.activity-row,.bubble.approval) .diff-line[data-ref-main]:hover::after{content:"\2197 " attr(data-line);float:right;padding-left:12px;color:var(--text-faint)}
390
390
  .activity-row .read-line[data-ref-main]:hover{cursor:pointer;box-shadow:inset 2px 0 var(--accent)}
391
391
  .activity-row .read-line[data-ref-main]:hover::before{color:var(--accent)}
392
392
  /* A ref's ▾ and its menu (ref_menu.js): one each, placed next to the ref. */
393
- .ref-more{position:fixed;z-index:60;display:inline-flex;align-items:center;justify-content:center;width:20px;height:20px;padding:0;border:1px solid var(--border-strong);border-radius:5px;background:var(--bg-panel);color:var(--text-muted);font-size:12px;line-height:1;cursor:pointer;box-shadow:0 1px 3px var(--shadow-strong)}
393
+ .ref-more{position:fixed;z-index:60;display:inline-flex;align-items:center;justify-content:center;width:16px;height:16px;padding:0;border:1px solid var(--border-strong);border-radius:4px;background:var(--bg-panel);color:var(--text-muted);font-size:12px;line-height:1;cursor:pointer;box-shadow:0 1px 3px var(--shadow-strong)}
394
394
  .ref-more[hidden]{display:none}
395
+ /* A ref makes room after itself for its ▾ while it shows (ref_menu.js), and keeps its hover look (its ↗ too) until the ▾ goes: the ▾ never covers what follows it. */
396
+ .ref[data-ref-more]{padding-right:19px}
395
397
  .ref-more:hover{color:var(--accent);border-color:var(--accent)}
396
398
  .ref-menu{position:fixed;z-index:61;min-width:160px;max-width:calc(100vw - 16px);padding:4px;border:1px solid var(--border-strong);border-radius:8px;background:var(--bg-panel);box-shadow:var(--toast-shadow);animation:fadeIn .12s ease-out}
397
399
  .ref-menu [role="menuitem"]{display:block;width:100%;padding:6px 10px;border:0;border-radius:5px;background:none;color:var(--text);font:inherit;font-size:13px;text-align:left;cursor:pointer;white-space:nowrap}
@@ -10,6 +10,9 @@
10
10
  const HIDE_MS = 300;
11
11
  // Kept from the viewport's edges (a phone's width).
12
12
  const EDGE = 8;
13
+ // The ▾ sits in room its ref makes after itself while it shows
14
+ // (data-ref-more, index.html), so it never covers what follows the ref:
15
+ // the next words of an answer, a row's duration.
13
16
 
14
17
  // +doc+'s ▾ and menu. open(el, items, {focus}) shows +items+ ({id, label,
15
18
  // run}) under +el+; showMore(el, onOpen) puts the ▾ after +el+, whose click
@@ -50,18 +53,23 @@ export function createRefMenu(doc = globalThis.document) {
50
53
  function showMore(el, onOpen) {
51
54
  clearTimeout(hideTimer);
52
55
  const button = moreButton();
56
+ if (moreFor && moreFor !== el) delete moreFor.dataset.refMore;
53
57
  moreFor = el;
54
58
  onMore = onOpen;
59
+ el.dataset.refMore = "";
55
60
  button.hidden = false;
56
61
  const rect = lastRect(el);
62
+ const width = button.offsetWidth || 16;
57
63
  const top = rect.top + (rect.height - (button.offsetHeight || rect.height)) / 2;
58
- button.style.left = `${Math.min(rect.right + 2, win.innerWidth - (button.offsetWidth || 16) - 2)}px`;
64
+ const left = rect.right - width - 1;
65
+ button.style.left = `${Math.min(left, win.innerWidth - width - 2)}px`;
59
66
  button.style.top = `${top}px`;
60
67
  }
61
68
 
62
69
  function hideMore() {
63
70
  clearTimeout(hideTimer);
64
71
  if (more) more.hidden = true;
72
+ if (moreFor) delete moreFor.dataset.refMore;
65
73
  moreFor = null;
66
74
  }
67
75
 
@@ -103,13 +103,15 @@ function resolved(ref, touched, cwd) {
103
103
  return path ? { kind: "file", path, line: ref.line, end_line: ref.endLine } : null;
104
104
  }
105
105
 
106
- // The absolute paths of the file refs under +roots+ (the session's rows,
107
- // approval cards): what a text ref may name.
106
+ // The absolute paths of the file refs in the session's rows and approval
107
+ // cards under +roots+: what a text ref may name. Not an answer's own refs
108
+ // (a scanned one repeats a row's; a plugin's file ref names a file the
109
+ // session may not have touched).
108
110
  export function touchedFiles(roots) {
109
111
  const touched = new Set();
110
112
  for (const root of roots) {
111
113
  for (const el of root.querySelectorAll('[data-ref-kind="file"][data-ref-path]')) {
112
- touched.add(el.dataset.refPath);
114
+ if (el.closest(".activity-row, .bubble.approval")) touched.add(el.dataset.refPath);
113
115
  }
114
116
  }
115
117
  return touched;
@@ -34,18 +34,34 @@ export function refAttrs(ref) {
34
34
  // A kind, as the server's Ref allows it (Ref::KIND).
35
35
  const KIND = /^[a-z][a-z0-9_-]{0,31}$/;
36
36
 
37
+ // +kind+ is one bindRef binds.
38
+ export function refKindOk(kind) {
39
+ return typeof kind === "string" && KIND.test(kind);
40
+ }
41
+
37
42
  // A plugin's ref data per element (bindRef): {kind, target, label,
38
43
  // actions}. A WeakMap, so a redrawn element takes its data with it.
39
44
  const REF_DATA = new WeakMap();
40
45
 
41
- // +url+ as the page opens it when it is an http(s) one, else null. The
42
- // server checks it too (Ref, RefAction); the page never opens another
43
- // scheme a plugin gave (javascript:, data:, an editor's).
46
+ // The longest URL the page opens: the server's RefAction::MAX_URL.
47
+ const MAX_URL = 2000;
48
+
49
+ // +url+ as the page opens it when it is an http(s) one with no userinfo
50
+ // and at most MAX_URL characters (trimmed), else null. The server's
51
+ // RefAction.http_url? checks it too; the page is at least as strict on
52
+ // userinfo, and the URL parser refuses an http(s) URL without a host. The
53
+ // page never opens another scheme a plugin gave (javascript:, data:, an
54
+ // editor's), nor a "https://github.com@evil.example/" (one host shown,
55
+ // another opened).
44
56
  export function httpUrl(url) {
45
57
  if (typeof url !== "string") return null;
58
+ const trimmed = url.trim();
59
+ if (trimmed.length > MAX_URL) return null;
46
60
  try {
47
- const parsed = new URL(url.trim());
48
- return parsed.protocol === "http:" || parsed.protocol === "https:" ? parsed.href : null;
61
+ const parsed = new URL(trimmed);
62
+ if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
63
+ if (parsed.username || parsed.password) return null;
64
+ return parsed.href;
49
65
  } catch {
50
66
  return null;
51
67
  }
@@ -58,7 +74,7 @@ export function httpUrl(url) {
58
74
  // nothing. A rebind replaces the target attributes. +el+, or null without one.
59
75
  export function bindRef(el, ref) {
60
76
  if (!el) return null;
61
- if (typeof ref?.kind !== "string" || !KIND.test(ref.kind)) return el;
77
+ if (!refKindOk(ref?.kind)) return el;
62
78
  const target = ref.target || {};
63
79
  el.classList.add("ref");
64
80
  el.dataset.refKind = ref.kind;
@@ -74,6 +90,12 @@ export function bindRef(el, ref) {
74
90
  return el;
75
91
  }
76
92
 
93
+ // The plugin ref bound to +el+ (bindRef's data), or null: a row title or
94
+ // a scanned ref (attributes only) has none.
95
+ export function boundRef(el) {
96
+ return (el && REF_DATA.get(el)) || null;
97
+ }
98
+
77
99
  // The same URL either way (a link's href against an action's URL).
78
100
  function sameUrl(a, b) {
79
101
  const url = httpUrl(a);
@@ -141,12 +141,14 @@ export function createStreamHandlers(deps) {
141
141
  if (display) await display.promise;
142
142
  turnDom.answerReady?.();
143
143
  },
144
- // An after_turn hook presented the answer (AnswerDisplay): it comes after
145
- // turn_completed, so re-read the answer once that read is done.
144
+ // An after_turn hook presented the answer (AnswerDisplay) or attached
145
+ // refs to it (AnswerRefs: refs, a count; display null when only refs):
146
+ // it comes after turn_completed, so re-read the answer once that read
147
+ // is done.
146
148
  answer_display: async (data) => {
147
149
  try {
148
150
  await liveTurn.endRead;
149
- if (!selected() || !data.display) return;
151
+ if (!selected() || (!data.display && !data.refs)) return;
150
152
  const d = await getSession(selected(), { tail: true, recent: true });
151
153
  if (Array.isArray(d.messages)) applyFinalMarkdown(d.messages);
152
154
  } catch (_) {
@@ -21,6 +21,7 @@
21
21
  // chrome (CSS :has), so a plain answer streams like today's bubble, and is
22
22
  // removed at the end: the answer bubble takes its place.
23
23
  import { html, nothing, render } from "./vendor/lit-html.js";
24
+ import { setAnswerRefs } from "./answer_refs.js";
24
25
  import { leavesBlock } from "./card.js";
25
26
  import { copySourceAttr } from "./copy.js";
26
27
  import { escapeHtml, failedTurnText, messageBodyHtml, noteHtml, normalize, sourceLabel, steerRowHtml } from "./format.js";
@@ -779,10 +780,12 @@ export function turnHistoryNodes(items, timing, { thumbs }) {
779
780
  : "";
780
781
  };
781
782
  // The timing line after the bubble, not in it: where the live view
782
- // leaves it at the turn's end.
783
- const answerHtml = (m, timing) => {
783
+ // leaves it at the turn's end. +at+: the message's index, for its refs
784
+ // (data-answer-at, gone before the nodes are returned).
785
+ const answerHtml = (at, timing) => {
786
+ const m = items[at];
784
787
  const { body, renderedMarkdown } = messageBodyHtml(m);
785
- return `<div class="bubble output${renderedMarkdown ? " markdown" : ""}"${copySourceAttr(m, renderedMarkdown)}>${body}</div>${timing}`;
788
+ return `<div class="bubble output${renderedMarkdown ? " markdown" : ""}" data-answer-at="${at}"${copySourceAttr(m, renderedMarkdown)}>${body}</div>${timing}`;
786
789
  };
787
790
  // A user's line, labelled when chi brought it in (a delegate report). A
788
791
  // `!cmd`'s saved message (the worker's "!(<command>)\n<output>") renders
@@ -799,14 +802,14 @@ export function turnHistoryNodes(items, timing, { thumbs }) {
799
802
  const j = items.findIndex((m, k) => k > i && m?.role === "assistant" && String(m.content ?? "").trim());
800
803
  if (j < 0 || j === group.answer || items.slice(i + 1, j).some((m) => m?.role === "user" && !m.merged)) return "";
801
804
  const { body, renderedMarkdown } = messageBodyHtml(items[j]);
802
- return `<div class="bubble output report-reply${renderedMarkdown ? " markdown" : ""}"${copySourceAttr(items[j], renderedMarkdown)}>${body}</div>`;
805
+ return `<div class="bubble output report-reply${renderedMarkdown ? " markdown" : ""}" data-answer-at="${j}"${copySourceAttr(items[j], renderedMarkdown)}>${body}</div>`;
803
806
  };
804
807
  const steerHtml = (i, bubble) => steerRowHtml({ source: items[i].source, text: items[i].content }, { bubble });
805
808
  const parts = [];
806
809
  for (const group of turnGroups(items, timing)) {
807
810
  if (group.kind === "note") { parts.push(`<div class="bubble note">${noteHtml(items[group.i])}</div>`); continue; }
808
811
  if (group.kind === "steer") { parts.push(steerHtml(group.i, true)); continue; }
809
- if (group.kind === "answer") { parts.push(answerHtml(items[group.answer], "")); continue; }
812
+ if (group.kind === "answer") { parts.push(answerHtml(group.answer, "")); continue; }
810
813
  const prompt = items[group.user];
811
814
  // A context wake turn starts with its note.
812
815
  parts.push(prompt.role === "note" ? `<div class="bubble note">${noteHtml(prompt)}</div>` : userHtml(prompt));
@@ -846,11 +849,18 @@ export function turnHistoryNodes(items, timing, { thumbs }) {
846
849
  // A turn with no answer (canceled, or running: no record yet) shows its
847
850
  // timing after its prompt and steps, where the live view leaves it.
848
851
  else if (group.answer === null) parts.push(timingHtml(group));
849
- else parts.push(answerHtml(items[group.answer], timingHtml(group)));
852
+ else parts.push(answerHtml(group.answer, timingHtml(group)));
850
853
  // A canceled turn ends with its cancel line, a failed one whose steps
851
854
  // stayed with its failure line, as the live view left them.
852
855
  parts.push(cancelLineHtml(group.record, escapeHtml));
853
856
  parts.push(failedLineHtml(group.record));
854
857
  }
855
- return fragmentOf(parts);
858
+ const out = fragmentOf(parts);
859
+ // Each answer bubble knows its message's refs (answer_refs.js; the
860
+ // page pairs them after the render), the index stays out of the DOM.
861
+ for (const bubble of out.querySelectorAll("[data-answer-at]")) {
862
+ setAnswerRefs(bubble, items[Number(bubble.dataset.answerAt)]?.display_refs);
863
+ bubble.removeAttribute("data-answer-at");
864
+ }
865
+ return out;
856
866
  }
@@ -12,10 +12,12 @@ require_relative "steer"
12
12
  require_relative "worker_idle_exit"
13
13
  require_relative "worker_wakes"
14
14
  require_relative "worker_inbound"
15
+ require_relative "worker_signals"
15
16
  require_relative "session_manager"
16
17
  require_relative "archive_store"
17
18
  require_relative "log"
18
19
  require_relative "turn_flow"
20
+ require_relative "turn_checkpoint"
19
21
  require_relative "llm/errors"
20
22
  require_relative "continue_offer"
21
23
  require_relative "iteration_limit"
@@ -52,6 +54,13 @@ module Samagotchi
52
54
  # /continue) run on the loop between turns, before any queued prompt. One
53
55
  # taken while a turn runs (at an iteration boundary, or right after the
54
56
  # turn) is refused as busy.
57
+ #
58
+ # A TERM, HUP or INT (WorkerSignals, trapped in a spawned worker only)
59
+ # stops the running turn as a Stop does (reason :signal), the turn's
60
+ # work is saved as after any Stop, and the worker leaves (:signal) taking
61
+ # no new work. A turn that hasn't ended SIGNAL_GRACE seconds after the
62
+ # signal is cut: the session is saved as it is and the signal raised in
63
+ # the loop's thread, as Ruby's default would (the teardown still runs).
55
64
  class Worker
56
65
  FALLBACK_TICK_SECONDS = 5
57
66
  # A worker just started runs no wake turn for delegate reports this
@@ -64,6 +73,18 @@ module Samagotchi
64
73
  BUSY_OUTPUT = "busy: wait for the turn to end"
65
74
  # !rollback mid-turn would mean "the turn I'm in".
66
75
  ROLLBACK_BUSY_OUTPUT = "busy: Ctrl-C the turn first, then !rollback"
76
+ # How long a signalled worker's turn has to end (it is canceled at once,
77
+ # and again every SIGNAL_RETRY until it ends: a signal that lands just
78
+ # before the turn begins finds none to cancel).
79
+ SIGNAL_GRACE = 3.0
80
+ # A second signal, or a TERM after `chi sessions stop` marked the
81
+ # session stopped (that stop already canceled over the Bridge and waits
82
+ # only seconds for the lock): this much.
83
+ SIGNAL_SHORT_GRACE = 0.5
84
+ SIGNAL_RETRY = 0.1
85
+ # After the grace ran out and the signal was raised: the process ends
86
+ # this much later at the latest (a teardown that hangs).
87
+ SIGNAL_HARD_EXIT = 10.0
67
88
 
68
89
  # Wakes the worker loop. Whoever queues work writes it first and wakes
69
90
  # after, and #wait drains every wake before the loop looks for work: a
@@ -90,7 +111,12 @@ module Samagotchi
90
111
 
91
112
  # @param idle_exit_minutes [Numeric, nil] nil: session.idle_exit_minutes
92
113
  # @param poll_interval [Numeric, nil] seconds between fallback ticks
93
- def initialize(session_id:, state_dir:, session_dir:, idle_exit_minutes: nil, poll_interval: nil)
114
+ # @param signals [WorkerSignals, WorkerSignals::None] the process's
115
+ # trapped signals (SessionManager.run_session_loop); none by default
116
+ # @param signal_grace [Array(Numeric, Numeric, Numeric)] the grace, the
117
+ # short grace and the hard exit's delay (specs shorten them)
118
+ def initialize(session_id:, state_dir:, session_dir:, idle_exit_minutes: nil, poll_interval: nil,
119
+ signals: WorkerSignals::NONE, signal_grace: [SIGNAL_GRACE, SIGNAL_SHORT_GRACE, SIGNAL_HARD_EXIT])
94
120
  @session_id = session_id
95
121
  @state_dir = state_dir
96
122
  @session_dir = session_dir
@@ -116,6 +142,20 @@ module Samagotchi
116
142
  @asked_parent = false
117
143
  @owes_parent = false
118
144
  @initial_turn = false
145
+ @signals = signals
146
+ @signal_grace, @signal_short_grace, @signal_hard_exit = signal_grace
147
+ # Set as the loop starts, and as it returns (or raises): a signal's
148
+ # grace runs out only in between.
149
+ @loop_started = false
150
+ @left_loop = false
151
+ @signal_mutex = Mutex.new
152
+ @grace_expired = false
153
+ # The running turn has ended (its end event) and its after_turn hooks
154
+ # run: a signal's grace waits for them (#signalled).
155
+ @turn_ending = false
156
+ # The worker's own saves and a signal's (#signal_grace_expired), one
157
+ # at a time.
158
+ @save_mutex = Mutex.new
119
159
  end
120
160
 
121
161
  # Whether the session was empty as the worker left it, so the caller
@@ -135,30 +175,46 @@ module Samagotchi
135
175
  # @return [Symbol] :idle_exit, :exit_requested when a client asked it
136
176
  # to exit (Bridge POST /exit), :restart when it asked for a restart
137
177
  # (the caller starts a new worker), :stopped when the session was stopped
138
- # (chi stop), :crashed when the loop raised (the session is marked
139
- # errored); SessionManager.run_session_loop turns it into the exit
178
+ # (chi stop), :signal when a signal stopped it (WorkerSignals),
179
+ # :crashed when the loop raised (the session is marked errored);
180
+ # SessionManager.run_session_loop turns it into the exit
140
181
  def run
182
+ # The thread a signal's expired grace raises in (the main thread in a
183
+ # worker process).
184
+ @run_thread = Thread.current
185
+ @signals.on_signal { |name, count| signalled(name, count) }
141
186
  start
142
187
  serve
143
- begin
144
- # A session stopped before this worker took the lock (e.g. a stop
145
- # right after create) must not run its initial prompt.
146
- return :stopped if stopped_on_disk?
147
-
148
- start_background
149
- loop do
150
- leaving = pass
151
- return leaving if leaving
152
- end
188
+ # That raise lands in the loop only (#signal_grace_expired arms once
189
+ # it started, and never after it left): the teardown always runs
190
+ # whole, a raise already on its way waits until it is done.
191
+ Thread.handle_interrupt(SignalException => :never) do
192
+ Thread.handle_interrupt(SignalException => :immediate) { run_loop }
153
193
  rescue StandardError => e
154
194
  crashed(e)
155
195
  ensure
196
+ @signal_mutex.synchronize { @left_loop = true }
156
197
  teardown
157
198
  end
158
199
  end
159
200
 
160
201
  private
161
202
 
203
+ # @return [Symbol] what #run returns
204
+ def run_loop
205
+ @signal_mutex.synchronize { @loop_started = true }
206
+ # A session stopped before this worker took the lock (e.g. a stop
207
+ # right after create) must not run its initial prompt.
208
+ return :stopped if stopped_on_disk?
209
+ return signal_leave if signalled?
210
+
211
+ start_background
212
+ loop do
213
+ leaving = pass
214
+ return leaving if leaving
215
+ end
216
+ end
217
+
162
218
  # The session, its Engine and what runs its turns, before anyone can
163
219
  # reach it.
164
220
  def start
@@ -171,7 +227,10 @@ module Samagotchi
171
227
  ChildMove.adopt(@session_id, state_dir: @state_dir)
172
228
  @session = Session.load(@session_id, state_dir: @state_dir)
173
229
  drop_dead_question
230
+ # The running turn, on disk at its iteration boundaries.
231
+ @checkpoint = TurnCheckpoint.new(session: @session, state_dir: @state_dir)
174
232
  @engine = build_engine
233
+ settle_interrupted_turn
175
234
  # Before the Bridge serves anything: a UI joining a resumed worker's
176
235
  # stream gets the session's history and status in its snapshot, not
177
236
  # an empty session until the first turn.
@@ -200,8 +259,12 @@ module Samagotchi
200
259
  if Events::TURN_END.include?(event[:type])
201
260
  @turn_end_seq = event[:event_seq]
202
261
  @turn_end_type = event[:type]
262
+ @turn_ending = true
263
+ end
264
+ if event[:type] == :turn_started
265
+ @turn_id = event[:turn_id]
266
+ @turn_ending = false
203
267
  end
204
- @turn_id = event[:turn_id] if event[:type] == :turn_started
205
268
  ring_question(event) if event[:type] == :question_requested
206
269
  })
207
270
  # This session's own delegate children's news (rings in children/).
@@ -268,6 +331,9 @@ module Samagotchi
268
331
  # next pass looks again; with none, #idle_pass.
269
332
  # @return [Symbol, nil] what #run returns as the worker leaves, nil to go on
270
333
  def pass
334
+ # A signalled worker takes no new work.
335
+ return signal_leave if signalled?
336
+
271
337
  # Check if the session was externally marked as stopped. Not
272
338
  # stopped_on_disk?, which reads a vanished file as "not stopped":
273
339
  # a session file deleted under a running worker crashes it here
@@ -316,8 +382,8 @@ module Samagotchi
316
382
  # @return [nil]
317
383
  def run_input_files(input_files)
318
384
  input_files.sort.each do |input_file|
319
- # A stop between two queued turns leaves the rest queued.
320
- break if stopped_on_disk?
385
+ # A stop (or a signal) between two queued turns leaves the rest queued.
386
+ break if signalled? || stopped_on_disk?
321
387
 
322
388
  # Commands that came before this file (D1: a queued /model X sent
323
389
  # before this prompt, a command an earlier file carried) run first.
@@ -385,6 +451,17 @@ module Samagotchi
385
451
  engine
386
452
  end
387
453
 
454
+ # A turn an earlier worker died in (TurnCheckpoint's mark in the file):
455
+ # ended now as canceled, with its note, and saved, before the Bridge
456
+ # serves the session (TurnCheckpoint.settle).
457
+ def settle_interrupted_turn
458
+ mark = TurnCheckpoint.settle(@session)
459
+ return unless mark
460
+
461
+ Log.info(:worker, "interrupted_turn_settled", turn_id: mark.id, iteration: mark.iteration, at: mark.at)
462
+ save_or_log(:interrupted_turn) { @session.save(state_dir: @state_dir) }
463
+ end
464
+
388
465
  # A question saved by a worker that died while it waited (an
389
466
  # ask_user_question, an approval, a hook's question): the turn that
390
467
  # asked is gone, so nobody can answer it, and left in the file it
@@ -572,7 +649,8 @@ module Samagotchi
572
649
  # @return [Boolean] whether any command ran
573
650
  def run_queued_commands
574
651
  ran = false
575
- while (command = next_ready_command)
652
+ # A signalled worker runs no more (a /continue would start a turn).
653
+ while !signalled? && (command = next_ready_command)
576
654
  run_command(command)
577
655
  ran = true
578
656
  end
@@ -822,6 +900,7 @@ module Samagotchi
822
900
  @reporting_turn = reports_to_parent?(turn_args[:origin])
823
901
  restores = @turn_flow.restores
824
902
  begin
903
+ @checkpoint.begin_turn(prompt: prompt)
825
904
  @session.status = Session::STATUS_RUNNING
826
905
  # A continue that moved this session to the chain's new link wrote
827
906
  # the new parent beside it (Session::PARENT_FILE): this turn's
@@ -840,6 +919,8 @@ module Samagotchi
840
919
  Log.warn(:worker, "turn_not_begun", error: e.class.name, msg: e.message)
841
920
  end
842
921
  ensure
922
+ # Ended (or never begun): the saves write the session's own messages.
923
+ @checkpoint.end_turn
843
924
  refuse_queued_commands
844
925
  end
845
926
  yield result, error
@@ -995,6 +1076,9 @@ module Samagotchi
995
1076
  reports.map(&:line)
996
1077
  # A wake turn starts with its first line, whoever sent it.
997
1078
  lines[0] = lines[0].with(mark: mark) if mark && lines.any?
1079
+ # The turn so far, with what this boundary merges (its input files
1080
+ # are gone now), on disk at most every TurnCheckpoint::INTERVAL.
1081
+ @checkpoint.boundary(@engine.live_conversation, merging: lines, turn_id: @turn_id)
998
1082
  lines
999
1083
  end
1000
1084
  end
@@ -1127,11 +1211,74 @@ module Samagotchi
1127
1211
  SessionManager.stopped_on_disk?(@session_id, state_dir: @state_dir)
1128
1212
  end
1129
1213
 
1130
- # Save the session unless it was stopped meanwhile: the stop (chi stop,
1131
- # from another process) saved its status, and this save would write
1132
- # the worker's over it.
1214
+ # Save the session, a stopped one too: a stop is the marker file
1215
+ # (Session.mark_stopped), which every load reads over the saved status,
1216
+ # so this save can't hide it, and the turn the stop canceled is kept.
1133
1217
  def save_session
1134
- @session.save(state_dir: @state_dir) unless stopped_on_disk?
1218
+ @save_mutex.synchronize { @session.save(state_dir: @state_dir) }
1219
+ end
1220
+
1221
+ def signalled? = !@signals.signal.nil?
1222
+
1223
+ # @return [Symbol] :stopped when the session was stopped too (chi
1224
+ # sessions stop TERMs its worker), else :signal
1225
+ def signal_leave
1226
+ Session.stopped_marker?(@session_id, state_dir: @state_dir) ? :stopped : :signal
1227
+ end
1228
+
1229
+ # A signal (WorkerSignals#fire), on a thread of its own: the loop wakes
1230
+ # (an idle worker leaves at once), the running turn is canceled as a
1231
+ # Stop (again every SIGNAL_RETRY, until the loop has left), and when
1232
+ # the grace runs out first, #signal_grace_expired. A turn that has
1233
+ # ended and runs its after_turn hooks (which store the answer's display
1234
+ # and refs) is no longer canceled, and gets until SIGNAL_HARD_EXIT
1235
+ # past the grace, the worker's own end of the turn (its saves, after
1236
+ # the turn stopped running) too: the loop leaves next. A grace that runs out before the loop started waits
1237
+ # for it (the loop's first look leaves). A signal after the loop left
1238
+ # (the teardown, the caller's last steps) only brings the hard exit.
1239
+ def signalled(name, count)
1240
+ Log.info(:worker, "signal", signal: name, count: count, turn_running: @engine&.turn_running? || false)
1241
+ return signal_hard_exit if @left_loop
1242
+
1243
+ @waker.wake
1244
+ short = count > 1 || Session.stopped_marker?(@session_id, state_dir: @state_dir)
1245
+ now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
1246
+ deadline = now + (short ? @signal_short_grace : @signal_grace)
1247
+ hooks_deadline = deadline + @signal_hard_exit
1248
+ ended = false
1249
+ until @left_loop
1250
+ running = @engine&.turn_running?
1251
+ ended ||= running && @turn_ending
1252
+ @engine.cancel_current_turn!(:signal) if running && !ended
1253
+ break if @loop_started && Process.clock_gettime(Process::CLOCK_MONOTONIC) >= (ended ? hooks_deadline : deadline)
1254
+
1255
+ sleep(SIGNAL_RETRY)
1256
+ end
1257
+ signal_grace_expired(name) unless @left_loop
1258
+ rescue StandardError => e
1259
+ Log.warn(:worker, "signal_failed", signal: name, error: e.class.name, msg: e.message)
1260
+ end
1261
+
1262
+ # The process ends SIGNAL_HARD_EXIT from now at the latest.
1263
+ def signal_hard_exit
1264
+ sleep(@signal_hard_exit)
1265
+ @signals.hard_exit
1266
+ end
1267
+
1268
+ # The turn didn't end in time (a tool that ignores the cancel, slow
1269
+ # after_turn hooks): the session is saved as it is now (a turn that has
1270
+ # ended: with it), and the signal raised in the loop's thread, so
1271
+ # Worker#run's teardown and the lock's release still run. The process
1272
+ # ends SIGNAL_HARD_EXIT later at the latest. Once, whatever the signals.
1273
+ def signal_grace_expired(name)
1274
+ expired = @signal_mutex.synchronize { !@grace_expired && !@left_loop && (@grace_expired = true) }
1275
+ return unless expired
1276
+
1277
+ Log.warn(:worker, "signal_grace_expired", signal: name, turn_running: @engine&.turn_running? || false)
1278
+ save_or_log(:signal) { save_session }
1279
+ # Not once the loop has left (it may have, during the save).
1280
+ @signal_mutex.synchronize { @run_thread&.raise(SignalException.new(name)) unless @left_loop }
1281
+ signal_hard_exit
1135
1282
  end
1136
1283
  end
1137
1284
  end