samagotchi 0.60.0 → 0.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +41 -1
- data/docs/architecture.md +7 -4
- data/docs/hooks.md +72 -1
- data/docs/internals/engine.md +34 -5
- data/docs/internals/prompt-caching.md +3 -1
- data/docs/plugins.md +7 -3
- data/docs/sessions.md +7 -3
- data/lib/samagotchi/answer_display.rb +4 -3
- data/lib/samagotchi/answer_refs.rb +210 -0
- data/lib/samagotchi/bridge/card_store.rb +6 -1
- data/lib/samagotchi/bundles/check-in/manifest.yml +2 -2
- data/lib/samagotchi/bundles/check-in/plugin.rb +27 -3
- data/lib/samagotchi/bundles/system/manifest.yml +1 -1
- data/lib/samagotchi/engine.rb +38 -9
- data/lib/samagotchi/hooks/registry.rb +3 -1
- data/lib/samagotchi/interrupted_turn.rb +39 -0
- data/lib/samagotchi/kernel_loop.rb +6 -4
- data/lib/samagotchi/llm/chat_loop.rb +5 -2
- data/lib/samagotchi/log_subscriber.rb +6 -3
- data/lib/samagotchi/ref.rb +25 -8
- data/lib/samagotchi/reply_wait.rb +6 -2
- data/lib/samagotchi/session.rb +40 -4
- data/lib/samagotchi/session_manager.rb +64 -6
- data/lib/samagotchi/terminal_ui.rb +4 -0
- data/lib/samagotchi/turn_checkpoint.rb +128 -0
- data/lib/samagotchi/version.rb +1 -1
- data/lib/samagotchi/web/app.rb +5 -0
- data/lib/samagotchi/web/public/answer_refs.js +217 -0
- data/lib/samagotchi/web/public/app.js +15 -18
- data/lib/samagotchi/web/public/index.html +5 -3
- data/lib/samagotchi/web/public/ref_menu.js +9 -1
- data/lib/samagotchi/web/public/ref_scan.js +5 -3
- data/lib/samagotchi/web/public/refs.js +28 -6
- data/lib/samagotchi/web/public/stream_handlers.js +5 -3
- data/lib/samagotchi/web/public/turn_view.js +17 -7
- data/lib/samagotchi/worker.rb +168 -21
- data/lib/samagotchi/worker_signals.rb +81 -0
- metadata +6 -1
|
@@ -46,7 +46,7 @@ import { createTaskStopper } from "./task_stop.js";
|
|
|
46
46
|
import { appendQuote } from "./annotations.js";
|
|
47
47
|
import { parsePresets } from "./annotate_presets.js";
|
|
48
48
|
import { createAnnotateBar } from "./annotate_bar.js";
|
|
49
|
-
import {
|
|
49
|
+
import { copyText, installCopy } from "./copy.js";
|
|
50
50
|
import { ctxBarText, ctxBarTip, lastSpeedText, speedText } from "./ctx.js";
|
|
51
51
|
import { createQuestionCards } from "./question_cards.js";
|
|
52
52
|
import { cardClass, cardInnerHtml, cardPlace, isCard, isNoticeCard, noticeInnerHtml, splitSnapshotCards, turnNoticePlace } from "./card.js";
|
|
@@ -65,6 +65,7 @@ import {
|
|
|
65
65
|
appendAboveLiveTiming, formatDuration, mergeTiming, normalizeTiming, sessionDurationNow, STOPPED_LINE_CLASS,
|
|
66
66
|
} from "./timing.js";
|
|
67
67
|
import { createLiveTurn } from "./live_turn.js";
|
|
68
|
+
import { applyFinalAnswer, pairAnswerRefs } from "./answer_refs.js";
|
|
68
69
|
import { installRefClicks } from "./refs.js";
|
|
69
70
|
import { linkRefs, touchedFiles } from "./ref_scan.js";
|
|
70
71
|
import { DONE_MS, applyInitEvent, dropDone, initRowHtml, newInitTasks, seedInitTasks } from "./init_row.js";
|
|
@@ -904,30 +905,26 @@ async function readTurnEnd() {
|
|
|
904
905
|
} catch(_){}
|
|
905
906
|
}
|
|
906
907
|
|
|
907
|
-
// The turn's answer as its rendered markdown
|
|
908
|
-
// turn-end read, an answer_display re-read
|
|
908
|
+
// The turn's answer as its rendered markdown and with its plugin refs,
|
|
909
|
+
// once it is final (the turn-end read, an answer_display re-read:
|
|
910
|
+
// answer_refs.js applyFinalAnswer); then the refs.
|
|
909
911
|
function applyFinalMarkdown(messages) {
|
|
910
|
-
|
|
911
|
-
(item) => item?.role === "assistant" && typeof item.html === "string",
|
|
912
|
-
);
|
|
913
|
-
const bubble = message && findInTurnRoots(".bubble.output")
|
|
914
|
-
.reverse()
|
|
915
|
-
.find((item) => showsAnswer(item, message, normalize));
|
|
916
|
-
if (bubble) {
|
|
917
|
-
bubble.classList.add("markdown");
|
|
918
|
-
bubble.setAttribute(COPY_SOURCE_ATTR, copySource(message));
|
|
919
|
-
bubble.innerHTML = message.html;
|
|
920
|
-
}
|
|
912
|
+
applyFinalAnswer(messages, findInTurnRoots(".bubble.output"), normalize);
|
|
921
913
|
linkTextRefs();
|
|
922
914
|
}
|
|
923
915
|
|
|
924
|
-
// The
|
|
925
|
-
//
|
|
926
|
-
//
|
|
927
|
-
//
|
|
916
|
+
// The refs on the answers' and closed steps' texts, on this page: first
|
|
917
|
+
// each answer's own (a plugin's, answer_refs.js), then the `path:line`
|
|
918
|
+
// refs that name a file the session's rows (and cards) touched
|
|
919
|
+
// (ref_scan.js), which leave a plugin's alone. Not mid-stream: at a
|
|
920
|
+
// turn's end and after a reload's render. A ref already made stays as it
|
|
921
|
+
// is.
|
|
928
922
|
function linkTextRefs() {
|
|
929
923
|
const roots = turnRoots();
|
|
930
924
|
const touched = touchedFiles(roots);
|
|
925
|
+
for (const root of roots) {
|
|
926
|
+
for (const el of root.querySelectorAll(".bubble.output")) pairAnswerRefs(el);
|
|
927
|
+
}
|
|
931
928
|
if (!touched.size) return;
|
|
932
929
|
for (const root of roots) {
|
|
933
930
|
for (const el of root.querySelectorAll(".bubble.output, .gen:not(.live) > .gen-text")) linkRefs(el, touched, current.dir);
|
|
@@ -382,16 +382,18 @@ button.task-stop:hover:not(:disabled){background:var(--danger-bg-dim);filter:non
|
|
|
382
382
|
.diff code{display:block;min-width:max-content}
|
|
383
383
|
.diff-line{display:block;padding:0 10px;white-space:pre;color:var(--text-2)}
|
|
384
384
|
/* Refs (refs.js): text until hovered or focused, and only with a target for this viewer (refs.js marks it: data-ref-main, the main one). A link ref keeps its link look. */
|
|
385
|
-
.ref[data-ref-main]:not(a):is(:hover,:focus-visible){color:var(--accent);text-decoration:underline dotted;text-underline-offset:3px;cursor:pointer}
|
|
386
|
-
.ref[data-ref-main="editor"]:not(a):is(:hover,:focus-visible)::after{content:"\2197";margin-left:.3em;color:var(--text-faint);display:inline-block}
|
|
385
|
+
.ref[data-ref-main]:not(a):is(:hover,:focus-visible,[data-ref-more]){color:var(--accent);text-decoration:underline dotted;text-underline-offset:3px;cursor:pointer}
|
|
386
|
+
.ref[data-ref-main="editor"]:not(a):is(:hover,:focus-visible,[data-ref-more])::after{content:"\2197";margin-left:.3em;color:var(--text-faint);display:inline-block}
|
|
387
387
|
.ref[data-ref-kind]:focus-visible{outline:2px solid var(--accent);outline-offset:2px;border-radius:3px}
|
|
388
388
|
:is(.activity-row,.bubble.approval) .diff-line[data-ref-main]:hover{cursor:pointer;box-shadow:inset 2px 0 var(--accent)}
|
|
389
389
|
:is(.activity-row,.bubble.approval) .diff-line[data-ref-main]:hover::after{content:"\2197 " attr(data-line);float:right;padding-left:12px;color:var(--text-faint)}
|
|
390
390
|
.activity-row .read-line[data-ref-main]:hover{cursor:pointer;box-shadow:inset 2px 0 var(--accent)}
|
|
391
391
|
.activity-row .read-line[data-ref-main]:hover::before{color:var(--accent)}
|
|
392
392
|
/* A ref's ▾ and its menu (ref_menu.js): one each, placed next to the ref. */
|
|
393
|
-
.ref-more{position:fixed;z-index:60;display:inline-flex;align-items:center;justify-content:center;width:
|
|
393
|
+
.ref-more{position:fixed;z-index:60;display:inline-flex;align-items:center;justify-content:center;width:16px;height:16px;padding:0;border:1px solid var(--border-strong);border-radius:4px;background:var(--bg-panel);color:var(--text-muted);font-size:12px;line-height:1;cursor:pointer;box-shadow:0 1px 3px var(--shadow-strong)}
|
|
394
394
|
.ref-more[hidden]{display:none}
|
|
395
|
+
/* A ref makes room after itself for its ▾ while it shows (ref_menu.js), and keeps its hover look (its ↗ too) until the ▾ goes: the ▾ never covers what follows it. */
|
|
396
|
+
.ref[data-ref-more]{padding-right:19px}
|
|
395
397
|
.ref-more:hover{color:var(--accent);border-color:var(--accent)}
|
|
396
398
|
.ref-menu{position:fixed;z-index:61;min-width:160px;max-width:calc(100vw - 16px);padding:4px;border:1px solid var(--border-strong);border-radius:8px;background:var(--bg-panel);box-shadow:var(--toast-shadow);animation:fadeIn .12s ease-out}
|
|
397
399
|
.ref-menu [role="menuitem"]{display:block;width:100%;padding:6px 10px;border:0;border-radius:5px;background:none;color:var(--text);font:inherit;font-size:13px;text-align:left;cursor:pointer;white-space:nowrap}
|
|
@@ -10,6 +10,9 @@
|
|
|
10
10
|
const HIDE_MS = 300;
|
|
11
11
|
// Kept from the viewport's edges (a phone's width).
|
|
12
12
|
const EDGE = 8;
|
|
13
|
+
// The ▾ sits in room its ref makes after itself while it shows
|
|
14
|
+
// (data-ref-more, index.html), so it never covers what follows the ref:
|
|
15
|
+
// the next words of an answer, a row's duration.
|
|
13
16
|
|
|
14
17
|
// +doc+'s ▾ and menu. open(el, items, {focus}) shows +items+ ({id, label,
|
|
15
18
|
// run}) under +el+; showMore(el, onOpen) puts the ▾ after +el+, whose click
|
|
@@ -50,18 +53,23 @@ export function createRefMenu(doc = globalThis.document) {
|
|
|
50
53
|
function showMore(el, onOpen) {
|
|
51
54
|
clearTimeout(hideTimer);
|
|
52
55
|
const button = moreButton();
|
|
56
|
+
if (moreFor && moreFor !== el) delete moreFor.dataset.refMore;
|
|
53
57
|
moreFor = el;
|
|
54
58
|
onMore = onOpen;
|
|
59
|
+
el.dataset.refMore = "";
|
|
55
60
|
button.hidden = false;
|
|
56
61
|
const rect = lastRect(el);
|
|
62
|
+
const width = button.offsetWidth || 16;
|
|
57
63
|
const top = rect.top + (rect.height - (button.offsetHeight || rect.height)) / 2;
|
|
58
|
-
|
|
64
|
+
const left = rect.right - width - 1;
|
|
65
|
+
button.style.left = `${Math.min(left, win.innerWidth - width - 2)}px`;
|
|
59
66
|
button.style.top = `${top}px`;
|
|
60
67
|
}
|
|
61
68
|
|
|
62
69
|
function hideMore() {
|
|
63
70
|
clearTimeout(hideTimer);
|
|
64
71
|
if (more) more.hidden = true;
|
|
72
|
+
if (moreFor) delete moreFor.dataset.refMore;
|
|
65
73
|
moreFor = null;
|
|
66
74
|
}
|
|
67
75
|
|
|
@@ -103,13 +103,15 @@ function resolved(ref, touched, cwd) {
|
|
|
103
103
|
return path ? { kind: "file", path, line: ref.line, end_line: ref.endLine } : null;
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
-
// The absolute paths of the file refs
|
|
107
|
-
//
|
|
106
|
+
// The absolute paths of the file refs in the session's rows and approval
|
|
107
|
+
// cards under +roots+: what a text ref may name. Not an answer's own refs
|
|
108
|
+
// (a scanned one repeats a row's; a plugin's file ref names a file the
|
|
109
|
+
// session may not have touched).
|
|
108
110
|
export function touchedFiles(roots) {
|
|
109
111
|
const touched = new Set();
|
|
110
112
|
for (const root of roots) {
|
|
111
113
|
for (const el of root.querySelectorAll('[data-ref-kind="file"][data-ref-path]')) {
|
|
112
|
-
touched.add(el.dataset.refPath);
|
|
114
|
+
if (el.closest(".activity-row, .bubble.approval")) touched.add(el.dataset.refPath);
|
|
113
115
|
}
|
|
114
116
|
}
|
|
115
117
|
return touched;
|
|
@@ -34,18 +34,34 @@ export function refAttrs(ref) {
|
|
|
34
34
|
// A kind, as the server's Ref allows it (Ref::KIND).
|
|
35
35
|
const KIND = /^[a-z][a-z0-9_-]{0,31}$/;
|
|
36
36
|
|
|
37
|
+
// +kind+ is one bindRef binds.
|
|
38
|
+
export function refKindOk(kind) {
|
|
39
|
+
return typeof kind === "string" && KIND.test(kind);
|
|
40
|
+
}
|
|
41
|
+
|
|
37
42
|
// A plugin's ref data per element (bindRef): {kind, target, label,
|
|
38
43
|
// actions}. A WeakMap, so a redrawn element takes its data with it.
|
|
39
44
|
const REF_DATA = new WeakMap();
|
|
40
45
|
|
|
41
|
-
//
|
|
42
|
-
|
|
43
|
-
|
|
46
|
+
// The longest URL the page opens: the server's RefAction::MAX_URL.
|
|
47
|
+
const MAX_URL = 2000;
|
|
48
|
+
|
|
49
|
+
// +url+ as the page opens it when it is an http(s) one with no userinfo
|
|
50
|
+
// and at most MAX_URL characters (trimmed), else null. The server's
|
|
51
|
+
// RefAction.http_url? checks it too; the page is at least as strict on
|
|
52
|
+
// userinfo, and the URL parser refuses an http(s) URL without a host. The
|
|
53
|
+
// page never opens another scheme a plugin gave (javascript:, data:, an
|
|
54
|
+
// editor's), nor a "https://github.com@evil.example/" (one host shown,
|
|
55
|
+
// another opened).
|
|
44
56
|
export function httpUrl(url) {
|
|
45
57
|
if (typeof url !== "string") return null;
|
|
58
|
+
const trimmed = url.trim();
|
|
59
|
+
if (trimmed.length > MAX_URL) return null;
|
|
46
60
|
try {
|
|
47
|
-
const parsed = new URL(
|
|
48
|
-
|
|
61
|
+
const parsed = new URL(trimmed);
|
|
62
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
|
|
63
|
+
if (parsed.username || parsed.password) return null;
|
|
64
|
+
return parsed.href;
|
|
49
65
|
} catch {
|
|
50
66
|
return null;
|
|
51
67
|
}
|
|
@@ -58,7 +74,7 @@ export function httpUrl(url) {
|
|
|
58
74
|
// nothing. A rebind replaces the target attributes. +el+, or null without one.
|
|
59
75
|
export function bindRef(el, ref) {
|
|
60
76
|
if (!el) return null;
|
|
61
|
-
if (
|
|
77
|
+
if (!refKindOk(ref?.kind)) return el;
|
|
62
78
|
const target = ref.target || {};
|
|
63
79
|
el.classList.add("ref");
|
|
64
80
|
el.dataset.refKind = ref.kind;
|
|
@@ -74,6 +90,12 @@ export function bindRef(el, ref) {
|
|
|
74
90
|
return el;
|
|
75
91
|
}
|
|
76
92
|
|
|
93
|
+
// The plugin ref bound to +el+ (bindRef's data), or null: a row title or
|
|
94
|
+
// a scanned ref (attributes only) has none.
|
|
95
|
+
export function boundRef(el) {
|
|
96
|
+
return (el && REF_DATA.get(el)) || null;
|
|
97
|
+
}
|
|
98
|
+
|
|
77
99
|
// The same URL either way (a link's href against an action's URL).
|
|
78
100
|
function sameUrl(a, b) {
|
|
79
101
|
const url = httpUrl(a);
|
|
@@ -141,12 +141,14 @@ export function createStreamHandlers(deps) {
|
|
|
141
141
|
if (display) await display.promise;
|
|
142
142
|
turnDom.answerReady?.();
|
|
143
143
|
},
|
|
144
|
-
// An after_turn hook presented the answer (AnswerDisplay)
|
|
145
|
-
//
|
|
144
|
+
// An after_turn hook presented the answer (AnswerDisplay) or attached
|
|
145
|
+
// refs to it (AnswerRefs: refs, a count; display null when only refs):
|
|
146
|
+
// it comes after turn_completed, so re-read the answer once that read
|
|
147
|
+
// is done.
|
|
146
148
|
answer_display: async (data) => {
|
|
147
149
|
try {
|
|
148
150
|
await liveTurn.endRead;
|
|
149
|
-
if (!selected() || !data.display) return;
|
|
151
|
+
if (!selected() || (!data.display && !data.refs)) return;
|
|
150
152
|
const d = await getSession(selected(), { tail: true, recent: true });
|
|
151
153
|
if (Array.isArray(d.messages)) applyFinalMarkdown(d.messages);
|
|
152
154
|
} catch (_) {
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
// chrome (CSS :has), so a plain answer streams like today's bubble, and is
|
|
22
22
|
// removed at the end: the answer bubble takes its place.
|
|
23
23
|
import { html, nothing, render } from "./vendor/lit-html.js";
|
|
24
|
+
import { setAnswerRefs } from "./answer_refs.js";
|
|
24
25
|
import { leavesBlock } from "./card.js";
|
|
25
26
|
import { copySourceAttr } from "./copy.js";
|
|
26
27
|
import { escapeHtml, failedTurnText, messageBodyHtml, noteHtml, normalize, sourceLabel, steerRowHtml } from "./format.js";
|
|
@@ -779,10 +780,12 @@ export function turnHistoryNodes(items, timing, { thumbs }) {
|
|
|
779
780
|
: "";
|
|
780
781
|
};
|
|
781
782
|
// The timing line after the bubble, not in it: where the live view
|
|
782
|
-
// leaves it at the turn's end.
|
|
783
|
-
|
|
783
|
+
// leaves it at the turn's end. +at+: the message's index, for its refs
|
|
784
|
+
// (data-answer-at, gone before the nodes are returned).
|
|
785
|
+
const answerHtml = (at, timing) => {
|
|
786
|
+
const m = items[at];
|
|
784
787
|
const { body, renderedMarkdown } = messageBodyHtml(m);
|
|
785
|
-
return `<div class="bubble output${renderedMarkdown ? " markdown" : ""}"${copySourceAttr(m, renderedMarkdown)}>${body}</div>${timing}`;
|
|
788
|
+
return `<div class="bubble output${renderedMarkdown ? " markdown" : ""}" data-answer-at="${at}"${copySourceAttr(m, renderedMarkdown)}>${body}</div>${timing}`;
|
|
786
789
|
};
|
|
787
790
|
// A user's line, labelled when chi brought it in (a delegate report). A
|
|
788
791
|
// `!cmd`'s saved message (the worker's "!(<command>)\n<output>") renders
|
|
@@ -799,14 +802,14 @@ export function turnHistoryNodes(items, timing, { thumbs }) {
|
|
|
799
802
|
const j = items.findIndex((m, k) => k > i && m?.role === "assistant" && String(m.content ?? "").trim());
|
|
800
803
|
if (j < 0 || j === group.answer || items.slice(i + 1, j).some((m) => m?.role === "user" && !m.merged)) return "";
|
|
801
804
|
const { body, renderedMarkdown } = messageBodyHtml(items[j]);
|
|
802
|
-
return `<div class="bubble output report-reply${renderedMarkdown ? " markdown" : ""}"${copySourceAttr(items[j], renderedMarkdown)}>${body}</div>`;
|
|
805
|
+
return `<div class="bubble output report-reply${renderedMarkdown ? " markdown" : ""}" data-answer-at="${j}"${copySourceAttr(items[j], renderedMarkdown)}>${body}</div>`;
|
|
803
806
|
};
|
|
804
807
|
const steerHtml = (i, bubble) => steerRowHtml({ source: items[i].source, text: items[i].content }, { bubble });
|
|
805
808
|
const parts = [];
|
|
806
809
|
for (const group of turnGroups(items, timing)) {
|
|
807
810
|
if (group.kind === "note") { parts.push(`<div class="bubble note">${noteHtml(items[group.i])}</div>`); continue; }
|
|
808
811
|
if (group.kind === "steer") { parts.push(steerHtml(group.i, true)); continue; }
|
|
809
|
-
if (group.kind === "answer") { parts.push(answerHtml(
|
|
812
|
+
if (group.kind === "answer") { parts.push(answerHtml(group.answer, "")); continue; }
|
|
810
813
|
const prompt = items[group.user];
|
|
811
814
|
// A context wake turn starts with its note.
|
|
812
815
|
parts.push(prompt.role === "note" ? `<div class="bubble note">${noteHtml(prompt)}</div>` : userHtml(prompt));
|
|
@@ -846,11 +849,18 @@ export function turnHistoryNodes(items, timing, { thumbs }) {
|
|
|
846
849
|
// A turn with no answer (canceled, or running: no record yet) shows its
|
|
847
850
|
// timing after its prompt and steps, where the live view leaves it.
|
|
848
851
|
else if (group.answer === null) parts.push(timingHtml(group));
|
|
849
|
-
else parts.push(answerHtml(
|
|
852
|
+
else parts.push(answerHtml(group.answer, timingHtml(group)));
|
|
850
853
|
// A canceled turn ends with its cancel line, a failed one whose steps
|
|
851
854
|
// stayed with its failure line, as the live view left them.
|
|
852
855
|
parts.push(cancelLineHtml(group.record, escapeHtml));
|
|
853
856
|
parts.push(failedLineHtml(group.record));
|
|
854
857
|
}
|
|
855
|
-
|
|
858
|
+
const out = fragmentOf(parts);
|
|
859
|
+
// Each answer bubble knows its message's refs (answer_refs.js; the
|
|
860
|
+
// page pairs them after the render), the index stays out of the DOM.
|
|
861
|
+
for (const bubble of out.querySelectorAll("[data-answer-at]")) {
|
|
862
|
+
setAnswerRefs(bubble, items[Number(bubble.dataset.answerAt)]?.display_refs);
|
|
863
|
+
bubble.removeAttribute("data-answer-at");
|
|
864
|
+
}
|
|
865
|
+
return out;
|
|
856
866
|
}
|
data/lib/samagotchi/worker.rb
CHANGED
|
@@ -12,10 +12,12 @@ require_relative "steer"
|
|
|
12
12
|
require_relative "worker_idle_exit"
|
|
13
13
|
require_relative "worker_wakes"
|
|
14
14
|
require_relative "worker_inbound"
|
|
15
|
+
require_relative "worker_signals"
|
|
15
16
|
require_relative "session_manager"
|
|
16
17
|
require_relative "archive_store"
|
|
17
18
|
require_relative "log"
|
|
18
19
|
require_relative "turn_flow"
|
|
20
|
+
require_relative "turn_checkpoint"
|
|
19
21
|
require_relative "llm/errors"
|
|
20
22
|
require_relative "continue_offer"
|
|
21
23
|
require_relative "iteration_limit"
|
|
@@ -52,6 +54,13 @@ module Samagotchi
|
|
|
52
54
|
# /continue) run on the loop between turns, before any queued prompt. One
|
|
53
55
|
# taken while a turn runs (at an iteration boundary, or right after the
|
|
54
56
|
# turn) is refused as busy.
|
|
57
|
+
#
|
|
58
|
+
# A TERM, HUP or INT (WorkerSignals, trapped in a spawned worker only)
|
|
59
|
+
# stops the running turn as a Stop does (reason :signal), the turn's
|
|
60
|
+
# work is saved as after any Stop, and the worker leaves (:signal) taking
|
|
61
|
+
# no new work. A turn that hasn't ended SIGNAL_GRACE seconds after the
|
|
62
|
+
# signal is cut: the session is saved as it is and the signal raised in
|
|
63
|
+
# the loop's thread, as Ruby's default would (the teardown still runs).
|
|
55
64
|
class Worker
|
|
56
65
|
FALLBACK_TICK_SECONDS = 5
|
|
57
66
|
# A worker just started runs no wake turn for delegate reports this
|
|
@@ -64,6 +73,18 @@ module Samagotchi
|
|
|
64
73
|
BUSY_OUTPUT = "busy: wait for the turn to end"
|
|
65
74
|
# !rollback mid-turn would mean "the turn I'm in".
|
|
66
75
|
ROLLBACK_BUSY_OUTPUT = "busy: Ctrl-C the turn first, then !rollback"
|
|
76
|
+
# How long a signalled worker's turn has to end (it is canceled at once,
|
|
77
|
+
# and again every SIGNAL_RETRY until it ends: a signal that lands just
|
|
78
|
+
# before the turn begins finds none to cancel).
|
|
79
|
+
SIGNAL_GRACE = 3.0
|
|
80
|
+
# A second signal, or a TERM after `chi sessions stop` marked the
|
|
81
|
+
# session stopped (that stop already canceled over the Bridge and waits
|
|
82
|
+
# only seconds for the lock): this much.
|
|
83
|
+
SIGNAL_SHORT_GRACE = 0.5
|
|
84
|
+
SIGNAL_RETRY = 0.1
|
|
85
|
+
# After the grace ran out and the signal was raised: the process ends
|
|
86
|
+
# this much later at the latest (a teardown that hangs).
|
|
87
|
+
SIGNAL_HARD_EXIT = 10.0
|
|
67
88
|
|
|
68
89
|
# Wakes the worker loop. Whoever queues work writes it first and wakes
|
|
69
90
|
# after, and #wait drains every wake before the loop looks for work: a
|
|
@@ -90,7 +111,12 @@ module Samagotchi
|
|
|
90
111
|
|
|
91
112
|
# @param idle_exit_minutes [Numeric, nil] nil: session.idle_exit_minutes
|
|
92
113
|
# @param poll_interval [Numeric, nil] seconds between fallback ticks
|
|
93
|
-
|
|
114
|
+
# @param signals [WorkerSignals, WorkerSignals::None] the process's
|
|
115
|
+
# trapped signals (SessionManager.run_session_loop); none by default
|
|
116
|
+
# @param signal_grace [Array(Numeric, Numeric, Numeric)] the grace, the
|
|
117
|
+
# short grace and the hard exit's delay (specs shorten them)
|
|
118
|
+
def initialize(session_id:, state_dir:, session_dir:, idle_exit_minutes: nil, poll_interval: nil,
|
|
119
|
+
signals: WorkerSignals::NONE, signal_grace: [SIGNAL_GRACE, SIGNAL_SHORT_GRACE, SIGNAL_HARD_EXIT])
|
|
94
120
|
@session_id = session_id
|
|
95
121
|
@state_dir = state_dir
|
|
96
122
|
@session_dir = session_dir
|
|
@@ -116,6 +142,20 @@ module Samagotchi
|
|
|
116
142
|
@asked_parent = false
|
|
117
143
|
@owes_parent = false
|
|
118
144
|
@initial_turn = false
|
|
145
|
+
@signals = signals
|
|
146
|
+
@signal_grace, @signal_short_grace, @signal_hard_exit = signal_grace
|
|
147
|
+
# Set as the loop starts, and as it returns (or raises): a signal's
|
|
148
|
+
# grace runs out only in between.
|
|
149
|
+
@loop_started = false
|
|
150
|
+
@left_loop = false
|
|
151
|
+
@signal_mutex = Mutex.new
|
|
152
|
+
@grace_expired = false
|
|
153
|
+
# The running turn has ended (its end event) and its after_turn hooks
|
|
154
|
+
# run: a signal's grace waits for them (#signalled).
|
|
155
|
+
@turn_ending = false
|
|
156
|
+
# The worker's own saves and a signal's (#signal_grace_expired), one
|
|
157
|
+
# at a time.
|
|
158
|
+
@save_mutex = Mutex.new
|
|
119
159
|
end
|
|
120
160
|
|
|
121
161
|
# Whether the session was empty as the worker left it, so the caller
|
|
@@ -135,30 +175,46 @@ module Samagotchi
|
|
|
135
175
|
# @return [Symbol] :idle_exit, :exit_requested when a client asked it
|
|
136
176
|
# to exit (Bridge POST /exit), :restart when it asked for a restart
|
|
137
177
|
# (the caller starts a new worker), :stopped when the session was stopped
|
|
138
|
-
# (chi stop), :
|
|
139
|
-
#
|
|
178
|
+
# (chi stop), :signal when a signal stopped it (WorkerSignals),
|
|
179
|
+
# :crashed when the loop raised (the session is marked errored);
|
|
180
|
+
# SessionManager.run_session_loop turns it into the exit
|
|
140
181
|
def run
|
|
182
|
+
# The thread a signal's expired grace raises in (the main thread in a
|
|
183
|
+
# worker process).
|
|
184
|
+
@run_thread = Thread.current
|
|
185
|
+
@signals.on_signal { |name, count| signalled(name, count) }
|
|
141
186
|
start
|
|
142
187
|
serve
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
start_background
|
|
149
|
-
loop do
|
|
150
|
-
leaving = pass
|
|
151
|
-
return leaving if leaving
|
|
152
|
-
end
|
|
188
|
+
# That raise lands in the loop only (#signal_grace_expired arms once
|
|
189
|
+
# it started, and never after it left): the teardown always runs
|
|
190
|
+
# whole, a raise already on its way waits until it is done.
|
|
191
|
+
Thread.handle_interrupt(SignalException => :never) do
|
|
192
|
+
Thread.handle_interrupt(SignalException => :immediate) { run_loop }
|
|
153
193
|
rescue StandardError => e
|
|
154
194
|
crashed(e)
|
|
155
195
|
ensure
|
|
196
|
+
@signal_mutex.synchronize { @left_loop = true }
|
|
156
197
|
teardown
|
|
157
198
|
end
|
|
158
199
|
end
|
|
159
200
|
|
|
160
201
|
private
|
|
161
202
|
|
|
203
|
+
# @return [Symbol] what #run returns
|
|
204
|
+
def run_loop
|
|
205
|
+
@signal_mutex.synchronize { @loop_started = true }
|
|
206
|
+
# A session stopped before this worker took the lock (e.g. a stop
|
|
207
|
+
# right after create) must not run its initial prompt.
|
|
208
|
+
return :stopped if stopped_on_disk?
|
|
209
|
+
return signal_leave if signalled?
|
|
210
|
+
|
|
211
|
+
start_background
|
|
212
|
+
loop do
|
|
213
|
+
leaving = pass
|
|
214
|
+
return leaving if leaving
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
|
|
162
218
|
# The session, its Engine and what runs its turns, before anyone can
|
|
163
219
|
# reach it.
|
|
164
220
|
def start
|
|
@@ -171,7 +227,10 @@ module Samagotchi
|
|
|
171
227
|
ChildMove.adopt(@session_id, state_dir: @state_dir)
|
|
172
228
|
@session = Session.load(@session_id, state_dir: @state_dir)
|
|
173
229
|
drop_dead_question
|
|
230
|
+
# The running turn, on disk at its iteration boundaries.
|
|
231
|
+
@checkpoint = TurnCheckpoint.new(session: @session, state_dir: @state_dir)
|
|
174
232
|
@engine = build_engine
|
|
233
|
+
settle_interrupted_turn
|
|
175
234
|
# Before the Bridge serves anything: a UI joining a resumed worker's
|
|
176
235
|
# stream gets the session's history and status in its snapshot, not
|
|
177
236
|
# an empty session until the first turn.
|
|
@@ -200,8 +259,12 @@ module Samagotchi
|
|
|
200
259
|
if Events::TURN_END.include?(event[:type])
|
|
201
260
|
@turn_end_seq = event[:event_seq]
|
|
202
261
|
@turn_end_type = event[:type]
|
|
262
|
+
@turn_ending = true
|
|
263
|
+
end
|
|
264
|
+
if event[:type] == :turn_started
|
|
265
|
+
@turn_id = event[:turn_id]
|
|
266
|
+
@turn_ending = false
|
|
203
267
|
end
|
|
204
|
-
@turn_id = event[:turn_id] if event[:type] == :turn_started
|
|
205
268
|
ring_question(event) if event[:type] == :question_requested
|
|
206
269
|
})
|
|
207
270
|
# This session's own delegate children's news (rings in children/).
|
|
@@ -268,6 +331,9 @@ module Samagotchi
|
|
|
268
331
|
# next pass looks again; with none, #idle_pass.
|
|
269
332
|
# @return [Symbol, nil] what #run returns as the worker leaves, nil to go on
|
|
270
333
|
def pass
|
|
334
|
+
# A signalled worker takes no new work.
|
|
335
|
+
return signal_leave if signalled?
|
|
336
|
+
|
|
271
337
|
# Check if the session was externally marked as stopped. Not
|
|
272
338
|
# stopped_on_disk?, which reads a vanished file as "not stopped":
|
|
273
339
|
# a session file deleted under a running worker crashes it here
|
|
@@ -316,8 +382,8 @@ module Samagotchi
|
|
|
316
382
|
# @return [nil]
|
|
317
383
|
def run_input_files(input_files)
|
|
318
384
|
input_files.sort.each do |input_file|
|
|
319
|
-
# A stop between two queued turns leaves the rest queued.
|
|
320
|
-
break if stopped_on_disk?
|
|
385
|
+
# A stop (or a signal) between two queued turns leaves the rest queued.
|
|
386
|
+
break if signalled? || stopped_on_disk?
|
|
321
387
|
|
|
322
388
|
# Commands that came before this file (D1: a queued /model X sent
|
|
323
389
|
# before this prompt, a command an earlier file carried) run first.
|
|
@@ -385,6 +451,17 @@ module Samagotchi
|
|
|
385
451
|
engine
|
|
386
452
|
end
|
|
387
453
|
|
|
454
|
+
# A turn an earlier worker died in (TurnCheckpoint's mark in the file):
|
|
455
|
+
# ended now as canceled, with its note, and saved, before the Bridge
|
|
456
|
+
# serves the session (TurnCheckpoint.settle).
|
|
457
|
+
def settle_interrupted_turn
|
|
458
|
+
mark = TurnCheckpoint.settle(@session)
|
|
459
|
+
return unless mark
|
|
460
|
+
|
|
461
|
+
Log.info(:worker, "interrupted_turn_settled", turn_id: mark.id, iteration: mark.iteration, at: mark.at)
|
|
462
|
+
save_or_log(:interrupted_turn) { @session.save(state_dir: @state_dir) }
|
|
463
|
+
end
|
|
464
|
+
|
|
388
465
|
# A question saved by a worker that died while it waited (an
|
|
389
466
|
# ask_user_question, an approval, a hook's question): the turn that
|
|
390
467
|
# asked is gone, so nobody can answer it, and left in the file it
|
|
@@ -572,7 +649,8 @@ module Samagotchi
|
|
|
572
649
|
# @return [Boolean] whether any command ran
|
|
573
650
|
def run_queued_commands
|
|
574
651
|
ran = false
|
|
575
|
-
|
|
652
|
+
# A signalled worker runs no more (a /continue would start a turn).
|
|
653
|
+
while !signalled? && (command = next_ready_command)
|
|
576
654
|
run_command(command)
|
|
577
655
|
ran = true
|
|
578
656
|
end
|
|
@@ -822,6 +900,7 @@ module Samagotchi
|
|
|
822
900
|
@reporting_turn = reports_to_parent?(turn_args[:origin])
|
|
823
901
|
restores = @turn_flow.restores
|
|
824
902
|
begin
|
|
903
|
+
@checkpoint.begin_turn(prompt: prompt)
|
|
825
904
|
@session.status = Session::STATUS_RUNNING
|
|
826
905
|
# A continue that moved this session to the chain's new link wrote
|
|
827
906
|
# the new parent beside it (Session::PARENT_FILE): this turn's
|
|
@@ -840,6 +919,8 @@ module Samagotchi
|
|
|
840
919
|
Log.warn(:worker, "turn_not_begun", error: e.class.name, msg: e.message)
|
|
841
920
|
end
|
|
842
921
|
ensure
|
|
922
|
+
# Ended (or never begun): the saves write the session's own messages.
|
|
923
|
+
@checkpoint.end_turn
|
|
843
924
|
refuse_queued_commands
|
|
844
925
|
end
|
|
845
926
|
yield result, error
|
|
@@ -995,6 +1076,9 @@ module Samagotchi
|
|
|
995
1076
|
reports.map(&:line)
|
|
996
1077
|
# A wake turn starts with its first line, whoever sent it.
|
|
997
1078
|
lines[0] = lines[0].with(mark: mark) if mark && lines.any?
|
|
1079
|
+
# The turn so far, with what this boundary merges (its input files
|
|
1080
|
+
# are gone now), on disk at most every TurnCheckpoint::INTERVAL.
|
|
1081
|
+
@checkpoint.boundary(@engine.live_conversation, merging: lines, turn_id: @turn_id)
|
|
998
1082
|
lines
|
|
999
1083
|
end
|
|
1000
1084
|
end
|
|
@@ -1127,11 +1211,74 @@ module Samagotchi
|
|
|
1127
1211
|
SessionManager.stopped_on_disk?(@session_id, state_dir: @state_dir)
|
|
1128
1212
|
end
|
|
1129
1213
|
|
|
1130
|
-
# Save the session
|
|
1131
|
-
#
|
|
1132
|
-
#
|
|
1214
|
+
# Save the session, a stopped one too: a stop is the marker file
|
|
1215
|
+
# (Session.mark_stopped), which every load reads over the saved status,
|
|
1216
|
+
# so this save can't hide it, and the turn the stop canceled is kept.
|
|
1133
1217
|
def save_session
|
|
1134
|
-
@session.save(state_dir: @state_dir)
|
|
1218
|
+
@save_mutex.synchronize { @session.save(state_dir: @state_dir) }
|
|
1219
|
+
end
|
|
1220
|
+
|
|
1221
|
+
def signalled? = !@signals.signal.nil?
|
|
1222
|
+
|
|
1223
|
+
# @return [Symbol] :stopped when the session was stopped too (chi
|
|
1224
|
+
# sessions stop TERMs its worker), else :signal
|
|
1225
|
+
def signal_leave
|
|
1226
|
+
Session.stopped_marker?(@session_id, state_dir: @state_dir) ? :stopped : :signal
|
|
1227
|
+
end
|
|
1228
|
+
|
|
1229
|
+
# A signal (WorkerSignals#fire), on a thread of its own: the loop wakes
|
|
1230
|
+
# (an idle worker leaves at once), the running turn is canceled as a
|
|
1231
|
+
# Stop (again every SIGNAL_RETRY, until the loop has left), and when
|
|
1232
|
+
# the grace runs out first, #signal_grace_expired. A turn that has
|
|
1233
|
+
# ended and runs its after_turn hooks (which store the answer's display
|
|
1234
|
+
# and refs) is no longer canceled, and gets until SIGNAL_HARD_EXIT
|
|
1235
|
+
# past the grace, the worker's own end of the turn (its saves, after
|
|
1236
|
+
# the turn stopped running) too: the loop leaves next. A grace that runs out before the loop started waits
|
|
1237
|
+
# for it (the loop's first look leaves). A signal after the loop left
|
|
1238
|
+
# (the teardown, the caller's last steps) only brings the hard exit.
|
|
1239
|
+
def signalled(name, count)
|
|
1240
|
+
Log.info(:worker, "signal", signal: name, count: count, turn_running: @engine&.turn_running? || false)
|
|
1241
|
+
return signal_hard_exit if @left_loop
|
|
1242
|
+
|
|
1243
|
+
@waker.wake
|
|
1244
|
+
short = count > 1 || Session.stopped_marker?(@session_id, state_dir: @state_dir)
|
|
1245
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
1246
|
+
deadline = now + (short ? @signal_short_grace : @signal_grace)
|
|
1247
|
+
hooks_deadline = deadline + @signal_hard_exit
|
|
1248
|
+
ended = false
|
|
1249
|
+
until @left_loop
|
|
1250
|
+
running = @engine&.turn_running?
|
|
1251
|
+
ended ||= running && @turn_ending
|
|
1252
|
+
@engine.cancel_current_turn!(:signal) if running && !ended
|
|
1253
|
+
break if @loop_started && Process.clock_gettime(Process::CLOCK_MONOTONIC) >= (ended ? hooks_deadline : deadline)
|
|
1254
|
+
|
|
1255
|
+
sleep(SIGNAL_RETRY)
|
|
1256
|
+
end
|
|
1257
|
+
signal_grace_expired(name) unless @left_loop
|
|
1258
|
+
rescue StandardError => e
|
|
1259
|
+
Log.warn(:worker, "signal_failed", signal: name, error: e.class.name, msg: e.message)
|
|
1260
|
+
end
|
|
1261
|
+
|
|
1262
|
+
# The process ends SIGNAL_HARD_EXIT from now at the latest.
|
|
1263
|
+
def signal_hard_exit
|
|
1264
|
+
sleep(@signal_hard_exit)
|
|
1265
|
+
@signals.hard_exit
|
|
1266
|
+
end
|
|
1267
|
+
|
|
1268
|
+
# The turn didn't end in time (a tool that ignores the cancel, slow
|
|
1269
|
+
# after_turn hooks): the session is saved as it is now (a turn that has
|
|
1270
|
+
# ended: with it), and the signal raised in the loop's thread, so
|
|
1271
|
+
# Worker#run's teardown and the lock's release still run. The process
|
|
1272
|
+
# ends SIGNAL_HARD_EXIT later at the latest. Once, whatever the signals.
|
|
1273
|
+
def signal_grace_expired(name)
|
|
1274
|
+
expired = @signal_mutex.synchronize { !@grace_expired && !@left_loop && (@grace_expired = true) }
|
|
1275
|
+
return unless expired
|
|
1276
|
+
|
|
1277
|
+
Log.warn(:worker, "signal_grace_expired", signal: name, turn_running: @engine&.turn_running? || false)
|
|
1278
|
+
save_or_log(:signal) { save_session }
|
|
1279
|
+
# Not once the loop has left (it may have, during the save).
|
|
1280
|
+
@signal_mutex.synchronize { @run_thread&.raise(SignalException.new(name)) unless @left_loop }
|
|
1281
|
+
signal_hard_exit
|
|
1135
1282
|
end
|
|
1136
1283
|
end
|
|
1137
1284
|
end
|