onionskin-cli 0.0.2 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/CHANGELOG.md +111 -0
  2. package/INSTALL.md +5 -1
  3. package/NOTICE +34 -0
  4. package/README.md +101 -33
  5. package/docs/contracts/batch.schema.json +27 -4
  6. package/docs/contracts/ops.schema.json +117 -5
  7. package/package.json +4 -2
  8. package/skills/onionskin/SKILL.md +9 -6
  9. package/skills/onionskin/reference/batch.md +5 -3
  10. package/skills/onionskin/reference/formats.md +5 -1
  11. package/skills/onionskin/reference/ops.md +47 -3
  12. package/skills/onionskin/reference/troubleshooting.md +31 -0
  13. package/src/agent-detect.js +28 -2
  14. package/src/agent-prompt.js +12 -1
  15. package/src/agent-runner.js +77 -19
  16. package/src/anchor-work.js +154 -0
  17. package/src/anchor-worker.js +38 -0
  18. package/src/anchors.js +61 -11
  19. package/src/apply.js +119 -47
  20. package/src/batch.js +32 -8
  21. package/src/check-worker.js +8 -5
  22. package/src/cli.js +91 -25
  23. package/src/constants.js +34 -1
  24. package/src/diff.js +3 -1
  25. package/src/docx.js +368 -58
  26. package/src/edit-hash.js +19 -0
  27. package/src/fsutil.js +1 -1
  28. package/src/history-worker.js +8 -8
  29. package/src/history.js +135 -31
  30. package/src/html-active.js +384 -0
  31. package/src/html-envelope.js +51 -0
  32. package/src/html-parser.js +8 -0
  33. package/src/html-render-worker.js +25 -0
  34. package/src/html-render.js +880 -0
  35. package/src/html-utf8.js +65 -0
  36. package/src/html-write-worker.js +14 -0
  37. package/src/html-write.js +380 -0
  38. package/src/html.js +1257 -0
  39. package/src/local.js +68 -4
  40. package/src/merge-package.js +1110 -0
  41. package/src/merge-pptx.js +98 -38
  42. package/src/merge-proof.js +1019 -0
  43. package/src/merge-worker.js +60 -0
  44. package/src/merge-xlsx.js +84 -29
  45. package/src/merge.js +286 -83
  46. package/src/nfc.js +46 -0
  47. package/src/notes.js +122 -0
  48. package/src/offload.js +400 -1
  49. package/src/open-worker.js +27 -0
  50. package/src/ops/common.js +2 -1
  51. package/src/ops/docx.js +58 -29
  52. package/src/ops/html.js +394 -0
  53. package/src/ops/pptx.js +74 -21
  54. package/src/ops/run.js +23 -3
  55. package/src/ops/runs.js +215 -13
  56. package/src/ops/schema.js +8 -1
  57. package/src/ops/xlsx.js +12 -7
  58. package/src/ops-worker.js +17 -0
  59. package/src/package.js +26 -2
  60. package/src/person.js +8 -1
  61. package/src/pptx.js +85 -21
  62. package/src/quick-plan.js +13 -6
  63. package/src/redline-worker.js +10 -14
  64. package/src/reply.js +46 -19
  65. package/src/retain.js +223 -0
  66. package/src/review.js +32 -18
  67. package/src/rewrite-check.js +2 -2
  68. package/src/sanitize.js +16 -3
  69. package/src/server.js +88 -37
  70. package/src/session.js +1143 -208
  71. package/src/stop-gate.js +34 -0
  72. package/src/suggest-reply.js +42 -7
  73. package/src/suggest.js +14 -29
  74. package/src/text.js +161 -11
  75. package/src/update.js +288 -0
  76. package/src/voice-guide.js +4 -3
  77. package/src/voice-learn.js +4 -3
  78. package/src/voice-sample-worker.js +7 -6
  79. package/src/voice-worker.js +4 -6
  80. package/src/watch.js +188 -0
  81. package/src/worker-host.js +23 -0
  82. package/src/xlsx.js +355 -94
  83. package/src/xml-tree.js +6 -3
  84. package/src/xml.js +82 -34
  85. package/src/zip-write.js +2 -1
  86. package/src/zip.js +9 -5
  87. package/vendor/parse5/LICENSE.txt +32 -0
  88. package/vendor/parse5/parse5.mjs +7822 -0
  89. package/vendor/parse5.lock.json +23 -0
  90. package/web/dist/assets/index-DkNx0LSH.css +1 -0
  91. package/web/dist/assets/{index-BUtofdtr.js → index-DytKkMdi.js} +117 -107
  92. package/web/dist/index.html +2 -2
  93. package/web/dist/assets/index-Dfizlg0i.css +0 -1
package/src/session.js CHANGED
@@ -2,9 +2,11 @@ import crypto from "node:crypto";
2
2
  import { EventEmitter } from "node:events";
3
3
  import fs from "node:fs";
4
4
  import path from "node:path";
5
- import { REFUSED_EXTENSIONS, FORMATS, MAX_DOC_BYTES, PRODUCT, env } from "./constants.js";
6
- import { buildDurable, readUnits, resolveDurableAny, resolveView, viewDurable } from "./anchors.js";
7
- import { durableFromEngine, headingBefore, resolveDurable } from "./docx.js";
5
+ import { EDITS_OVERFLOW, ONLY_FORMATS, REFUSED_EXTENSIONS, REFUSED_PAGES, MAX_DOC_BYTES, PRODUCT, env, formatOfExt } from "./constants.js";
6
+ import { readAll, unitsFor } from "./anchors.js";
7
+ import { anchorWork, weakDurable } from "./anchor-work.js";
8
+ import { scanBudget } from "./text.js";
9
+ import { CapError } from "./xml.js";
8
10
  import { checkRegularOwned, readDoc, replaceGuarded, sha256, Snapshots, WriteConflict, writeSiblingCopy } from "./fsutil.js";
9
11
  import { buildBatch, compactBatch, withOutline } from "./batch.js";
10
12
  import { styleChangedOutside, styleNote } from "./style.js";
@@ -13,23 +15,28 @@ import { cleanVoiceDraft } from "./voice-guide.js";
13
15
  import { voiceJob } from "./voice-learn.js";
14
16
  import { clearTickets, saveTicket, ticketOf } from "./lease.js";
15
17
  import { analyze } from "./analyze.js";
18
+ import { anchorTimeoutMs, anchorsOffThread, htmlWriteOffThread, mergeOffThread, readOffThread, renderOffThread, startWorker } from "./offload.js";
16
19
  import { deepNotice, validatePackage } from "./package.js";
20
+ import { HtmlEncodingError } from "./html.js";
21
+ import { ACTIVE_REFUSED } from "./html-active.js";
17
22
  import { execFile } from "node:child_process";
23
+ import { reopenCommand } from "./person.js";
18
24
  import { sessionKey, stateDir, sub, writeStateFile } from "./paths.js";
19
25
  import { cleanFlag, hasControlChars, untrusted, userText } from "./sanitize.js";
20
26
  import { addUpdate, historyDetail, historyList, visibleUpdates } from "./history.js";
21
27
  import { BAD_QUESTION_NOTE, QUESTIONS_PER_BATCH, cleanQuestion } from "./question.js";
22
28
  import { validateOps } from "./ops/schema.js";
23
- import { Worker } from "node:worker_threads";
24
29
  import { AUTHORS, wordDate } from "./redline.js";
25
30
  import { deepRedline } from "./checker.js";
26
31
  import { TRACKED, formattingText, savedText, unsupportedText } from "./tracked-words.js";
27
32
  import { LOCAL_HOLDER } from "./local.js";
28
33
  import { bgHolder, isAgentHolder, isBgHolder, termHolder, TERM_PREFIX } from "./agent-detect.js";
29
- import { OUTLINE_FILE_MAX, REVIEW_ASKS, REVIEW_KINDS, SUGGESTIONS_MAX, UNITS_MAX, cardsFrom, outlineOf, reviewSummary, unitsOf } from "./review.js";
34
+ import { OUTLINE_FILE_MAX, REVIEW_ASKS, REVIEW_KINDS, SUGGESTIONS_MAX, UNITS_MAX, outlineOf, reviewSummary, unitsOf } from "./review.js";
30
35
  import { VOICE_BATCH_MAX_CHARS } from "./voice-samples.js";
31
36
  import { STAMP_WORDS } from "./stamp-words.js";
32
- import { USE_SUMMARY, buildUseReply, placeOf } from "./suggest-reply.js";
37
+ import { USE_SUMMARY } from "./suggest-reply.js";
38
+ import { mergeNote } from "./notes.js";
39
+ import { COMMENT_MAX, COMMENT_TEXT_MAX, COMMENTS_MAX, EDITS_MAX, NOTE_MAX, TOO_MANY_COMMENTS, REQUEST_MAX, SESSION_MAX, capComment, cleanEdit, cleanPageAnchor, cleanView, cutText, jsonSize } from "./retain.js";
33
40
 
34
41
  /** The line under a comment whose quick fixes could not be written on this Mac (docs/contracts/routes.md). */
35
42
  export const LOCAL_HINT = "This could not be changed on this Mac, so nothing was written. Press Send to ask the agent.";
@@ -58,8 +65,33 @@ export class HttpError extends Error {
58
65
  }
59
66
  }
60
67
 
68
+ /** One comment more than a file keeps (src/retain.js COMMENTS_MAX): 413 too_many_comments, in words the page shows as they are. */
69
+ const tooMany = () => new HttpError(413, TOO_MANY_COMMENTS, { code: "too_many_comments" });
70
+ /** A note longer than Onionskin keeps (src/retain.js NOTE_MAX): 413 note_too_long. `joined`: a waiting note and this one together. */
71
+ const noteTooLong = (joined) =>
72
+ new HttpError(413, joined ? `The note waiting for the agent and this one together are longer than Onionskin takes (${NOTE_MAX.toLocaleString("en-US")} characters), so nothing was sent. Shorten this note, then Send again.` : `This note is longer than Onionskin takes (${NOTE_MAX.toLocaleString("en-US")} characters), so it was not saved. Shorten it, then try again.`, { code: "note_too_long" });
73
+ /** The page's note as stored: refused (never cut) past NOTE_MAX. */
74
+ function noteOf(v) {
75
+ const t = userText(v, NOTE_MAX + 1);
76
+ if ([...t].length > NOTE_MAX) throw noteTooLong(false);
77
+ return t;
78
+ }
79
+ /** Two notes, the earlier first, as one (a waiting batch's or the queued ones, then the new one): refused past NOTE_MAX. */
80
+ function joinNotes(a, b) {
81
+ const t = [a, b].filter(Boolean).join("\n\n");
82
+ if ([...t].length > NOTE_MAX) throw noteTooLong(true);
83
+ return t;
84
+ }
85
+
86
+ /** A request whose own data is more than Onionskin keeps (src/retain.js): 413 too_large_request, in words the page shows as they are. */
87
+ export const tooLarge = (what) => new HttpError(413, `${what} is larger than Onionskin takes (${Math.round(REQUEST_MAX / 1024 / 1024)} MB), so nothing was saved. Remove some comments or shorten the longest ones, then try again.`, { code: "too_large_request" });
88
+
61
89
  const LEASE_MS = () => Number(env("LEASE_MS", 15 * 60 * 1000));
90
+ const SENT_TABS = 16; // tabs of one file whose newest committed Send a session remembers (s.sent)
91
+ const DELETED_KEPT = 200; // ids and words of deleted drafts a session remembers for a handover (Sessions.handOver)
92
+ const HANDOVER_MS = 60_000; // how long a tab that lost editing may hand over its unsaved words (Sessions.handOver)
62
93
  const MAX_BYTES = MAX_DOC_BYTES;
94
+ const TOO_BIG = `Onionskin cannot read this file: it is larger than ${Math.round(MAX_BYTES / 1024 / 1024)} MB, more than Onionskin can read. Make it smaller (pictures and videos are often most of a file) or split it, and open that.`;
63
95
  /** "Show in folder" exists on macOS (`open -R`); elsewhere the page shows the path. ONIONSKIN_NO_OPEN=1 turns it off, ONIONSKIN_OPEN_STUB=<file> records instead of launching. */
64
96
  const canReveal = () => !!env("OPEN_STUB") || (process.platform === "darwin" && !env("NO_OPEN"));
65
97
  /** readDoc with the size refusal as an HTTP error */
@@ -67,31 +99,81 @@ function readCapped(file) {
67
99
  try {
68
100
  return readDoc(file);
69
101
  } catch (err) {
70
- if (err.code === "TOO_LARGE") throw new HttpError(413, `document too large: ${err.message}`, { code: "too_large" });
102
+ if (err.code === "TOO_LARGE") throw new HttpError(413, `Onionskin cannot read this file: ${err.message}.`, { code: "too_large" });
71
103
  throw err;
72
104
  }
73
105
  }
74
106
 
75
- const num = (v) => (Number.isFinite(Number(v)) ? Math.round(Number(v) * 1000) / 1000 : 0);
107
+ /**
108
+ * Every part the server reads later, of bytes about to become the file (a Send's, an agent's write), so they are refused before anything is
109
+ * written: past a reader's cap with the reason (CapError: 422 too_large), damaged where the readers look as an invalid package (422), never
110
+ * as an internal error. The reading is kept for what reads the same version next (unitsFor).
111
+ */
112
+ function readWhole(format, bytes, sha, refusal = "refusing to write") {
113
+ try {
114
+ const units = readAll(unitsFor(format, bytes, sha));
115
+ readInCaps(format, sha);
116
+ return units;
117
+ } catch (err) {
118
+ if (err instanceof CapError || err instanceof HttpError) throw err;
119
+ throw new HttpError(422, `${refusal}: ${untrusted(err?.message || err, 200)}`, { code: "invalid_package" });
120
+ }
121
+ }
122
+
123
+ // "<format>:<sha256>" of the versions read whole within the readers' caps, on this thread (readWhole) or by the open worker, so a version
124
+ // read again is not read whole again. Ceiling: VERIFIED_KEPT entries, the oldest dropped first (one dropped is read whole again when needed).
125
+ const VERIFIED_KEPT = 256;
126
+ const verified = new Set();
127
+ function readInCaps(format, sha) {
128
+ const id = `${format}:${sha}`;
129
+ verified.delete(id);
130
+ verified.add(id);
131
+ if (verified.size > VERIFIED_KEPT) verified.delete(verified.values().next().value);
132
+ }
133
+
134
+ /**
135
+ * Bytes about to be handed to the engine on this thread (a quick preview, a merge, a change made on this Mac), which reads without the
136
+ * readers' caps: they are read whole within the caps first (readWhole) unless this version already was, so a file that changed on disk
137
+ * past a cap is refused with the reason (CapError: 422 too_large) before the engine parses any of it. Returns `bytes`.
138
+ */
139
+ export function readChecked(format, bytes, { sha = sha256(bytes), refusal = "cannot read the file" } = {}) {
140
+ if (!verified.has(`${format}:${sha}`)) readWhole(format, bytes, sha, refusal);
141
+ return bytes;
142
+ }
143
+
144
+ // bounded before it is rounded, so the result is always finite (1e308 * 1000 is Infinity, stored as null); anything that is not a finite
145
+ // number is 0, as src/retain.js drops what is not of its type
146
+ const num = (v, max) => {
147
+ const n = Number(v);
148
+ return Number.isFinite(n) ? Math.round(Math.min(max, Math.max(0, n)) * 1000) / 1000 : 0;
149
+ };
150
+ const frac = (v) => num(v, 1);
151
+ const PX_MAX = 1_000_000; // a paragraph taller than this is not drawn by any page
76
152
  /** A pin: x/y as fractions of the element it sits on (`yPx` is kept for DOCX paragraphs); `on` says what that element is. */
77
153
  function cleanPin(p) {
78
- const out = { xFrac: num(p.xFrac), yPx: num(p.yPx) };
79
- if (p.yFrac != null) out.yFrac = num(p.yFrac);
154
+ const out = { xFrac: frac(p.xFrac), yPx: num(p.yPx, PX_MAX) };
155
+ if (p.yFrac != null) out.yFrac = frac(p.yFrac);
80
156
  if (["paragraph", "cell", "shape", "slide"].includes(p.on)) out.on = p.on;
81
157
  if (p.sxFrac != null) {
82
- out.sxFrac = num(p.sxFrac);
83
- out.syFrac = num(p.syFrac);
158
+ out.sxFrac = frac(p.sxFrac);
159
+ out.syFrac = frac(p.syFrac);
84
160
  }
85
161
  return out;
86
162
  }
87
163
 
88
164
  const APPLY_STATUS = new Set(["refused", "skipped_protected", "anchor_not_found", "expect_mismatch", "not_applied"]);
89
- // the three-way merge of each format (loaded on first use: they pull in the engine)
90
- const MERGERS = { docx: { module: "./merge.js", fn: "mergeDocx" }, xlsx: { module: "./merge-xlsx.js", fn: "mergeXlsx" }, pptx: { module: "./merge-pptx.js", fn: "mergePptx" } };
91
165
  const HELD_SHOWN = 5; // items the page's "not in the document" message lists (web/src/Banners.jsx)
92
166
  const WRITES_KEPT = 20; // writes per file the merge can trace (src/merge-xlsx.js agentChanges); an older base finds no record and is handed back
93
- /** What the page says the user did during the run: kinds and places only, used to hold edits back or to name what is not carried, never to write. */
94
- const editKinds = (edits) => (Array.isArray(edits) ? edits.slice(0, 2000) : []).map((e) => ({ kind: String(e?.kind ?? "").slice(0, 40), sheet: typeof e?.sheet === "string" ? e.sheet.slice(0, 100) : null, slide: Number.isInteger(e?.slide) ? e.slide : null }));
167
+ /**
168
+ * What the page says the user did during the run: kinds and places only, used to hold edits back or to name what is not carried, never to
169
+ * write. The first EDITS_KEPT; past them one more entry of kind EDITS_OVERFLOW says the list was cut, so a planner that needs the whole list
170
+ * refuses the merge (src/merge-xlsx.js) instead of reading the cut list as complete. The proof of the merge never reads it.
171
+ */
172
+ const EDITS_KEPT = 2000;
173
+ export const editKinds = (edits) => {
174
+ const list = (Array.isArray(edits) ? edits.slice(0, EDITS_KEPT) : []).map((e) => ({ kind: String(e?.kind ?? "").slice(0, 40), sheet: typeof e?.sheet === "string" ? e.sheet.slice(0, 100) : null, slide: Number.isInteger(e?.slide) ? e.slide : null }));
175
+ return Array.isArray(edits) && edits.length > EDITS_KEPT ? [...list, { kind: EDITS_OVERFLOW, sheet: null, slide: null }] : list;
176
+ };
95
177
  /** One op `apply` did not apply, as the page shows it: codes and short reasons only (never document text). */
96
178
  function cleanProblem(p, run = "") {
97
179
  return {
@@ -100,7 +182,8 @@ function cleanProblem(p, run = "") {
100
182
  op: untrusted(p?.op, 40),
101
183
  status: APPLY_STATUS.has(p?.status) ? p.status : "refused",
102
184
  code: p?.code ? untrusted(p.code, 60) : null,
103
- reason: untrusted(p?.reason, 200),
185
+ // the R1 refusal in the server's own words, never the reason text an agent's report carries
186
+ reason: p?.code === "active_content" ? ACTIVE_REFUSED : untrusted(p?.reason, 200),
104
187
  };
105
188
  }
106
189
 
@@ -108,17 +191,47 @@ function shellQuote(s) {
108
191
  return `'${String(s).replace(/'/g, `'\\''`)}'`;
109
192
  }
110
193
 
111
- /** A stored durable anchor re-resolved against `units` (the bytes the agent will read next); null when the unit is gone. Without a durable, one is built from the browser's anchor. */
112
- function reanchor(units, c) {
113
- const d = c.durable;
114
- if (!d) return buildDurable(units, c.anchor);
115
- const r = resolveDurableAny(units, d);
116
- if (r.method === "none") return null;
117
- if (units.format === "docx") return { ...d, ...r, format: "docx", quote: d.quote, prefix: d.prefix, suffix: d.suffix, paraText: r.text, engineDelta: r.engineDelta };
118
- if (units.format === "xlsx") return buildDurable(units, { location: { sheetIndex: r.sheetIndex, sheetName: r.sheetName, range: r.range } });
119
- return buildDurable(units, { location: { slideIndex: r.slideIndex, slideId: r.slideId, shapeId: r.shapeId ?? undefined, shapeName: r.shapeName, cell: d.cell || undefined }, quote: d.quote, prefix: d.prefix, suffix: d.suffix, hitKind: d.quote ? undefined : "shape" });
194
+ /**
195
+ * An HTML file that is not saved as UTF-8 (PLAN-html R10, R18): 400 with the fixed sentence as it is and `code` (not_utf8 or the
196
+ * reason). The page shows the sentence; the UTF-8 copy it offers is a CLI action (`onionskin open` prints its command: src/html-utf8.js).
197
+ */
198
+ function encodingRefusal(real, r) {
199
+ return new HttpError(400, r.error, { code: r.code === "not_utf8" ? "not_utf8" : "encoding" });
120
200
  }
121
201
 
202
+ /** The person's pending HTML text edits a Send or "Save my edits" carries (web/src/html-view.js createEdits): checked fields only. */
203
+ const HTML_EDITS_MAX = 2000;
204
+ const HTML_EDIT_TEXT_MAX = 1_000_000; // one text node's new text; the whole request is held to REQUEST_MAX (src/server.js)
205
+ export function cleanHtmlEdits(list) {
206
+ if (list == null) return [];
207
+ if (!Array.isArray(list) || list.length > HTML_EDITS_MAX) throw new HttpError(400, `htmlEdits must be a list of at most ${HTML_EDITS_MAX} edits`, { code: "bad_edits" });
208
+ return list.map((e) => {
209
+ const ok = e && typeof e === "object" && Number.isSafeInteger(e.nodeId) && e.nodeId >= 0 && typeof e.versionSha === "string" && /^[0-9a-f]{64}$/.test(e.versionSha) && typeof e.baseTextHash === "string" && /^[0-9a-f]{14}$/.test(e.baseTextHash) && typeof e.newText === "string" && e.newText.length <= HTML_EDIT_TEXT_MAX;
210
+ if (!ok) throw new HttpError(400, "an HTML text edit must be {nodeId, versionSha, baseTextHash, newText}", { code: "bad_edits" });
211
+ return { nodeId: e.nodeId, versionSha: e.versionSha, baseTextHash: e.baseTextHash, newText: e.newText };
212
+ });
213
+ }
214
+ /** 409 version_changed: the page's edits were made on a version that is not the file on disk now (it reloads the view and places them again). */
215
+ const VERSION_CHANGED = "The file changed since this page loaded it, so your text edits were not written. The page shows the new version and puts your edits back where it can find their text; check them, then Send again.";
216
+ /** The write check refused the person's text edits: 409 for an edit made on other text (the page reloads), 422 for the others; nothing was written. */
217
+ function htmlEditRefusal(c) {
218
+ if (c.code === "edit_moved") return new HttpError(409, VERSION_CHANGED, { code: "version_changed" });
219
+ if (c.code === "check_failed" || c.failed) return new HttpError(503, `Your text edits could not be checked: ${c.reason || c.why || "the check did not finish"}. Nothing was written; your edits are still in this page. Try again in a moment.`, { code: "check_failed" });
220
+ return new HttpError(422, `Your text edits were not written: ${c.reason}.`, { code: c.code, ...(c.reasons ? { reasons: c.reasons } : {}) });
221
+ }
222
+
223
+ // The tasks the anchor work does for a request (src/anchor-work.js), and how a result is matched to its task: by the task's own content,
224
+ // so a comment changed while the worker ran finds no answer and goes out weaker instead of with another one's anchor.
225
+ const buildTask = (c) => ({ op: "build", anchor: c.anchor, pin: c.kind === "pin" });
226
+ const reanchorTask = (c) => ({ op: "reanchor", c: { durable: c.durable ?? null, anchor: c.durable ? null : c.anchor } });
227
+ const editAnchor = (e) => ({ location: e.location, quote: e.after, ...(e.cut ? { cut: true } : {}) });
228
+ const cardsTask = (results, ix, run) => ({ op: "cards", list: results.suggestions, units: ix.units, index: ix.index, reviewId: run.batch.batch_id });
229
+ // a quick fix's Use: the page anchor's location and text only (its kind of selection does not change where the sentence is)
230
+ const useTask = (a, rec, id) => ({ op: "use", anchor: { location: a.location, quote: a.quote, prefix: a.prefix, suffix: a.suffix }, rec: { quote: rec.quote, before: rec.before, after: rec.after }, id });
231
+ const taskKey = (t) => JSON.stringify(t);
232
+ /** A task's result that can be used: not missing, not an error of its own. A null anchor (the unit is not in the file) is a result. */
233
+ const usable = (r) => r !== undefined && !(r && typeof r === "object" && "error" in r && Object.keys(r).length === 1);
234
+
122
235
  /**
123
236
  * All session state and transitions, with no HTTP in it.
124
237
  * States (derived): idle, listening, working, queued, stranded, closed. poll results add timeout and superseded.
@@ -127,17 +240,31 @@ function reanchor(units, c) {
127
240
  * - Send while working: comments only, kept as `queued`; released to the next poller after `done`.
128
241
  * - Lease expiry or Take back: `delivered` is withdrawn, the document unlocks.
129
242
  */
243
+ // What one request may spend scanning a whole reading for moved anchors (src/text.js scanBudget): about 0.1 to 0.3 s here. Fifty moved
244
+ // comments on a Word document of 26,000 paragraphs took 10 s before. Past it, the rest resolve as moved or removed, and an anchor being
245
+ // made is made without the text it cannot pay for (src/xlsx.js xlsxDurable). The server's routes make and resolve anchors in the anchor
246
+ // worker (src/anchor-worker.js, ten times this budget, within a time limit); the synchronous methods (Send, done, revert, resolve, Use of
247
+ // a quick fix) do it on the caller's thread under this budget.
248
+ export const SCAN_BUDGET = 40_000_000;
249
+ const REFUSED_KEPT = 64;
250
+ const REFUSED_MS = 2 * 60_000;
251
+
130
252
  export class Sessions extends EventEmitter {
131
253
  // delivered run -> styleChangedOutside() when it went out; memory only, so a run reloaded after a restart has none and is not flagged
132
254
  #styleSeen = new WeakMap();
133
255
  // delivered run -> guideChangedOutside() when it went out (the same idea for voice-guide.json); taking it also arms the guide's guard
134
256
  #guideSeen = new WeakMap();
135
- constructor({ now = Date.now } = {}) {
257
+ // the last outline built for an agent Onionskin started: {key, sha, all} (one entry; ceiling: two such agents in turn build twice)
258
+ #outline = null;
259
+ // `anchors`: the anchor worker's call (src/offload.js anchorsOffThread); a test passes its own to control when the work answers
260
+ constructor({ now = Date.now, anchors = anchorsOffThread } = {}) {
136
261
  super();
137
262
  this.now = now;
263
+ this.anchors = anchors;
138
264
  this.map = new Map(); // key -> session
139
265
  this.createdCopies = new Set(); // absolute paths of conflict copies THIS server wrote: the only paths `showCopy` will reveal
140
266
  this.editors = new Map(); // key -> id of the one browser tab that may edit (in memory only: a tab's id lives as long as its page)
267
+ this.handovers = new Map(); // key -> {tab, until}: the tab another tab just took editing from (memory only), see handOver()
141
268
  this.briefed = new Map(); // key -> Set of holders that already received a complete batch in this session (memory only: a restarted server briefs again)
142
269
  // key -> how the file got from one version to the next, oldest first: {from, to, route: "apply", ops, partial, anchors} for an agent's
143
270
  // apply, {from, to, route: "commit"} for a commit, {from, to, route: "merge", stable} for a pass of the user's merge. The merge reads it
@@ -156,12 +283,69 @@ export class Sessions extends EventEmitter {
156
283
  if (s.closed) return false; // a closed session has no editor; the tab that reopens it claims it
157
284
  if (cur && cur !== tab && !takeover) return false;
158
285
  if (cur !== tab) {
286
+ if (cur) this.handovers.set(key, { tab: cur, until: this.now() + HANDOVER_MS }); // the tab that lost editing may hand over its unsaved words once
159
287
  this.editors.set(key, tab);
160
288
  this.emit("change", s, "state");
161
289
  }
162
290
  return true;
163
291
  }
164
292
 
293
+ /**
294
+ * The tab that just lost editing to another ("Edit here instead") hands over the words it had not saved yet: once, within HANDOVER_MS, and
295
+ * only that tab. Nothing another tab saved is replaced: a comment it made is added as a draft; words it typed into a draft nobody changed since
296
+ * it last saved it (`base`, per comment) replace it; words typed into one the store holds otherwise become a new draft next to it; its note is merged into the stored one from the note it started from (`base`, src/notes.js mergeNote),
297
+ * and the tab that edits now gets it as a note the server gave back (its stale save cannot overwrite it). Refused (409 `no_handover`) for
298
+ * any other tab or later, and like a draft save past a cap (413, nothing changes). The page that cannot hand over keeps the words visible
299
+ * with Copy my comments (web/src/App.jsx).
300
+ */
301
+ // key -> (tab -> {sent, seq}): the note the newest draft save of each tab carried, and the session `seq` it stored at (memory only, the last 8 tabs). A tab that hands over a note the server already took
302
+ // from it (its save's answer was lost, so it did not know) hands over nothing new: whatever happened to that note since is not its loss
303
+ #noteFrom = new Map();
304
+ handOver(key, tab, { comments, note, base, baseSeq, noteRev } = {}) {
305
+ const s = this.get(key);
306
+ const h = this.handovers.get(key);
307
+ if (!h || h.tab !== tab || h.until < this.now()) throw new HttpError(409, "This tab no longer edits the file.", { code: "no_handover" });
308
+ if (!Array.isArray(comments)) throw new HttpError(400, "comments must be an array");
309
+ const byId = new Map(s.comments.map((c) => [c.id, c]));
310
+ const next = [...s.comments];
311
+ for (const c of comments.filter((x) => x && typeof x === "object")) {
312
+ const id = cutText(String(c.id), 64);
313
+ const old = byId.get(id);
314
+ const text = userText(c.text, COMMENT_TEXT_MAX);
315
+ const del = s.deletedDrafts?.[id];
316
+ if (!old && del === text) continue; // deleted in the tab that edits now, with these very words: it stays deleted
317
+ if (!old) next.push(this.#draft(c, del === undefined ? id : cutText(`${id}~h${crypto.randomBytes(3).toString("hex")}`, 64), null)); // deleted there, with other words: these come back as a new draft
318
+ else if (text === old.text) continue;
319
+ else if (old.status === "draft" && !old.answer && typeof c.base === "string" && userText(c.base, COMMENT_TEXT_MAX) === old.text) next[next.indexOf(old)] = this.#draft(c, id, old); // nobody changed it since: the newer words replace it
320
+ else next.push(this.#draft(c, cutText(`${id}~h${crypto.randomBytes(3).toString("hex")}`, 64), null));
321
+ }
322
+ if (next.length > COMMENTS_MAX) throw tooMany();
323
+ for (const c of next) if (c.status === "draft" && jsonSize(c) > COMMENT_MAX) throw tooLarge("This draft");
324
+ let text = s.note;
325
+ const from = this.#noteFrom.get(key);
326
+ const last = from?.get(tab);
327
+ const already = typeof note === "string" && (last?.sent === userText(note, NOTE_MAX + 1) || this.#noteTaken(s, noteRev, tab)); // or a Send of that tab took it
328
+ if (typeof note === "string" && note !== s.note && !already) {
329
+ // what it started from: the note the page's box is based on (`base`) when that came from a state at or after that tab's last save
330
+ // (`baseSeq`: the page took that save's answer, or something newer, as a Take back's); else the note that save carried (its answer
331
+ // was lost, so the box is that note and what was typed since); with no save of that tab, what the page says
332
+ const fresh = typeof base === "string" && Number.isSafeInteger(baseSeq) && (!last || baseSeq >= last.seq);
333
+ const from = fresh ? base : last ? last.sent : typeof base === "string" ? base : "";
334
+ const mine = noteOf(note);
335
+ // the box was emptied here since (a Send took the note that tab started from): only what that tab added after it is handed over
336
+ text = !s.note && from && mine.startsWith(from) ? mine.slice(from.length).replace(/^\s+/, "") : mergeNote(from, s.note, mine);
337
+ }
338
+ if ([...text].length > NOTE_MAX && [...text].length > [...s.note].length) throw noteTooLong(true);
339
+ const before = s.note;
340
+ this.#commitDraft(s, { comments: next, note: before }, null, tab, true);
341
+ this.handovers.delete(key);
342
+ if (text !== before) {
343
+ this.#giveNote(s, text);
344
+ this.#changed(s, "draft");
345
+ } else this.emit("change", s, "draft");
346
+ return s;
347
+ }
348
+
165
349
  /** The tab's event stream ended (closed, reloaded): the file has no editor until a tab claims it. */
166
350
  releaseEditor(key, tab) {
167
351
  if (this.editors.get(key) !== tab) return;
@@ -177,8 +361,50 @@ export class Sessions extends EventEmitter {
177
361
  }
178
362
 
179
363
  // ---- open ------------------------------------------------------------------------------------------------
180
- /** `opts.agent === "background"`: the person opened it from a terminal, so the server's background agent works on it (stored on the session). */
181
- open(input, opts = {}) {
364
+ /**
365
+ * open() as the server's routes call it: the file is first read in a worker thread with a time and memory limit (src/offload.js
366
+ * readOffThread: the package check and every part the server reads later), so a slow or hostile file never stalls the server, and a
367
+ * file no reader accepts is refused here, with the reason, instead of failing later at Send.
368
+ */
369
+ async openAsync(input, opts = {}) {
370
+ const { real, format } = this.#checkInput(input);
371
+ // A file that changes between the worker's reading and open()'s own (a swap, a save in progress) is read in the worker again, once:
372
+ // open() never takes bytes the worker has not read, so nothing on this thread reads a version past the readers' caps.
373
+ for (let attempt = 0; ; attempt++) {
374
+ const bytes = readCapped(real);
375
+ const sha = sha256(bytes);
376
+ const known = this.#refused.get(`${format}:${sha}`);
377
+ const r = known && this.now() - known.at < REFUSED_MS ? known : await readOffThread(bytes, format);
378
+ if (r.ok) readInCaps(format, r.sha);
379
+ if (!r.ok) {
380
+ // the same bytes are refused again at once (an agent's polls fall back to opening a file that is not open, again and again)
381
+ if (r.stage !== "busy") this.#refuse(`${format}:${sha}`, r);
382
+ if (r.stage === "encoding") throw encodingRefusal(real, r);
383
+ if (r.stage === "invalid") throw new HttpError(400, `Not a valid .${format}: ${r.error}`);
384
+ if (r.stage === "busy") throw new HttpError(503, `Onionskin is busy opening other files: ${r.error}. Try again in a minute.`, { code: "busy" });
385
+ const hint = r.stage === "timeout" || r.stage === "memory" ? " It may be damaged, or made to stall programs that read it." : "";
386
+ throw new HttpError(400, `This file could not be opened: ${r.error}.${hint}`, { code: r.stage === "timeout" ? "open_timeout" : "unreadable" });
387
+ }
388
+ try {
389
+ return this.open(input, { ...opts, checkedSha: r.sha, onlyChecked: true });
390
+ } catch (err) {
391
+ if (err?.extra?.code !== "changed_while_opening" || attempt >= 1) throw err;
392
+ }
393
+ }
394
+ }
395
+
396
+ // "<format>:<sha256>" -> what the open worker said about those bytes when it refused them, for REFUSED_MS (a refusal for time may not hold once the
397
+ // Mac is less busy); at most REFUSED_KEPT, oldest dropped first
398
+ #refused = new Map();
399
+ #refuse(id, r) {
400
+ if (r.at) return; // already a remembered answer
401
+ this.#refused.delete(id);
402
+ this.#refused.set(id, { ok: false, stage: r.stage, error: r.error, code: r.code, copyable: r.copyable, at: this.now() });
403
+ if (this.#refused.size > REFUSED_KEPT) this.#refused.delete(this.#refused.keys().next().value);
404
+ }
405
+
406
+ /** The path checks of open(): the real path, its format and its lstat. Throws HttpError. */
407
+ #checkInput(input) {
182
408
  let real;
183
409
  try {
184
410
  real = fs.realpathSync(path.resolve(input));
@@ -189,23 +415,36 @@ export class Sessions extends EventEmitter {
189
415
  if (hasControlChars(real)) throw new HttpError(400, "This file's path contains control or bidirectional-text characters. Rename or move the file and open it again.", { code: "unsafe_path" });
190
416
  const ext = path.extname(real).toLowerCase();
191
417
  if (REFUSED_EXTENSIONS.has(ext)) throw new HttpError(400, `${ext} (macro-enabled) files are refused`);
192
- const format = ext.slice(1);
193
- if (!FORMATS[format]) throw new HttpError(400, "Only .docx, .xlsx and .pptx files are supported");
418
+ if (Object.hasOwn(REFUSED_PAGES, ext)) throw new HttpError(400, REFUSED_PAGES[ext], { code: "refused_kind" });
419
+ const format = formatOfExt(ext);
420
+ if (!format) throw new HttpError(400, ONLY_FORMATS);
194
421
  let st;
195
422
  try {
196
423
  st = checkRegularOwned(real);
197
424
  } catch (err) {
198
425
  throw new HttpError(400, err.message);
199
426
  }
427
+ return { real, format, st };
428
+ }
429
+
430
+ /** `opts.agent === "background"`: the person opened it from a terminal, so the server's background agent works on it (stored on the session). */
431
+ open(input, opts = {}) {
432
+ const { real, format, st } = this.#checkInput(input);
200
433
  const bytes = readCapped(real);
201
- try {
202
- validatePackage(bytes, format);
203
- } catch (err) {
204
- throw new HttpError(400, `Not a valid .${format}: ${err.message}`);
434
+ const sha = sha256(bytes);
435
+ // openAsync already read these exact bytes in a worker; for it, other bytes (the file changed since) go back to the worker. Any other
436
+ // caller (tests, tools) gets the package check here.
437
+ if (opts.onlyChecked && opts.checkedSha !== sha) throw new HttpError(409, "The file kept changing while Onionskin opened it. Open it again once it is saved.", { code: "changed_while_opening" });
438
+ if (opts.checkedSha !== sha) {
439
+ try {
440
+ validatePackage(bytes, format);
441
+ } catch (err) {
442
+ if (err instanceof HtmlEncodingError) throw encodingRefusal(real, { error: err.message, code: err.code, copyable: err.copyable });
443
+ throw new HttpError(400, `Not a valid .${format}: ${err.message}`);
444
+ }
205
445
  }
206
446
  const key = sessionKey(real);
207
447
  let s = this.map.get(key);
208
- const sha = sha256(bytes);
209
448
  if (!s) {
210
449
  s = this.#load(key) || this.#fresh(key, real, format, sha);
211
450
  s.file = real;
@@ -268,8 +507,128 @@ export class Sessions extends EventEmitter {
268
507
  }
269
508
 
270
509
  #save(s) {
510
+ const { raw, cut } = this.#encode(s);
511
+ if (cut) Object.assign(s, cut); // in memory too
512
+ writeStateFile(path.join(sub("sessions"), `${s.key}.json`), raw);
513
+ }
514
+
515
+ /**
516
+ * The session file's text for `s`: {raw, cut}. The one cap on a session file (src/retain.js SESSION_MAX, in bytes): past it every
517
+ * comment's page anchor and every typed edit are cut further (`cut`: those lists, for the caller to keep); nothing is dropped. Each part
518
+ * is cut where it is stored, so this is a last line. `raw` can still be past the cap; a change the page asked for is then refused (#fits).
519
+ */
520
+ #encode(s) {
271
521
  const { pollers, snapshots, ...rest } = s;
272
- writeStateFile(path.join(sub("sessions"), `${s.key}.json`), JSON.stringify(rest));
522
+ const raw = JSON.stringify(rest);
523
+ if (Buffer.byteLength(raw) <= SESSION_MAX) return { raw, cut: null };
524
+ const cut = {
525
+ comments: rest.comments.map((c) => capComment({ ...c, anchor: c.anchor && { ...c.anchor, cut: true, quote: cutText(c.anchor.quote, 2000), paraText: cutText(c.anchor.paraText, 2000) } })),
526
+ edits: rest.edits.map((e) => ({ ...e, before: cutText(e.before, 2000), after: cutText(e.after, 2000), cut: true })),
527
+ };
528
+ return { raw: JSON.stringify({ ...rest, ...cut }), cut };
529
+ }
530
+
531
+ /** Whether the session with `patch` fits its file once cut (#encode): checked before a change the page asked for touches anything. */
532
+ #fits(s, patch) {
533
+ return Buffer.byteLength(this.#encode({ ...s, ...patch }).raw) <= SESSION_MAX;
534
+ }
535
+
536
+ /** The page's typed edits since the last Send, as they are kept for the next batch: each field cut (src/retain.js), at most EDITS_MAX in all. */
537
+ #editsToTake(s, edits) {
538
+ return (Array.isArray(edits) ? edits.slice(0, Math.max(0, EDITS_MAX - s.edits.length)) : []).map(cleanEdit);
539
+ }
540
+
541
+ // ---- anchor work: the anchor worker, or this thread (src/anchor-work.js) ------------------------------------------------
542
+ /**
543
+ * The results of `tasks` on the version `sha` (`bytes`: a Buffer, or a function that returns it). With `prepared` (what the anchor
544
+ * worker answered for the same tasks, src/session.js #offThread) they are taken from it, and nothing is read here: a task it has no
545
+ * answer for (its work did not finish, or it was for another version) reads as not done. Without, they are worked out on this thread
546
+ * under the request's budget, as the synchronous methods always did. Returns {read, outline, at(i)}.
547
+ */
548
+ #results(format, bytes, sha, tasks, prepared, { check = false } = {}) {
549
+ if (prepared) {
550
+ const ok = prepared.ok && prepared.sha === sha;
551
+ return { read: ok ? prepared.read : null, outline: ok ? prepared.outline : null, at: (i) => (ok ? prepared.map.get(taskKey(tasks[i])) : undefined) };
552
+ }
553
+ // an HTML file is read in workers only (the parser is not on this thread): without the worker's answer the work is not done, and each
554
+ // caller makes its anchors weaker, as when the worker did not finish
555
+ if (format === "html") return { read: null, outline: null, at: () => undefined };
556
+ const r = anchorWork({ format, bytes, sha, check, tasks, budget: SCAN_BUDGET });
557
+ return { read: r.read, outline: null, at: (i) => r.results[i] };
558
+ }
559
+
560
+ /** Bytes about to become the file: what their reading found. Past a cap 422 (CapError); not read in time and never read whole before: 503. */
561
+ #readVerdict(work, format, sha, refusal) {
562
+ const read = work.read;
563
+ if (read?.ok) return readInCaps(format, sha);
564
+ if (read?.cap) throw new CapError(read.cap);
565
+ if (read?.invalid) throw new HttpError(422, `${refusal}: ${untrusted(read.invalid, 200)}`, { code: "invalid_package" });
566
+ if (verified.has(`${format}:${sha}`)) return; // the work did not finish, but this version was read whole within the caps before
567
+ throw new HttpError(503, "Onionskin could not finish reading the file in time, so nothing was sent. Your comments are kept. Try again in a moment.", { code: "read_timeout" });
568
+ }
569
+
570
+ /** `job` ({source, check, tasks, outline}) in the anchor worker: {ok: true, sha, read, outline, map: task key -> result} or {ok: false, why}. */
571
+ async #offThread(format, job) {
572
+ const r = await this.anchors({ format, ...job });
573
+ if (!r.ok) return { ok: false, why: r.failed };
574
+ const map = new Map();
575
+ job.tasks.forEach((t, i) => map.set(taskKey(t), r.r.results[i]));
576
+ return { ok: true, sha: r.sha, read: r.r.read, outline: r.r.outline ?? null, map };
577
+ }
578
+
579
+ // the async methods of one file run in turn (another file's never wait for them); synchronous ones keep their own order
580
+ #chains = new Map();
581
+ #serial(key, fn) {
582
+ const prev = this.#chains.get(key) || Promise.resolve();
583
+ const run = prev.then(fn, fn);
584
+ const tail = run.then(() => {}, () => {});
585
+ this.#chains.set(key, tail);
586
+ tail.then(() => this.#chains.get(key) === tail && this.#chains.delete(key));
587
+ return run;
588
+ }
589
+
590
+ /**
591
+ * `fn` (a synchronous change the page asks for: a draft save, Take back, an answer) in this file's turn, as the server's routes call it.
592
+ * One that arrives while a Send waits for the anchor worker is applied after that Send commits, against what it committed, as when Send
593
+ * was synchronous. Never called from inside a turn (it would wait for itself).
594
+ */
595
+ inTurn(key, fn) {
596
+ this.get(key);
597
+ return this.#serial(key, () => fn());
598
+ }
599
+
600
+ /** setDraft() in this file's turn (the route POST /api/draft). */
601
+ setDraftAsync(key, args) {
602
+ return this.inTurn(key, () => this.setDraft(key, args));
603
+ }
604
+
605
+ /**
606
+ * Resolves once every request of this file that arrived before now is answered, so a page that loads (or reloads) during a Send starts
607
+ * from what it committed, never from the middle of it. Bounded: a Send plans its worker job at most twice, each within the worker's time
608
+ * limit; past that the page gets the state as it is then.
609
+ */
610
+ async settled(key, ms = 2 * anchorTimeoutMs() + 5_000) {
611
+ const tail = this.#chains.get(key);
612
+ if (!tail) return;
613
+ let timer;
614
+ await Promise.race([tail, new Promise((r) => (timer = setTimeout(r, ms)))]);
615
+ clearTimeout(timer);
616
+ }
617
+
618
+ /**
619
+ * An agent Onionskin started polls while a batch waits for it (src/server.js /cli/poll): the outline its delivery carries is built in
620
+ * the anchor worker first, so the delivery finds it (#withOutline's memo) instead of reading the file on this thread.
621
+ */
622
+ async outlineAhead(key, holder) {
623
+ const s = this.get(key);
624
+ if (!isAgentHolder(holder) || !s.pending || s.delivered || s.pending.instructions?.review || s.pending.instructions?.voice_request) return;
625
+ const r = await anchorsOffThread({ format: s.format, source: { path: s.file }, tasks: [], outline: true });
626
+ if (r.ok && r.r.read.ok && r.r.outline) this.#outline = { key: s.key, sha: r.sha, all: r.r.outline };
627
+ }
628
+
629
+ /** An agent Onionskin started is waiting for this file: its delivery carries the outline, which the anchor worker then builds too. */
630
+ #wantsOutline(s) {
631
+ return s.pollers.some((p) => isAgentHolder(p.holder));
273
632
  }
274
633
 
275
634
  get(key) {
@@ -301,6 +660,8 @@ export class Sessions extends EventEmitter {
301
660
  return {
302
661
  key: s.key,
303
662
  name: path.basename(s.file),
663
+ file: s.file, // the absolute path (like copy_path): the page words the commands that reopen the file from any folder
664
+ reopen: reopenCommand(s.file), // the exact command to paste in a terminal from any folder
304
665
  format: s.format,
305
666
  state,
306
667
  locked: !!s.delivered,
@@ -322,11 +683,15 @@ export class Sessions extends EventEmitter {
322
683
  strandedCommand: state === "stranded" ? `${PRODUCT.bin} poll ${shellQuote(s.file)}` : null,
323
684
  startCommand: state === "stranded" ? `${PRODUCT.bin} ${shellQuote(s.file)}` : null, // what a person types in a terminal to start an agent on this file
324
685
  seq: s.seq,
686
+ // per tab, the newest Send (or quick fix's Use) of it that committed: {tab: {rev, noteRev}}. A page whose Send got no answer learns
687
+ // from it whether that Send went through (web/src/App.jsx checkSent)
688
+ sent: s.sent || {},
325
689
  editor: this.editors.get(s.key) || null,
326
690
  conflict: s.conflict || null,
327
691
  canReveal: canReveal(),
328
692
  // the agent's review cards (src/review.js); their ops stay on the server
329
693
  suggestions: (s.suggestions || []).map(({ op, acceptedBatch, ...c }) => c),
694
+ changedOutside: !!this.watch?.outside(s), // src/watch.js: the file on disk is a version Onionskin does not know (a boolean only, never a name)
330
695
  // the background agent (src/agent-runner.js): {state, cli, show, activity, failure}; never a path, a holder id or agent text
331
696
  agent: this.agent ? this.agent.viewFor(s.key) : { state: "none", cli: null, show: false, activity: [], failure: null },
332
697
  };
@@ -339,31 +704,198 @@ export class Sessions extends EventEmitter {
339
704
  }
340
705
 
341
706
  // ---- drafts ----------------------------------------------------------------------------------------------
342
- /** Browser replaces its list of comments/pins and the overall note. Server-owned fields (status, results) are kept. */
343
- setDraft(key, { comments, note }) {
707
+ // key -> {tab, rev}: the newest save or Send a tab numbered (memory only: a reloaded page is a new tab and counts again). Saves and Sends
708
+ // run in turn (inTurn), but two requests of one tab can still arrive out of order: an autosave sent just before Send was pressed can land
709
+ // after the Send committed, and would put the note the Send took back in the box. One numbered at or below what the tab already sent
710
+ // changes nothing (test/send-race.test.js).
711
+ #revs = new Map();
712
+ // s.sent = {tab: {rev, noteRev}}: per tab (the last SENT_TABS), the newest numbered Send (or Use) of it that committed, and the revision
713
+ // of its note box the newest such Send took. Stored with the session, so a page still asking whether its Send went through gets the right
714
+ // answer after the server restarts. A save of that tab whose note is that revision or older (its save on the way out while the Send was in
715
+ // flight) leaves the note alone, and so does its handover: a note a Send delivered is never put back.
716
+ #sentRev(s, { rev, tab, noteRev }, took = false) {
717
+ if (!Number.isSafeInteger(rev) || typeof tab !== "string") return;
718
+ const all = s.sent && typeof s.sent === "object" ? s.sent : {};
719
+ const last = all[tab];
720
+ if (last && last.rev > rev) return;
721
+ const { [tab]: _, ...rest } = all;
722
+ const entry = { rev, noteRev: took && Number.isSafeInteger(noteRev) ? noteRev : last?.noteRev ?? null };
723
+ s.sent = Object.fromEntries([...Object.entries(rest), [tab, entry]].slice(-SENT_TABS));
724
+ }
725
+ /** Whether `noteRev` of `tab` is a note a committed Send of that tab already took (see s.sent). */
726
+ #noteTaken(s, noteRev, tab) {
727
+ const last = typeof tab === "string" ? s.sent?.[tab] : null;
728
+ return Number.isSafeInteger(noteRev) && Number.isSafeInteger(last?.noteRev) && noteRev <= last.noteRev;
729
+ }
730
+
731
+ /**
732
+ * Browser replaces its list of comments/pins and the overall note. Server-owned fields (status, results) are kept: a sent, queued or
733
+ * finished comment is as the server holds it, whatever words the page sends for it. `note` absent (not a string): the stored note is
734
+ * kept (the page's save while a Send is in flight, which took the note). `rev` (optional): see #revs. Every field is rebuilt from what
735
+ * Onionskin reads (src/retain.js). Refused (413) before anything changes: more than COMMENTS_MAX comments, a note past NOTE_MAX, a
736
+ * comment past COMMENT_MAX after its cuts, a session past SESSION_MAX after the writer's. Nothing is dropped or cut without a word.
737
+ */
738
+ setDraft(key, { comments, note, rev, tab, noteRev, seen, noteSeen } = {}) {
344
739
  const s = this.get(key);
345
740
  if (!Array.isArray(comments)) throw new HttpError(400, "comments must be an array");
741
+ if (this.#overtaken(s, rev, tab, true)) return s;
742
+ this.#commitDraft(s, this.#draftPlan(s, { comments, note: this.#noteTaken(s, noteRev, tab) ? undefined : note, seen, noteSeen, tab }), rev, tab);
743
+ if (typeof note === "string" && tab) {
744
+ const m = this.#noteFrom.get(key) || new Map();
745
+ m.delete(tab);
746
+ m.set(tab, { sent: userText(note, NOTE_MAX + 1), seq: s.seq, used: !this.#noteTaken(s, noteRev, tab) }); // see handOver and #draftPlan
747
+ if (m.size > 8) m.delete(m.keys().next().value); // the tabs of one file: a handful
748
+ this.#noteFrom.set(key, m);
749
+ }
750
+ return s;
751
+ }
752
+
753
+ /** Whether this tab already numbered a newer save or Send than `rev` (`orSame`: one numbered the same too). See #revs. */
754
+ #overtaken(s, rev, tab, orSame = false) {
755
+ if (!Number.isSafeInteger(rev)) return false;
756
+ const last = this.#revs.get(s.key);
757
+ return !!last && last.tab === tab && (orSame ? rev <= last.rev : rev < last.rev);
758
+ }
759
+
760
+ /**
761
+ * What a draft save of `comments` and `note` would store, {comments, note}, with every limit checked and nothing changed. `note` absent:
762
+ * the stored note. A note past NOTE_MAX is refused, except one not longer than the stored note (Take back can join notes past it: the
763
+ * page saves the box it was given, and may shorten it).
764
+ */
765
+ #draftPlan(s, { comments, note, seen, noteSeen, tab }) {
766
+ if (!Array.isArray(comments)) throw new HttpError(400, "comments must be an array");
767
+ const list = comments.filter((c) => c && typeof c === "object");
768
+ if (list.length > COMMENTS_MAX) throw tooMany();
769
+ const long = typeof note === "string" && note.length > NOTE_MAX && note.length <= s.note.length;
770
+ let text = typeof note !== "string" ? s.note : long ? userText(note, s.note.length) : noteOf(note);
771
+ // the server gave a note back to the box (#giveNote) that the page had not seen when it sent this (`seen`: the newest session `seq` it
772
+ // took): the page's note is merged into it, never put over it
773
+ // (`noteSeen`: the newest state whose note the page's box took or is based on; a page can see a state and keep its own words)
774
+ const given = s.noteGiven;
775
+ const ns = Number.isSafeInteger(noteSeen) ? noteSeen : seen;
776
+ if (typeof note === "string" && given && Number.isSafeInteger(ns) && ns <= given.seq) {
777
+ // from the note the server held before the first give the page had not seen (`steps`: each give; a session stored before them has
778
+ // the first one's only); or, when a save of this tab was merged in since (its answer still on its way), from the note that save
779
+ // carried: the box is that note and what was typed since, and the stored note already has that save's words
780
+ // (ceiling: the last 50 gives; past that, a page that saw none of the kept ones merges from the oldest note, as before `steps`)
781
+ const hit = given.steps?.find((g) => g.seq >= ns);
782
+ const first = hit && !(given.cut && hit === given.steps[0]) ? hit : given;
783
+ const last = this.#noteFrom.get(s.key)?.get(tab);
784
+ text = mergeNote(last?.used && last.seq > first.seq ? last.sent : first.before, s.note, text);
785
+ }
346
786
  const prev = new Map(s.comments.map((c) => [c.id, c]));
347
- s.comments = comments.slice(0, 500).map((c) => {
348
- const old = prev.get(String(c.id));
349
- const base = {
350
- id: String(c.id).slice(0, 64),
351
- kind: ["text", "element", "pin"].includes(c.kind) ? c.kind : "element",
352
- text: userText(c.text),
353
- anchor: c.anchor && typeof c.anchor === "object" ? c.anchor : null,
354
- pin: c.pin && typeof c.pin === "object" ? cleanPin(c.pin) : null,
355
- pinNo: Number.isInteger(c.pinNo) ? c.pinNo : null,
356
- };
357
- if (old && old.status !== "draft") return { ...old, text: old.text, kind: old.kind }; // sent/done comments are frozen
358
- // an answered question: the words, the question and the answer stay as the person left them (the page cannot change them)
359
- const keep = old?.skipLocal ? { skipLocal: true } : {}; // a reverted local run: the next Send of this comment goes to the agent
360
- if (old?.localHint && userText(c.text) === old.text) keep.localHint = old.localHint; // the words changed: the hint no longer applies
361
- if (old?.answer) return { ...base, text: old.text, status: "draft", durable: old.durable || null, question: old.question, answer: old.answer, ...keep };
362
- return { ...base, status: "draft", durable: old?.durable || null, ...keep };
787
+ const next = list.map((c) => {
788
+ const id = cutText(String(c.id), 64);
789
+ const old = prev.get(id);
790
+ return old && old.status !== "draft" ? old : this.#draft(c, id, old); // sent/queued/finished comments are frozen
363
791
  });
364
- s.note = userText(typeof note === "string" ? note : "");
365
- this.#changed(s, "draft");
366
- return s;
792
+ // a comment on its way to the agent (in a waiting batch, or queued) stays whatever list a save carries: a save never changes what a
793
+ // Send delivers (a page that had not seen it yet, after a reload, does not take it out of the batch). Nor does a save delete a draft
794
+ // another tab wrote after the state its page took (`seen`): the page did not know of it (a window that handed over its words, the save
795
+ // of a page that was left landing after the reloaded page loaded)
796
+ const named = new Set(next.map((c) => c.id));
797
+ const unseen = (c) => c.status === "draft" && Number.isSafeInteger(seen) && c.wroteBy && c.wroteBy.tab !== tab && c.wroteBy.seq > seen;
798
+ for (const c of s.comments) if ((c.status === "sent" || c.status === "queued" || unseen(c)) && !named.has(c.id)) next.push(c);
799
+ if (next.length > COMMENTS_MAX) throw tooMany();
800
+ for (const c of next) if (c.status === "draft" && jsonSize(c) > COMMENT_MAX) throw tooLarge("This draft");
801
+ return { comments: next, note: text, seen: Number.isSafeInteger(noteSeen) ? noteSeen : seen };
802
+ }
803
+
804
+ /**
805
+ * The server puts `text` in the note box itself (Take back, a run that stopped with comments queued behind it, the Revert of a run on this
806
+ * Mac). Until a save shows the page has seen it, a save of the note from the page is merged into it (#draftPlan): the note it had then is kept.
807
+ */
808
+ #giveNote(s, text) {
809
+ if (text === s.note) return;
810
+ const step = { seq: s.seq, before: s.note };
811
+ if (s.noteGiven) {
812
+ const steps = [...(s.noteGiven.steps || []), step];
813
+ s.noteGiven = { ...s.noteGiven, seq: s.seq, steps: steps.slice(-50), ...(steps.length > 50 || !s.noteGiven.steps ? { cut: true } : {}) };
814
+ } else s.noteGiven = { ...step, steps: [step] };
815
+ s.note = text;
816
+ }
817
+
818
+ /** Stores a plan of #draftPlan as a draft save: refused (413) past the session cap before anything changes. */
819
+ #commitDraft(s, plan, rev, tab, quiet = false) {
820
+ // each draft remembers which tab last wrote its words, and when (#draftPlan: a save does not delete one its page has not seen)
821
+ const prev = new Map(s.comments.map((c) => [c.id, c]));
822
+ for (const c of plan.comments) {
823
+ if (c.status !== "draft") continue;
824
+ const old = prev.get(c.id);
825
+ if (!old || old.status !== "draft" || old.text !== c.text) c.wroteBy = { tab: tab ?? null, seq: s.seq + 1 };
826
+ else if (old.wroteBy && c !== old) c.wroteBy = old.wroteBy;
827
+ }
828
+ // the words of drafts this save deletes, by id (the last DELETED_KEPT): a window handing over words afterwards does not bring them back
829
+ const kept = new Set(plan.comments.map((c) => c.id));
830
+ const gone = [...prev.values()].filter((c) => c.status === "draft" && !kept.has(c.id));
831
+ const deleted = gone.length ? Object.fromEntries([...Object.entries(s.deletedDrafts || {}), ...gone.map((c) => [c.id, c.text])].slice(-DELETED_KEPT)) : s.deletedDrafts;
832
+ const patch = { comments: plan.comments, note: plan.note, seq: s.seq + 1, ...(deleted ? { deletedDrafts: deleted } : {}) };
833
+ const { raw, cut } = this.#encode({ ...s, ...patch });
834
+ if (Buffer.byteLength(raw) > SESSION_MAX) throw tooLarge("This draft");
835
+ if (Number.isSafeInteger(rev) && !this.#overtaken(s, rev, tab)) this.#revs.set(s.key, { tab, rev }); // never lowered
836
+ Object.assign(s, patch, cut || {});
837
+ if (s.noteGiven && Number.isSafeInteger(plan.seen) && plan.seen > s.noteGiven.seq) delete s.noteGiven; // the page has seen it
838
+ writeStateFile(path.join(sub("sessions"), `${s.key}.json`), raw);
839
+ // `quiet`: a Send's (or a Use's) save, which its commit announces; an event for the moment between would show its note still in the box
840
+ if (!quiet) this.emit("change", s, "draft");
841
+ }
842
+
843
+ /** One draft as stored, from the page's comment `c` (`old`: the stored draft of the same id, if any). */
844
+ #draft(c, id, old) {
845
+ const base = {
846
+ id,
847
+ kind: ["text", "element", "pin"].includes(c.kind) ? c.kind : "element",
848
+ text: userText(c.text, COMMENT_TEXT_MAX),
849
+ anchor: cleanPageAnchor(c.anchor), // the fields the server and the page read, each cut (src/retain.js)
850
+ pin: c.pin && typeof c.pin === "object" ? cleanPin(c.pin) : null,
851
+ pinNo: Number.isInteger(c.pinNo) ? c.pinNo : null,
852
+ };
853
+ // an answered question: the words, the question and the answer stay as the person left them (the page cannot change them)
854
+ const keep = old?.skipLocal ? { skipLocal: true } : {}; // a reverted local run: the next Send of this comment goes to the agent
855
+ if (old?.localHint && base.text === old.text) keep.localHint = old.localHint; // the words changed: the hint no longer applies
856
+ if (old?.answer) return capComment({ ...base, text: old.text, status: "draft", durable: old.durable || null, question: old.question, answer: old.answer, ...keep });
857
+ return capComment({ ...base, status: "draft", durable: old?.durable || null, ...keep });
858
+ }
859
+
860
+ /**
861
+ * What a Send (or a quick fix's Use: `useId`, the stamp it delivers) stores before it commits, as #draftPlan: nothing changed.
862
+ * Invariant: a Send commits exactly its own payload, whatever draft save is numbered before or after it. Normally its list and note
863
+ * replace the stored ones, as a draft save does. When this tab already numbered a newer save (one sent after the Send was pressed whose
864
+ * body arrived first, as a page's visibility-change save can), that save is the stored list and the Send is authoritative only for what
865
+ * it delivers: the drafts it names (their words as the Send carried them) and its note. Everything else stays as the newer save left it:
866
+ * a comment it added is a draft, one it deleted stays deleted (the page cannot delete what a Send in flight carries), its note (none:
867
+ * it leaves the note to the Send). A Use delivers only its stamp, so then it changes nothing else, not the note either.
868
+ */
869
+ #sentPlan(s, { comments, note, rev, tab, seen, noteSeen, sends }, useId = null) {
870
+ const list = Array.isArray(comments) ? comments : s.comments;
871
+ // `sends` (optional): the ids the page holds read only for this Send, the drafts it means to deliver; a comment the page still showed as
872
+ // queued or sent when Send was pressed (and could be typed into after) is not delivered by it
873
+ const named = Array.isArray(sends) ? new Set(sends.slice(0, COMMENTS_MAX).map((id) => cutText(String(id), 64))) : null;
874
+ if (!this.#overtaken(s, rev, tab)) {
875
+ const plan = this.#draftPlan(s, { comments: list, note: typeof note === "string" ? note : s.note, seen, noteSeen, tab });
876
+ return named && useId == null ? { ...plan, only: named } : plan;
877
+ }
878
+ const own = this.#draftPlan(s, { comments: list.filter((c) => c && typeof c === "object" && (useId == null || String(c.id) === useId)), note: useId == null && typeof note === "string" ? note : s.note, seen, noteSeen, tab });
879
+ const mine = new Map(own.comments.map((c) => [c.id, c]));
880
+ // `only`: the drafts this Send delivers (the others are the newer save's, and stay drafts)
881
+ return { comments: s.comments.map((c) => (c.status === "draft" && mine.has(c.id) ? mine.get(c.id) : c)), note: own.note, only: new Set([...mine.keys()].filter((id) => !named || useId != null || named.has(id))) };
882
+ }
883
+
884
+ /** The drafts of `comments` a Send with `plan` delivers: all of them, or those it names when a newer save was stored first. */
885
+ #delivers(comments, plan) {
886
+ return comments.filter((c) => c.status === "draft" && (!plan.only || plan.only.has(c.id)));
887
+ }
888
+
889
+ /**
890
+ * The 413 refusals of a Send, on the state its plan would make (nothing changed yet): the bytes past the cap, a note that joined with
891
+ * the one waiting (the batch an agent has not taken, or the queued note during a run) would be past NOTE_MAX, a session that would not
892
+ * fit its file. Run before the worker is asked and again at the commit, against the state then.
893
+ */
894
+ #sendLimits(s, plan, payload) {
895
+ if (payload.bytes?.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
896
+ if (s.delivered) joinNotes(s.queuedNote, plan.note);
897
+ else joinNotes(s.pending && !readOnlyKind(s.pending) ? s.pending.instructions?.overall_note || "" : "", plan.note);
898
+ if (!this.#fits(s, { comments: plan.comments, edits: [...s.edits, ...this.#editsToTake(s, payload.edits)] })) throw tooLarge("This Send");
367
899
  }
368
900
 
369
901
  // ---- send ------------------------------------------------------------------------------------------------
@@ -372,58 +904,211 @@ export class Sessions extends EventEmitter {
372
904
  * Compare-and-swap on the disk hash. Returns the session.
373
905
  */
374
906
  send(key, payload) {
907
+ return scanBudget(SCAN_BUDGET, () => this.#send(key, payload));
908
+ }
909
+
910
+ /**
911
+ * send() as the server's route calls it: the reading of the bytes that become the file and every anchor of the batch are worked out
912
+ * first, in the warm anchor worker with a time limit (src/offload.js anchorsOffThread), then send() commits with what came back and
913
+ * reads nothing itself. Work that did not finish leaves its comments weaker (src/anchor-work.js weakDurable), never lost; bytes it
914
+ * could not read in time are refused with "try again" unless that version was read whole before.
915
+ * Invariant: a Send and the page's draft saves, Take back and answers of one file run in turn (inTurn), as when Send was synchronous.
916
+ * The Send saves its list and note when its turn comes, then commits exactly that; whatever arrives meanwhile waits and is applied after
917
+ * the commit, against what it committed. A page loaded meanwhile waits for it too (settled).
918
+ */
919
+ async sendAsync(key, payload) {
920
+ const s = this.get(key);
921
+ const hasBytes = !!payload.bytes?.length;
922
+ if (s.closed || (s.delivered && hasBytes)) return this.send(key, payload); // refused there, before anything changes
923
+ return this.#serial(key, async () => {
924
+ // refused there, before anything changes: closed, or locked
925
+ if (s.closed || (s.delivered && hasBytes)) return this.send(key, payload);
926
+ // nothing is stored before the commit: the worker works on what the Send would store, every limit is checked on it first (a refusal
927
+ // here changes nothing) and again at the commit
928
+ let prepared = null;
929
+ let html = {};
930
+ for (let attempt = 0; attempt < 2; attempt++) {
931
+ const queued = !!s.delivered;
932
+ const plan = this.#sentPlan(s, payload);
933
+ this.#sendLimits(s, plan, payload);
934
+ if (s.format === "html") this.#htmlSendChecks(s, payload);
935
+ const job = this.#sendJob(s, payload, plan);
936
+ // an HTML file that is not queued: the person's text edits are written through the write check, and every anchor is made on the
937
+ // version it was made on, then moved onto the bytes this Send writes (#htmlSendWork)
938
+ if (s.format === "html" && !s.delivered) ({ prepared, html } = await this.#htmlSendWork(s, payload, job));
939
+ else prepared = job ? await this.#offThread(s.format, job) : null;
940
+ if (!!s.delivered === queued) break; // a run started or ended while it worked: plan again once, for the other case
941
+ }
942
+ return this.send(key, { ...payload, ...html, prepared });
943
+ });
944
+ }
945
+
946
+ /** An HTML Send's own refusals, before any work: no bytes from the browser ever (PLAN-html 3.1), no text edits during a run (D5). */
947
+ #htmlSendChecks(s, payload) {
948
+ if (payload.bytes?.length && !payload.htmlChecked) throw new HttpError(400, "an HTML file's bytes never come from the page; send its text edits", { code: "no_bytes_for_html" });
949
+ if (payload.htmlEdits?.length && s.delivered) throw new HttpError(423, s.delivered.local ? "the document is locked while a change is made on this Mac" : "the document is locked while the agent works", { code: "locked" });
950
+ }
951
+
952
+ /**
953
+ * The work of an HTML Send that is not queued, off this thread (the parser exists only in workers):
954
+ * 1. the person's text edits (R9), when there are any: on the version the page loaded, which must be the file on disk (else 409
955
+ * version_changed: the page reloads the view and places its edits again), applied by the write check in a worker (src/html-write.js
956
+ * checkedHtmlWrite "edits": one write, the proof, the inventory). Its bytes become the file at the commit (#takePage), which checks
957
+ * the disk hash again after these awaits (R12).
958
+ * 2. the anchors: a page anchor names a node of ONE version (`location.version`, a node id means nothing in another). An anchor made on
959
+ * the version this Send writes is built there; one made on another version (a draft that outlived a reload, the typed edits'
960
+ * places when this Send changes the bytes) is built on that version first (the file on disk or a stored snapshot), then moved onto
961
+ * the bytes this Send writes (reanchor), as a queued comment is moved onto the agent's file at done.
962
+ * Returns {prepared (what #offThread returns, keyed by this Send's own tasks), html: {bytes, htmlChecked} or {}}.
963
+ */
964
+ async #htmlSendWork(s, payload, job) {
965
+ let disk;
966
+ try {
967
+ checkRegularOwned(s.file);
968
+ disk = readCapped(s.file);
969
+ } catch (err) {
970
+ throw new HttpError(409, `cannot read the file: ${err.message}`, { code: "gone" });
971
+ }
972
+ const diskSha = sha256(disk);
973
+ let target = { bytes: disk, sha: diskSha };
974
+ let html = {};
975
+ const edits = cleanHtmlEdits(payload.htmlEdits);
976
+ if (edits.length) {
977
+ if (payload.baseSha !== diskSha || edits.some((e) => e.versionSha !== diskSha)) throw new HttpError(409, VERSION_CHANGED, { code: "version_changed", diskSha });
978
+ const c = await htmlWriteOffThread({ before: disk, edits: edits.map((e) => ({ nodeId: e.nodeId, text: e.newText, baseTextHash: e.baseTextHash })) });
979
+ if (!c.ok) throw htmlEditRefusal(c);
980
+ const bytes = Buffer.from(c.bytes.buffer, c.bytes.byteOffset, c.bytes.byteLength);
981
+ target = { bytes, sha: sha256(bytes) };
982
+ html = { bytes, htmlChecked: { from: diskSha, to: target.sha } };
983
+ }
984
+ const prepared = await this.#htmlAnchorWork(s, job.tasks, target, { disk, diskSha, check: true, outline: job.outline });
985
+ return { prepared, html };
986
+ }
987
+
988
+ /**
989
+ * `tasks` (build, reanchor and view tasks of one request) on the version `target` ({bytes, sha}), in the anchor worker, with every build
990
+ * task whose page anchor was made on another version (`location.version`) built on that version first (`known.disk` when it is the file
991
+ * on disk, else a stored snapshot) and moved onto `target`. One made on a version no longer stored is moved as a weak anchor (its quote
992
+ * searched). Returns what #offThread returns, its map keyed by the tasks given.
993
+ */
994
+ async #htmlAnchorWork(s, tasks, target, { disk = null, diskSha = null, check = false, outline = false } = {}) {
995
+ const elsewhere = new Map(); // version -> [task index]
996
+ tasks.forEach((t, i) => {
997
+ const v = t.op === "build" ? t.anchor?.location?.version : null;
998
+ if (v && v !== target.sha) (elsewhere.get(v) || elsewhere.set(v, []).get(v)).push(i);
999
+ });
1000
+ const moved = new Map(); // task index -> the durable made on its own version (null: not there; undefined: not worked out)
1001
+ for (const [v, list] of elsewhere) {
1002
+ const source = v === diskSha && disk ? { bytes: disk } : s.snapshots.has(v) ? { path: s.snapshots.blob(v), sha: v } : null;
1003
+ if (!source) continue;
1004
+ const r = await this.#offThread("html", { source, tasks: list.map((i) => tasks[i]) });
1005
+ if (r.ok && r.read?.ok) list.forEach((i) => moved.set(i, r.map.get(taskKey(tasks[i]))));
1006
+ }
1007
+ const asked = tasks.map((t, i) => {
1008
+ if (!elsewhere.size || !(t.op === "build" && t.anchor?.location?.version && t.anchor.location.version !== target.sha)) return t;
1009
+ const d = moved.get(i);
1010
+ return { op: "reanchor", c: { durable: usable(d) && d ? d : weakDurable("html", t.anchor), anchor: null } };
1011
+ });
1012
+ const r = await this.#offThread("html", { source: { bytes: target.bytes }, check, tasks: asked, outline });
1013
+ if (!r.ok) return r;
1014
+ const map = new Map();
1015
+ asked.forEach((t, i) => map.set(taskKey(tasks[i]), r.map.get(taskKey(t))));
1016
+ return { ...r, sha: target.sha, map };
1017
+ }
1018
+
1019
+ /** What a Send asks of the anchor worker (the same tasks #send makes, for the list it would store), or null when it needs nothing read. */
1020
+ #sendJob(s, payload, plan) {
1021
+ if (s.delivered) {
1022
+ const need = this.#delivers(plan.comments, plan).filter((c) => !c.durable);
1023
+ if (!need.length || !s.snapshots.has(s.delivered.preSha)) return null;
1024
+ return { source: { path: s.snapshots.blob(s.delivered.preSha), sha: s.delivered.preSha }, tasks: need.map(buildTask) };
1025
+ }
1026
+ const { tasks } = this.#sendTasks(s, this.#editsToTake(s, payload.edits), payload.viewState, plan);
1027
+ const bytes = payload.bytes?.length ? payload.bytes : null;
1028
+ return { source: bytes ? { bytes } : { path: s.file }, check: true, tasks, outline: this.#wantsOutline(s) };
1029
+ }
1030
+
1031
+ /** The tasks of a Send that is not queued: every comment that goes out (drafts, and those of a batch still waiting), the typed edits without an anchor yet, the view. */
1032
+ #sendTasks(s, newEdits, viewState, plan = { comments: s.comments }) {
1033
+ const go = new Set(this.#delivers(plan.comments, plan));
1034
+ const eligible = plan.comments.filter((c) => go.has(c) || c.status === "sent");
1035
+ const edits = [...s.edits, ...newEdits].filter((e) => !e.durable);
1036
+ const vs = cleanView(viewState);
1037
+ return { eligible, edits, vs, tasks: [...eligible.map(buildTask), ...edits.map((e) => ({ op: "build", anchor: editAnchor(e), pin: false })), ...(vs ? [{ op: "view", vs }] : [])] };
1038
+ }
1039
+
1040
+ /**
1041
+ * The commit of a Send: the Send's own list and note are saved first (#sentPlan), then every draft goes out with the note. A batch still
1042
+ * waiting for an agent (not delivered) is replaced by one that carries its comments and its note, then this one's; during a run the
1043
+ * drafts and the note are queued after those already queued. Every 413 (#sendLimits) is refused before anything changes.
1044
+ */
1045
+ #send(key, payload) {
375
1046
  const s = this.get(key);
376
1047
  if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
1048
+ if (s.format === "html") this.#htmlSendChecks(s, payload);
377
1049
  const hasBytes = !!payload.bytes?.length;
378
1050
  if (s.delivered && hasBytes) throw new HttpError(423, s.delivered.local ? "the document is locked while a change is made on this Mac" : "the document is locked while the agent works", { code: "locked" }); // before touching any state
379
- this.setDraft(key, { comments: payload.comments ?? s.comments, note: payload.note ?? s.note });
1051
+ // the Send's own list and note (#sentPlan), every limit checked on them before anything changes; then stored as a draft save is, so a
1052
+ // refusal after this (the file changed, could not be read in time) leaves them as editable drafts
1053
+ const plan = this.#sentPlan(s, payload);
1054
+ this.#sendLimits(s, plan, payload);
1055
+ this.#commitDraft(s, plan, payload.rev, payload.tab, true);
1056
+ try {
1057
+ return this.#sendCommit(s, payload, plan);
1058
+ } catch (err) {
1059
+ this.emit("change", s, "draft"); // refused after the save: the page's tabs see the drafts it stored (a Send that commits says so once, at the end)
1060
+ throw err;
1061
+ }
1062
+ }
1063
+
1064
+ /** The rest of #send, once its list and note are stored: the batch (or the queue) and the commit. */
1065
+ #sendCommit(s, payload, plan) {
380
1066
  if (s.delivered) {
381
1067
  // queue: comments and note only. A new comment gets its durable anchor from the bytes the page shows (the pre-agent snapshot),
382
1068
  // never from the disk, which the agent may already have changed; done() then moves it onto the post-agent file.
383
- const fresh = s.comments.filter((c) => c.status === "draft");
384
- if (fresh.some((c) => !c.durable)) {
385
- let units = null;
386
- try {
387
- units = readUnits(s.format, s.snapshots.read(s.delivered.preSha));
388
- } catch {
389
- units = null; // no snapshot: done() falls back to building from the post-agent file
390
- }
391
- if (units) {
392
- for (const c of fresh) {
393
- if (c.durable) continue;
394
- c.durable = buildDurable(units, c.anchor);
395
- if (units.format === "pptx" && c.kind === "pin" && c.durable?.shapeId != null) c.durable.pinFallback = buildDurable(units, { location: { slideIndex: c.durable.slideIndex, slideId: c.durable.slideId } });
396
- }
397
- }
1069
+ const queuedNote = joinNotes(s.queuedNote, s.note); // before anything changes
1070
+ const fresh = this.#delivers(s.comments, plan);
1071
+ const need = fresh.filter((c) => !c.durable);
1072
+ if (need.length) {
1073
+ // no snapshot, or work that did not finish: the anchor stays unset, and done() builds it from the post-agent file
1074
+ const work = this.#results(s.format, () => s.snapshots.read(s.delivered.preSha), s.delivered.preSha, need.map(buildTask), payload.prepared);
1075
+ if (work.read?.ok) need.forEach((c, i) => (c.durable = usable(work.at(i)) ? work.at(i) : null));
398
1076
  }
399
1077
  for (const c of fresh) c.status = "queued";
400
- if (s.note) {
401
- // the note belongs to the queued batch now; the stored note is cleared so the next Send does not repeat it
402
- s.queuedNote = [s.queuedNote, s.note].filter(Boolean).join("\n\n");
403
- s.note = "";
404
- }
1078
+ // the note belongs to the queued batch now; it leaves the box so the next Send does not repeat it
1079
+ s.queuedNote = queuedNote;
1080
+ s.note = "";
405
1081
  if (fresh.length || s.queuedNote) s.queued = true; // a Send with nothing new queues nothing (no empty batch after this one)
1082
+ this.#sentRev(s, payload, true);
406
1083
  this.#changed(s);
407
1084
  this.#voice(s, { sent: fresh });
408
1085
  return s;
409
1086
  }
410
- const { diskSha, written, writtenSha, writeInfo, lastUpdate } = this.#takePage(s, payload);
1087
+ // a batch still waiting for an agent is replaced: its comments go again (they are "sent"), and so does its note, first
1088
+ const waiting = s.pending && !readOnlyKind(s.pending) ? s.pending.instructions?.overall_note || "" : "";
1089
+ const note = joinNotes(waiting, s.note); // before anything is written
1090
+ // the bytes that become the file are read whole before anything is written (one past a reader's cap is refused: CapError, nothing
1091
+ // changes), and every anchor of the batch is built from them, by the anchor worker (sendAsync) or here
1092
+ const newEdits = this.#editsToTake(s, payload.edits);
1093
+ const { eligible, edits, vs, tasks } = this.#sendTasks(s, newEdits, payload.viewState, { ...plan, comments: s.comments });
1094
+ // what is stored must fit its file once cut (src/retain.js SESSION_MAX): checked before the file is written or anything changes
1095
+ if (!this.#fits(s, { comments: s.comments, edits: [...s.edits, ...newEdits] })) throw tooLarge("This Send");
1096
+ let work = null;
1097
+ const { diskSha, writtenSha, writeInfo, lastUpdate } = this.#takePage(s, payload, (b, sha) => {
1098
+ work = this.#results(s.format, b, sha, tasks, payload.prepared, { check: true });
1099
+ this.#readVerdict(work, s.format, sha, "refusing to write");
1100
+ });
411
1101
 
412
1102
  // user edits since the last Send (document-derived text) accumulate until a batch is delivered
413
- for (const e of Array.isArray(payload.edits) ? payload.edits.slice(0, 2000) : []) s.edits.push({ kind: e.kind, before: e.before, after: e.after, label: e.label, location: e.location });
414
-
415
- const units = readUnits(s.format, written);
416
- const resolve = (anchor) => buildDurable(units, anchor);
417
- const eligible = s.comments.filter((c) => c.status === "draft" || c.status === "sent");
418
- for (const c of eligible) {
419
- c.durable = resolve(c.anchor);
420
- // a PPTX pin dropped on a shape also remembers its slide: if the shape is deleted the pin stays on the slide
421
- if (units.format === "pptx" && c.kind === "pin" && c.durable?.shapeId != null) c.durable.pinFallback = buildDurable(units, { location: { slideIndex: c.durable.slideIndex, slideId: c.durable.slideId } });
422
- }
423
- for (const e of s.edits) e.durable = e.durable || resolve({ location: e.location, quote: e.after });
424
- s.view = this.#resolveView(s, units, payload.viewState);
425
-
426
- const batch = buildBatch({ file: s.file, format: s.format, baseSha: writtenSha, note: s.note, comments: eligible, edits: s.edits, style: styleNote(), guide: voiceGuide() });
1103
+ s.edits.push(...newEdits);
1104
+ // a comment whose anchor work did not finish goes out weaker (src/anchor-work.js weakDurable); a PPTX pin on a shape also remembers its slide
1105
+ eligible.forEach((c, i) => (c.durable = usable(work.at(i)) ? work.at(i) : weakDurable(s.format, c.anchor)));
1106
+ edits.forEach((e, i) => (e.durable = usable(work.at(eligible.length + i)) ? work.at(eligible.length + i) : weakDurable(s.format, editAnchor(e))));
1107
+ const v = vs ? work.at(eligible.length + edits.length) : null;
1108
+ s.view = !vs ? null : usable(v) ? v : { format: s.format, raw: vs };
1109
+ if (work.outline) this.#outline = { key: s.key, sha: writtenSha, all: work.outline };
1110
+
1111
+ const batch = buildBatch({ file: s.file, format: s.format, baseSha: writtenSha, note, comments: eligible, edits: s.edits, style: styleNote(), guide: voiceGuide() });
427
1112
  const sent = eligible.filter((c) => c.status === "draft");
428
1113
  for (const c of eligible) {
429
1114
  c.status = "sent";
@@ -440,6 +1125,7 @@ export class Sessions extends EventEmitter {
440
1125
  this.#deliverIfPolled(s);
441
1126
  delete s.quickConfirm;
442
1127
  delete s.skipLocalNote; // it held for this Send only (the decision for its batch is made)
1128
+ this.#sentRev(s, payload, true);
443
1129
  this.#changed(s);
444
1130
  this.#voice(s, { last: lastUpdate, diskSha, writtenSha, sent });
445
1131
  if (this.state(s) === "stranded" && !s.pollers.length) this.emit("stranded", s); // nobody took it: the server may start its background agent
@@ -449,8 +1135,8 @@ export class Sessions extends EventEmitter {
449
1135
  /**
450
1136
  * The page's bytes become the file, as at Send (and at a quick fix's Use): the disk read, the compare-and-swap against the version
451
1137
  * the page loaded, the package check, the guarded replace, the "sent" snapshot, the new base and the person's own history update.
452
- * `check(bytes)` (optional) sees the bytes that are about to be the file, after the package check and before anything is written, and
453
- * may throw to refuse. Returns {disk, diskSha, written, writtenSha, writeInfo, lastUpdate}.
1138
+ * `check(bytes, sha)` (optional) sees the bytes that are about to be the file and their sha256, after the package check and before
1139
+ * anything is written, and may throw to refuse. Returns {disk, diskSha, written, writtenSha, writeInfo, lastUpdate}.
454
1140
  */
455
1141
  #takePage(s, payload, check = null) {
456
1142
  const hasBytes = !!payload.bytes?.length;
@@ -462,25 +1148,33 @@ export class Sessions extends EventEmitter {
462
1148
  throw new HttpError(409, `cannot read the file: ${err.message}`, { code: "gone" });
463
1149
  }
464
1150
  const diskSha = sha256(disk);
465
- if (diskSha !== payload.baseSha) {
466
- throw this.#outsideChange(s, { side: "user", bytes: payload.bytes, diskSha, baseSha: payload.baseSha });
1151
+ // an HTML file's text edits were applied by its write check (#htmlSendWork) to the version the page loaded: no bytes come from the page
1152
+ const htmlChecked = s.format === "html" ? payload.htmlChecked ?? null : null;
1153
+ if (diskSha !== payload.baseSha || (htmlChecked && htmlChecked.from !== diskSha)) {
1154
+ // an HTML page sent no bytes: nothing of its to keep as a copy (its edits stay in the page, which reloads and places them again)
1155
+ throw this.#outsideChange(s, { side: "user", bytes: s.format === "html" ? null : payload.bytes, diskSha, baseSha: payload.baseSha });
467
1156
  }
468
1157
  let written = disk;
469
1158
  let writeInfo = null;
470
1159
  if (hasBytes) {
471
- if (payload.bytes.length > MAX_BYTES) throw new HttpError(413, "document too large");
472
- try {
473
- validatePackage(payload.bytes, s.format);
474
- } catch (err) {
475
- throw new HttpError(422, `refusing to write: ${err.message}`, { code: "invalid_package" });
1160
+ if (payload.bytes.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
1161
+ if (s.format === "html") {
1162
+ if (htmlChecked?.to !== sha256(payload.bytes)) throw new HttpError(500, "an HTML file is written only after its write check", { code: "unchecked_write" });
1163
+ } else {
1164
+ try {
1165
+ validatePackage(payload.bytes, s.format);
1166
+ } catch (err) {
1167
+ throw new HttpError(422, `refusing to write: ${err.message}`, { code: "invalid_package" });
1168
+ }
476
1169
  }
477
1170
  }
478
- if (check) check(hasBytes ? payload.bytes : disk);
1171
+ const writtenSha = hasBytes ? sha256(payload.bytes) : diskSha;
1172
+ if (htmlChecked && hasBytes) readInCaps("html", writtenSha); // the write check read these bytes whole within the limits
1173
+ if (check) check(hasBytes ? payload.bytes : disk, writtenSha);
479
1174
  if (hasBytes) {
480
- writeInfo = this.#replace(s, payload.bytes, { expectSha: diskSha, label: "before send", side: "user" });
1175
+ writeInfo = this.#replace(s, payload.bytes, { expectSha: diskSha, label: "before send", side: "user", htmlChecked });
481
1176
  written = payload.bytes;
482
1177
  }
483
- const writtenSha = sha256(written);
484
1178
  s.snapshots.add(written, "sent");
485
1179
  s.baseSha = writtenSha;
486
1180
  const lastUpdate = s.history?.at(-1); // learning pairs an agent run with the person's edits right after it
@@ -504,16 +1198,21 @@ export class Sessions extends EventEmitter {
504
1198
  * (DOCX paragraphs, formatting and moves: src/merge.js; XLSX cells: src/merge-xlsx.js; PPTX shape text: src/merge-pptx.js) is carried onto it as ops (same engine, expect, protected and package checks
505
1199
  * as an agent's apply) and written as the user's own update. Refused while a batch is delivered (423): the page waits and asks
506
1200
  * again. Returns {merged, unplaced, sha, same, mineSha, copy_name}.
507
- * Ceiling: the engine runs in the server process (about 0.1 s on a 300-paragraph file); move it to a worker if large files stall polls.
1201
+ * The reading of the three versions within the readers' caps, the plan and the engine's run happen in a worker thread with a time and
1202
+ * memory limit (src/offload.js mergeOffThread): the server's thread stays free whatever the document. A merge that fails or runs out of
1203
+ * time writes nothing: the person's version is saved as a copy next to the file and the page keeps their edits (503 merge_failed).
1204
+ * One merge per session at a time (another is 423 locked; the page has only one in flight).
508
1205
  */
509
1206
  async mergeUserEdits(key, { baseSha, bytes, edits = [] }) {
510
1207
  const s = this.get(key);
511
1208
  if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
512
1209
  if (s.delivered) throw new HttpError(423, s.delivered.local ? "a change is being made on this Mac" : "the agent is working", { code: "locked" });
513
- const merger = MERGERS[s.format];
1210
+ if (s.merging) throw new HttpError(423, "your edits are being merged already", { code: "locked" });
1211
+ // an HTML view is read only during a run (PLAN-html D5): nothing is typed to merge, and its bytes never come from the page
1212
+ if (s.format === "html") throw new HttpError(400, "an HTML file's edits are never merged: its view is read only while the agent works", { code: "not_for_html" });
514
1213
  if (!/^[0-9a-f]{64}$/.test(baseSha)) throw new HttpError(400, "baseSha must be a sha256"); // it names a snapshot file
515
1214
  if (!bytes?.length) throw new HttpError(400, "no document bytes");
516
- if (bytes.length > MAX_BYTES) throw new HttpError(413, "document too large");
1215
+ if (bytes.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
517
1216
  try {
518
1217
  validatePackage(bytes, s.format);
519
1218
  } catch (err) {
@@ -535,8 +1234,19 @@ export class Sessions extends EventEmitter {
535
1234
  } catch {
536
1235
  throw new HttpError(409, "the version this page started from is no longer stored", { code: "no_base" });
537
1236
  }
538
- const [mod, { runOps }] = await Promise.all([import(merger.module), import("./ops/run.js")]);
539
- const r = await mod[merger.fn]({ base, mine: bytes, theirs: disk, name: path.basename(s.file), runOps, edits: editKinds(edits), agent: s.format === "xlsx" ? this.#writesBetween(s, baseSha, diskSha, disk) : null });
1237
+ // the three versions are read whole within the readers' caps in the worker first (the page's version, a file swapped on disk and an
1238
+ // old snapshot alike): past one, the merge is refused with the reason and nothing is written
1239
+ let r;
1240
+ s.merging = true;
1241
+ try {
1242
+ r = await mergeOffThread({ format: s.format, base, mine: bytes, theirs: disk, name: path.basename(s.file), edits: editKinds(edits), agent: s.format === "xlsx" ? this.#writesBetween(s, baseSha, diskSha, disk) : null });
1243
+ } finally {
1244
+ s.merging = false;
1245
+ }
1246
+ if (r.cap) throw new CapError(r.cap);
1247
+ if (r.invalid) throw new HttpError(422, `refusing to merge: ${untrusted(r.invalid, 200)}`, { code: "invalid_package" });
1248
+ if (!r.ok) throw this.#mergeFailed(s, bytes, r);
1249
+ if (r.bytes) r.bytes = Buffer.from(r.bytes.buffer, r.bytes.byteOffset, r.bytes.byteLength); // a Uint8Array once it crossed threads
540
1250
  if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
541
1251
  if (s.delivered) throw new HttpError(423, s.delivered.local ? "a change is being made on this Mac" : "the agent is working", { code: "locked" }); // a batch was delivered while the engine ran
542
1252
  // The page shows the first HELD_SHOWN items and drops its own version when it reloads. When the list stands for more than it
@@ -566,6 +1276,24 @@ export class Sessions extends EventEmitter {
566
1276
  return { same: false, merged: r.merged, unplaced: r.unplaced.slice(0, 50), sha, mineSha: sha256(bytes), copy_name: copy?.name ?? null };
567
1277
  }
568
1278
 
1279
+ /**
1280
+ * A merge that did not finish (out of time or memory, busy, crashed) or whose result the proof refused: nothing was written. The person's version is saved next to the
1281
+ * file (except when Onionskin was only busy) and the answer says so in words the page shows inside its own sentence ("could not be
1282
+ * merged into its version (...). Your edits are still in this page: download them before you reload.").
1283
+ */
1284
+ #mergeFailed(s, bytes, r) {
1285
+ if (r.failed === "busy") return new HttpError(503, "Onionskin is merging other files right now; nothing was written, so try again in a moment", { code: "merge_busy" });
1286
+ let copy = null;
1287
+ try {
1288
+ copy = writeSiblingCopy(s.file, bytes);
1289
+ this.createdCopies.add(copy.path);
1290
+ } catch {}
1291
+ // refused: the proof of the merge (src/merge-proof.js) could not account for every edit of the person's in the result; the reason
1292
+ // holds counts and kinds of edits, never their text
1293
+ const why = r.failed === "timeout" || r.failed === "memory" ? r.why : r.refused ? `the merged file could not be checked to hold every one of your edits (${r.refused})` : "the merge stopped with an error";
1294
+ return new HttpError(503, `${why}; nothing was written${copy ? `, and a copy of your version was saved as ${copy.name} next to the file` : ""}`, { code: "merge_failed", copy_name: copy?.name ?? null });
1295
+ }
1296
+
569
1297
  #logWrite(s, entry) {
570
1298
  const list = this.writes.get(s.key) || [];
571
1299
  list.push(entry);
@@ -606,7 +1334,9 @@ export class Sessions extends EventEmitter {
606
1334
  * replacement (see replaceGuarded). `expectSha` is the hash the caller compared against; a file that is different now, or becomes
607
1335
  * different just before the rename, is not overwritten (the newer bytes are snapshotted) and the caller gets the 409 conflict.
608
1336
  */
609
- #replace(s, bytes, { expectSha, label, side = null }) {
1337
+ #replace(s, bytes, { expectSha, label, side = null, htmlChecked = null }) {
1338
+ // R1, R12: an HTML file is written only with bytes its write check passed in a worker (agentWriteChecked); every other path refuses
1339
+ if (s.format === "html" && htmlChecked?.to !== sha256(bytes)) throw new HttpError(500, "an HTML file is written only after its write check", { code: "unchecked_write" });
610
1340
  try {
611
1341
  return replaceGuarded(s.file, bytes, {
612
1342
  expectSha,
@@ -680,14 +1410,6 @@ export class Sessions extends EventEmitter {
680
1410
  return { stale: !s.delivered && !!loadedSha && diskSha !== loadedSha, diskSha };
681
1411
  }
682
1412
 
683
- #resolveView(s, units, vs) {
684
- if (!vs) return null;
685
- if (units.format === "docx" && vs.format !== "docx") return { format: s.format, raw: vs };
686
- if (units.format !== "docx") return viewDurable(units, vs) || { format: s.format, raw: vs };
687
- const d = durableFromEngine(units.paragraphs, { blockPath: vs.blockPath }, { quote: vs.quote });
688
- return { format: "docx", durable: d, offsetPx: Number(vs.offsetPx) || 0, zoom: vs.zoom ?? "fit", fraction: vs.fraction };
689
- }
690
-
691
1413
  // ---- poll / delivery -------------------------------------------------------------------------------------
692
1414
  /**
693
1415
  * Registers a poller. Returns a handle {promise, cancel}. Resolves with the batch object, or {status:'superseded'|'closed'}.
@@ -772,17 +1494,30 @@ export class Sessions extends EventEmitter {
772
1494
  * written to <state>/agents/<key>/outline/outline-<batch id>.jsonl (0600, atomic, one unit per line), the only folder that agent may read,
773
1495
  * and the batch names it. Older outline files of the session go at each such delivery, and with the folder on Close.
774
1496
  * Any other holder, a review (it has `units`) and a voice request (it quotes only the sources the person ticked) get the batch as stored.
775
- * A file that cannot be read gets no outline (the agent declines what it cannot find); an outline file that cannot be written leaves the
776
- * inline part with `outline_more`. Ceiling: read and parsed on the main thread at each such delivery, as a review is; upgrade path as in unitsOf.
1497
+ * A file that cannot be read gets no outline, and `outline_unavailable` says why (the agent declines what it cannot find); an outline file that cannot be written leaves the
1498
+ * inline part with `outline_more`. The outline is built in the anchor worker at Send, at done and before a poll's delivery (outlineAhead),
1499
+ * and found here by the file's hash. Ceiling: a delivery that finds none (a synchronous caller, or the file changed in between) reads and
1500
+ * parses the file on this thread, as a review does; upgrade path as in unitsOf.
777
1501
  */
778
1502
  #withOutline(s, batch, holder) {
779
1503
  if (!isAgentHolder(holder) || batch.instructions?.review || batch.instructions?.voice_request) return batch;
780
1504
  let all;
781
1505
  try {
782
1506
  checkRegularOwned(s.file);
783
- all = outlineOf(s.format, readCapped(s.file), OUTLINE_FILE_MAX);
784
- } catch {
785
- return batch;
1507
+ const bytes = readCapped(s.file);
1508
+ const sha = sha256(bytes);
1509
+ // the same version's outline is built once (an agent's polls and a Send's delivery read the same file), on the memo's reading of it
1510
+ if (this.#outline?.key === s.key && this.#outline.sha === sha) all = this.#outline.all;
1511
+ // an HTML file is never parsed on this thread: its outline comes from the anchor worker (Send, done, outlineAhead) or not at all
1512
+ else if (s.format === "html") return { ...batch, untrusted_document_evidence: { ...batch.untrusted_document_evidence, outline_unavailable: "Onionskin did not finish reading this page's outline in time. Use the anchors and their quotes; poll again for an outline." } };
1513
+ else {
1514
+ all = outlineOf(s.format, bytes, OUTLINE_FILE_MAX, unitsFor(s.format, bytes, sha));
1515
+ this.#outline = { key: s.key, sha, all };
1516
+ }
1517
+ } catch (err) {
1518
+ // the agent is told why it has no outline (a file changed past a reader's cap behind Onionskin, or gone), in the reader's plain words
1519
+ const why = err instanceof CapError ? `Onionskin cannot read this file: ${err.message}.` : "Onionskin cannot read this file as it is now.";
1520
+ return { ...batch, untrusted_document_evidence: { ...batch.untrusted_document_evidence, outline_unavailable: why } };
786
1521
  }
787
1522
  const inline = withOutline(batch, all, VOICE_BATCH_MAX_CHARS, { inlineMax: UNITS_MAX });
788
1523
  const name = `outline-${batch.batch_id}.jsonl`;
@@ -844,7 +1579,7 @@ export class Sessions extends EventEmitter {
844
1579
  #queuedToDrafts(s) {
845
1580
  if (s.queued) {
846
1581
  for (const c of s.comments) if (c.status === "queued") c.status = "draft";
847
- s.note = [s.queuedNote, s.note].filter(Boolean).join("\n\n");
1582
+ this.#giveNote(s, [s.queuedNote, s.note].filter(Boolean).join("\n\n"));
848
1583
  s.queuedNote = "";
849
1584
  }
850
1585
  s.queued = false;
@@ -995,11 +1730,22 @@ export class Sessions extends EventEmitter {
995
1730
  if (token != null && token !== d.lease.token) throw stale();
996
1731
  }
997
1732
 
1733
+ /**
1734
+ * The delivered batch, for its lease holder only (`GET /cli/batch` with a claim): holder, batch id and token are all required and checked
1735
+ * before anything of the batch is handed out, so `reply` and `apply` read no anchors, comments or file text for a delivery that is not theirs.
1736
+ */
1737
+ deliveredBatch(key, claim) {
1738
+ const s = this.get(key);
1739
+ if (!s.delivered) throw new HttpError(409, "no batch is delivered for this file, so the ticket is void", { code: "no_batch" });
1740
+ this.#assertLease(s, { ...claim, strict: true });
1741
+ return s.delivered.batch;
1742
+ }
1743
+
998
1744
  /**
999
1745
  * `apply` through a live session: the single authority for the write. Accepted only while a batch is delivered (state working or
1000
1746
  * queued: both hold a live lease) and only for the lease holder; compare-and-swap on the disk hash; renews the lease.
1001
1747
  */
1002
- agentWrite(key, { baseSha, bytes, holder, token, batchId, label = "before agent write", route = "commit", ops = null, partial = false }) {
1748
+ agentWrite(key, { baseSha, bytes, holder, token, batchId, label = "before agent write", route = "commit", ops = null, partial = false, htmlChecked = null }) {
1003
1749
  const s = this.get(key);
1004
1750
  if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
1005
1751
  const state = this.state(s);
@@ -1010,11 +1756,20 @@ export class Sessions extends EventEmitter {
1010
1756
  // `commit` does not know the hash its candidate was edited from: the base is the last hash this server wrote or served for the delivery
1011
1757
  if (baseSha == null) baseSha = s.delivered.knownSha ?? s.delivered.preSha;
1012
1758
  if (!bytes?.length) throw new HttpError(400, "no bytes");
1013
- if (bytes.length > MAX_BYTES) throw new HttpError(413, "document too large");
1014
- try {
1015
- validatePackage(bytes, s.format);
1016
- } catch (err) {
1017
- throw new HttpError(422, `refusing to write: ${err.message}`, { code: "invalid_package" });
1759
+ if (bytes.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
1760
+ const newSha = sha256(bytes);
1761
+ if (s.format === "html") {
1762
+ // checked in a worker (agentWriteChecked) for exactly these bytes; the disk hash it was checked against is compared below
1763
+ if (htmlChecked?.to !== newSha) throw new HttpError(500, "an HTML file is written only after its write check", { code: "unchecked_write" });
1764
+ } else {
1765
+ try {
1766
+ validatePackage(bytes, s.format);
1767
+ } catch (err) {
1768
+ throw new HttpError(422, `refusing to write: ${err.message}`, { code: "invalid_package" });
1769
+ }
1770
+ // read whole before anything is written, as a Send's bytes are: bytes past a reader's cap are refused with the reason (CapError, 422
1771
+ // too_large), never written to leave the next outline, review and done without a reading. The reading is kept for them (unitsFor).
1772
+ readWhole(s.format, bytes, newSha);
1018
1773
  }
1019
1774
  let disk;
1020
1775
  try {
@@ -1025,9 +1780,10 @@ export class Sessions extends EventEmitter {
1025
1780
  }
1026
1781
  const diskSha = sha256(disk);
1027
1782
  if (diskSha !== baseSha) throw this.#outsideChange(s, { side: "agent", bytes, diskSha, baseSha, message: "the file changed on disk since apply read it" });
1783
+ // R12: the compare-and-swap again after the check's await: the version it checked against must still be the one on disk
1784
+ if (s.format === "html" && htmlChecked.from !== diskSha) throw this.#outsideChange(s, { side: "agent", bytes, diskSha, baseSha: htmlChecked.from, message: "the file changed on disk while the write was being checked" });
1028
1785
  this.setStage(key, "writing"); // only once the checks passed: a refused write does not claim to be writing
1029
- const info = this.#replace(s, bytes, { expectSha: diskSha, label, side: "agent" });
1030
- const newSha = sha256(bytes);
1786
+ const info = this.#replace(s, bytes, { expectSha: diskSha, label, side: "agent", htmlChecked });
1031
1787
  // what the merge may rely on later: the ops `apply` says it ran (checked against the bytes when a merge needs them), or that it was a commit
1032
1788
  const checked = route === "apply" && s.format === "xlsx" && Array.isArray(ops) && validateOps({ schema: "onionskin.ops/1", ops }).ok; // only the XLSX merge reads them
1033
1789
  this.#logWrite(s, checked ? { from: diskSha, to: newSha, route, ops, partial: partial === true, anchors: s.delivered.batch.untrusted_document_evidence?.anchors || [] } : { from: diskSha, to: newSha, route: route === "apply" ? "apply" : "commit", ops: null });
@@ -1039,6 +1795,41 @@ export class Sessions extends EventEmitter {
1039
1795
  return { ok: true, sha256: newSha, lease_expires_at: s.delivered.lease.expiresAt };
1040
1796
  }
1041
1797
 
1798
+ /**
1799
+ * The agent's write (`/cli/agent-write`, `/cli/commit`) for every format. An HTML file's bytes first go through the write check
1800
+ * (src/html-write.js checkedHtmlWrite: decoding, limits, a parse, the charset rule, the inventory of active content) in a worker; the
1801
+ * bytes it returns (the charset declaration may be added) are then written by agentWrite, which checks the lease, the state and the
1802
+ * disk hash again after that await (R12). Other formats: agentWrite as it is.
1803
+ */
1804
+ async agentWriteChecked(key, args, check = htmlWriteOffThread) {
1805
+ const s = this.get(key);
1806
+ if (s.format !== "html") return this.agentWrite(key, args);
1807
+ // the cheap refusals first, so a write that cannot land costs no worker; agentWrite makes every one of them again after the await
1808
+ if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
1809
+ if (!s.delivered) throw new HttpError(409, `no batch is delivered to an agent (state: ${this.state(s)}); run poll and wait for the user's Send`, { code: "not_working", state: this.state(s) });
1810
+ this.#assertLease(s, { holder: args.holder, token: args.token, batchId: args.batchId, strict: true });
1811
+ if (!args.bytes?.length) throw new HttpError(400, "no bytes");
1812
+ let disk;
1813
+ try {
1814
+ checkRegularOwned(s.file);
1815
+ disk = readCapped(s.file);
1816
+ } catch (err) {
1817
+ throw new HttpError(409, `cannot read the file: ${err.message}`, { code: "gone" });
1818
+ }
1819
+ // the bytes checked are the ones whose hash agentWrite compares with the base and the disk after this await (from === disk === base)
1820
+ const c = await check({ before: disk, after: args.bytes });
1821
+ if (!c.ok) {
1822
+ // a refused write is part of the run's result: the page lists it with the run (the R1 sentence for active content), nothing was written
1823
+ if (c.code !== "check_failed" && s.delivered && !s.closed) {
1824
+ const p = cleanProblem({ key: `write:${c.code}`, op: args.route === "apply" ? "apply" : "commit", status: "refused", code: c.code, reason: c.reason });
1825
+ s.delivered.applyProblems = [...(s.delivered.applyProblems || []).filter((x) => x.key !== p.key), p].slice(0, 40);
1826
+ this.#changed(s);
1827
+ }
1828
+ throw new HttpError(c.code === "check_failed" ? 503 : 422, c.reason, { code: c.code, ...(c.reasons ? { reasons: c.reasons } : {}) });
1829
+ }
1830
+ return this.agentWrite(key, { ...args, bytes: c.bytes, htmlChecked: { from: sha256(disk), to: sha256(c.bytes) } });
1831
+ }
1832
+
1042
1833
  /**
1043
1834
  * What `apply` could not do (refused by the engine, protected, anchor or expectation mismatch), told by the lease holder after each run
1044
1835
  * (also when nothing was written). Shown to the user with the activity entry at `done`.
@@ -1078,7 +1869,25 @@ export class Sessions extends EventEmitter {
1078
1869
  return { pre, post, format: s.format, postSha: sha256(post), batchId: s.delivered.batch.batch_id, deferSoffice: env("SOFFICE_SYNC", "") !== "1" };
1079
1870
  }
1080
1871
 
1081
- done(key, results, claim = {}, analysis = null) {
1872
+ /**
1873
+ * done() as the server calls it (src/server.js finishRun): the anchors of the comments queued during the run (moved onto the file the
1874
+ * agent leaves), a review's cards and the outline for a waiting agent are worked out first in the anchor worker, then done() commits
1875
+ * with them and reads none of it on this thread. Work that did not finish leaves those comments weaker, with a flag; never lost.
1876
+ */
1877
+ async doneAsync(key, results, claim = {}, analysis = null, opts = {}) {
1878
+ return this.#serial(key, async () => {
1879
+ const s = this.get(key);
1880
+ const run = s.delivered;
1881
+ const tasks = [];
1882
+ if (run && s.queued) tasks.push(...s.comments.filter((c) => c.status === "queued").map(reanchorTask));
1883
+ if (run?.batch.instructions?.review && s.review?.batchId === run.batch.batch_id && results && typeof results === "object") tasks.push(cardsTask(results, s.review, run));
1884
+ const prepared = run && (tasks.length || this.#wantsOutline(s)) ? await this.#offThread(s.format, { source: { path: s.file }, tasks, outline: this.#wantsOutline(s) }) : null;
1885
+ return this.done(key, results, claim, analysis, { ...opts, prepared });
1886
+ });
1887
+ }
1888
+
1889
+ /** `inline: false` (the server): never analyze on this thread; an analysis for other bytes (the file changed while it ran) is a stated failure. */
1890
+ done(key, results, claim = {}, analysis = null, { inline = true, prepared = null } = {}) {
1082
1891
  const s = this.get(key);
1083
1892
  if (!s.delivered) throw new HttpError(409, "no batch is in progress for this file (taken back or already finished)", { code: "no_batch" });
1084
1893
  this.#assertLease(s, claim, { staleOnHolder: true });
@@ -1092,10 +1901,10 @@ export class Sessions extends EventEmitter {
1092
1901
  throw new HttpError(409, `cannot read the file: ${err.message}`);
1093
1902
  }
1094
1903
  const postSha = sha256(post);
1095
- if (readOnlyKind(run.batch) === "voice") return this.#doneVoice(s, results, run, post, postSha);
1904
+ if (readOnlyKind(run.batch) === "voice") return this.#doneVoice(s, results, run, post, postSha, prepared);
1096
1905
  // a review that wrote nothing ends with cards. One whose file changed anyway (a direct save past review_only) is finished as a run below:
1097
1906
  // its write is flagged and can be reverted, and no card is kept
1098
- if (run.batch.instructions?.review && postSha === run.preSha) return this.#doneReview(s, results, run, post);
1907
+ if (run.batch.instructions?.review && postSha === run.preSha) return this.#doneReview(s, results, run, post, prepared);
1099
1908
  s.snapshots.add(post, "after agent");
1100
1909
 
1101
1910
  // ---- plan: everything that can throw happens here, on locals; the session is not touched until the commit below ------
@@ -1109,7 +1918,14 @@ export class Sessions extends EventEmitter {
1109
1918
  let found;
1110
1919
  if (!pre) found = { flags: [{ code: "no_pre_snapshot", severity: "warn" }], checks: { node: "skipped", deep: { status: "skipped", reason: "no pre-agent snapshot" }, notice: "Checks skipped: the pre-agent version is no longer stored." }, changes: null };
1111
1920
  else if (analysis && analysis.postSha === postSha && analysis.batchId === run.batch.batch_id) found = analysis;
1112
- else found = analyze(pre, post, s.format);
1921
+ else if (!inline) {
1922
+ const why = "the file changed while it was being checked";
1923
+ found = { flags: [{ code: "checks_failed", severity: "warn", detail: why }], checks: { node: "failed", deep: { status: "skipped", reason: why }, notice: `The package checks did not finish: ${why}. The file was not verified.` }, changes: null };
1924
+ } else if (s.format === "html") {
1925
+ // an HTML file is read in workers only: a caller without the worker's analysis gets it stated as not run, never as clean
1926
+ const why = "the checks of an HTML file run in a worker";
1927
+ found = { flags: [{ code: "checks_failed", severity: "warn", detail: why }], checks: { node: "failed", deep: { status: "skipped", reason: why }, notice: `The checks did not run: ${why}. The file was not verified.` }, changes: null };
1928
+ } else found = analyze(pre, post, s.format);
1113
1929
  const flags = [...found.flags];
1114
1930
  const { checks, changes } = found;
1115
1931
  // the file must be what this server last wrote (apply/commit) or delivered; anything else changed it behind the lease (a direct save by
@@ -1150,7 +1966,8 @@ export class Sessions extends EventEmitter {
1150
1966
  // style.json changed by something else while this batch was out (an agent with a shell, say): already put back by src/style.js
1151
1967
  this.#guardFlags(run, flags);
1152
1968
  if (results.voice_draft !== undefined) flags.push({ code: "voice_draft_not_allowed", severity: "warn" }); // only a voice batch takes a draft; nothing is stored
1153
- const queuedPlan = this.#planQueued(s, post, postSha, flags);
1969
+ const queuedPlan = this.#planQueued(s, post, postSha, flags, prepared);
1970
+ if (prepared?.ok && prepared.sha === postSha && prepared.outline) this.#outline = { key: s.key, sha: postSha, all: prepared.outline };
1154
1971
  const cleanFlags = flags.map(cleanFlag);
1155
1972
  const { list: ignored, more: ignoredMore } = ignoredInstructions(results.document_instructions_ignored);
1156
1973
  const entry = {
@@ -1276,27 +2093,21 @@ export class Sessions extends EventEmitter {
1276
2093
  }
1277
2094
 
1278
2095
  /** Comments queued while the agent worked become the next pending batch, anchored on the file the agent leaves. Pure planning: touches no session state; re-anchoring problems are pushed onto `flags`. */
1279
- #planQueued(s, post, postSha, flags) {
2096
+ // `prepared`: the anchor worker's answer for the same tasks (doneAsync); without it the anchors are moved on this thread.
2097
+ #planQueued(s, post, postSha, flags, prepared = null) {
1280
2098
  let plan = null;
1281
2099
  if (s.queued) {
1282
2100
  const queued = s.comments.filter((c) => c.status === "queued");
1283
2101
  if (queued.length || s.queuedNote) {
1284
- let units = null;
1285
- try {
1286
- units = readUnits(s.format, post);
1287
- } catch (err) {
1288
- flags.push({ code: "queued_reanchor_failed", severity: "warn", detail: err.message });
1289
- }
2102
+ const work = this.#results(s.format, post, postSha, queued.map(reanchorTask), prepared);
2103
+ const read = work.read;
2104
+ if (!read?.ok) flags.push({ code: "queued_reanchor_failed", severity: "warn", detail: read?.invalid || read?.cap || "the anchors could not be worked out in time" });
1290
2105
  const durables = new Map();
1291
- for (const c of queued) {
1292
- let d = null;
1293
- try {
1294
- if (units) d = reanchor(units, c);
1295
- } catch (err) {
1296
- flags.push({ code: "queued_reanchor_failed", severity: "warn", comment_id: c.id, detail: err.message });
1297
- }
1298
- durables.set(c.id, d);
1299
- }
2106
+ queued.forEach((c, i) => {
2107
+ const r = work.at(i);
2108
+ if (read?.ok && !usable(r)) flags.push({ code: "queued_reanchor_failed", severity: "warn", comment_id: c.id, detail: r?.error ?? "the anchor could not be worked out in time" });
2109
+ durables.set(c.id, read?.ok && usable(r) ? r : weakDurable(s.format, c.anchor));
2110
+ });
1300
2111
  const batch = buildBatch({ file: s.file, format: s.format, baseSha: postSha, note: s.queuedNote, comments: queued.map((c) => ({ ...c, durable: durables.get(c.id) })), edits: [], style: styleNote(), guide: voiceGuide() });
1301
2112
  plan = { queued, durables, batch };
1302
2113
  }
@@ -1314,6 +2125,7 @@ export class Sessions extends EventEmitter {
1314
2125
  const s = this.get(key);
1315
2126
  if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
1316
2127
  if (!REVIEW_KINDS.includes(kind)) throw new HttpError(400, `kind must be one of ${REVIEW_KINDS.join(", ")}`, { code: "bad_kind" });
2128
+ if (s.format === "html") throw new HttpError(409, "Review first is not available for HTML files in this version. Comment on the page and press Send.", { code: "not_for_html" }); // PLAN-html 2a: the page hides the button
1317
2129
  if (s.delivered || s.pending) throw new HttpError(409, s.delivered?.local ? "Wait for the change on this Mac to finish, then try again." : "Wait for the agent to finish, then try again.", { code: "busy" });
1318
2130
  let disk;
1319
2131
  try {
@@ -1330,7 +2142,7 @@ export class Sessions extends EventEmitter {
1330
2142
  }
1331
2143
 
1332
2144
  #reviewBatch(s, kind, bytes, baseSha) {
1333
- const { units, more, index } = unitsOf(s.format, bytes);
2145
+ const { units, more, index } = unitsOf(s.format, bytes, unitsFor(s.format, bytes, baseSha));
1334
2146
  const batch = buildBatch({ file: s.file, format: s.format, baseSha, note: "", comments: [], edits: [], review: { kind, ask: REVIEW_ASKS[kind] }, units, unitsMore: more });
1335
2147
  s.review = { batchId: batch.batch_id, kind, units: units.map((u) => ({ id: u.id, protected: u.protected })), index };
1336
2148
  s.decidedBatch = batch.batch_id; // never the local runner's: only an agent reads the file
@@ -1338,13 +2150,19 @@ export class Sessions extends EventEmitter {
1338
2150
  }
1339
2151
 
1340
2152
  /** `done` of a review that wrote nothing: the suggestions become cards (checked against the file), an activity entry of kind "review", no history update. */
1341
- #doneReview(s, results, run, post) {
2153
+ #doneReview(s, results, run, post, prepared = null) {
1342
2154
  const flags = [];
1343
2155
  const ix = s.review?.batchId === run.batch.batch_id ? s.review : null;
1344
- const { cards, dropped } = ix ? cardsFrom(results.suggestions, { bytes: post, format: s.format, units: ix.units, index: ix.index, reviewId: run.batch.batch_id }) : { cards: [], dropped: (Array.isArray(results.suggestions) ? results.suggestions : []).map((_, i) => ({ index: i, reason: "the review's outline is gone" })) };
2156
+ const gone = (why) => ({ cards: [], dropped: (Array.isArray(results.suggestions) ? results.suggestions : []).map((_, i) => ({ index: i, reason: why })) });
2157
+ let checked = gone("the review's outline is gone");
2158
+ if (ix) {
2159
+ const r = this.#results(s.format, post, run.preSha, [cardsTask(results, ix, run)], prepared).at(0);
2160
+ checked = usable(r) && r ? r : gone("the suggestions could not be checked against the file in time");
2161
+ }
2162
+ const { cards, dropped } = checked;
1345
2163
  for (const d of dropped) flags.push({ code: "bad_suggestion", severity: "warn", ...(d.index != null ? { index: d.index } : {}), reason: d.reason });
1346
2164
  if (results.voice_draft !== undefined) flags.push({ code: "voice_draft_not_allowed", severity: "warn" });
1347
- const queuedPlan = this.#planQueued(s, post, run.preSha, flags);
2165
+ const queuedPlan = this.#planQueued(s, post, run.preSha, flags, prepared);
1348
2166
  const cleanFlags = flags.map(cleanFlag);
1349
2167
  const { list: ignored, more: ignoredMore } = ignoredInstructions(results.document_instructions_ignored); // a review reads the whole file: its report counts most here
1350
2168
  const entry = {
@@ -1424,8 +2242,25 @@ export class Sessions extends EventEmitter {
1424
2242
  * and Revert, as for any local run. The text written is the stored suggestion, never text from the page. Every refusal happens before
1425
2243
  * anything is written. Other comments and the overall note stay for the next Send; typed edits ride with it too.
1426
2244
  */
1427
- useSuggestion(key, { suggestion: rec, commentId, comments, note, baseSha, bytes = null, edits = [] }) {
2245
+ /**
2246
+ * useSuggestion() as the server's route calls it (src/suggest.js): where the suggestion goes in the bytes about to be the file, and
2247
+ * the one reply that writes it, are worked out in the anchor worker first. Work that did not finish writes nothing: the comment stays a
2248
+ * draft and says it can go to the agent, as when the sentence is not found.
2249
+ */
2250
+ async useSuggestionAsync(key, args) {
1428
2251
  const s = this.get(key);
2252
+ this.#useChecks(s, args); // refused before it waits its turn
2253
+ return this.#serial(key, async () => {
2254
+ const { id } = this.#useChecks(s, args);
2255
+ const { c } = this.#usePlan(s, args, id); // nothing is stored before the commit, as a Send (sendAsync)
2256
+ const bytes = args.bytes?.length && args.bytes.length <= MAX_BYTES ? args.bytes : null;
2257
+ const prepared = await this.#offThread(s.format, { source: bytes ? { bytes } : { path: s.file }, tasks: [useTask(c.anchor, args.suggestion, id)] });
2258
+ return this.useSuggestion(key, { ...args, prepared });
2259
+ });
2260
+ }
2261
+
2262
+ /** The refusals of a quick fix's Use that need nothing read (useSuggestion runs them first). Returns {id}. */
2263
+ #useChecks(s, { suggestion: rec, commentId, comments }) {
1429
2264
  if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
1430
2265
  if (s.delivered || s.pending) throw new HttpError(409, "The agent started working on this file, so nothing was written. Ask the agent instead, or try again when it is done.", { code: "busy" });
1431
2266
  const words = rec && Object.hasOwn(STAMP_WORDS, rec.task) && rec.task !== "translate" ? STAMP_WORDS[rec.task] : null;
@@ -1435,22 +2270,37 @@ export class Sessions extends EventEmitter {
1435
2270
  const old = s.comments.find((c) => c.id === id);
1436
2271
  const sameAnchor = (a) => a && typeof a === "object" && a.quote === rec.quote && JSON.stringify(a.location ?? null) === JSON.stringify(rec.location ?? null);
1437
2272
  if (!mine || userText(mine.text) !== words || !sameAnchor(mine.anchor) || (old && old.status !== "draft")) throw new HttpError(409, "This comment is not the suggestion's stamp.", { code: "not_this_comment" });
2273
+ return { id };
2274
+ }
1438
2275
 
1439
- this.setDraft(key, { comments, note: note ?? s.note });
2276
+ /** What a Use stores before it commits (#sentPlan with its stamp), the stamp as planned, and every limit checked: nothing changed. */
2277
+ #usePlan(s, { comments, note, edits = [], rev, tab, seen, noteSeen, bytes = null }, id) {
2278
+ if (bytes?.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" }); // before anything is stored, as a Send
2279
+ const saved = this.#sentPlan(s, { comments, note, rev, tab, seen, noteSeen }, id);
2280
+ const c = saved.comments.find((x) => x.id === id);
2281
+ if (c?.status !== "draft") throw new HttpError(409, "This comment is not the suggestion's stamp.", { code: "not_this_comment" });
2282
+ if (!this.#fits(s, { comments: saved.comments, edits: [...s.edits, ...this.#editsToTake(s, edits)] })) throw tooLarge("This request");
2283
+ return { saved, c };
2284
+ }
2285
+
2286
+ useSuggestion(key, { suggestion: rec, commentId, comments, note, baseSha, bytes = null, edits = [], prepared = null, rev, tab, seen, noteSeen }) {
2287
+ const s = this.get(key);
2288
+ const { id } = this.#useChecks(s, { suggestion: rec, commentId, comments });
2289
+ // the page's list and note are saved first (only the stamp goes out), after every limit is checked on them
2290
+ this.#commitDraft(s, this.#usePlan(s, { comments, note, edits, rev, tab, seen, noteSeen, bytes }, id).saved, rev, tab, true);
1440
2291
  const c = s.comments.find((x) => x.id === id);
1441
- const anchor = { location: c.anchor.location, quote: c.anchor.quote, prefix: c.anchor.prefix, suffix: c.anchor.suffix };
1442
2292
  let plan = null;
1443
- // the place and the reply are worked out on the very bytes that are about to be the file; any refusal here writes nothing
1444
- const check = (b) => {
1445
- const units = readUnits(s.format, b);
1446
- const durable = buildDurable(units, anchor);
1447
- const place = placeOf(units, durable, rec.quote);
2293
+ // the place and the reply are worked out on the very bytes that are about to be the file (by the anchor worker, or here); any
2294
+ // refusal here writes nothing, and work that did not finish is "not here"
2295
+ const check = (b, sha) => {
2296
+ const r = this.#results(s.format, b, sha, [useTask(c.anchor, rec, id)], prepared).at(0);
2297
+ const { durable, place, reply } = usable(r) ? r : { durable: null, place: { ok: false, code: "not_here" }, reply: null };
1448
2298
  if (!place.ok) throw new HttpError(409, place.code === "protected" ? "This part of the file is protected, so it cannot be changed here." : LOCAL_HINT, { code: place.code });
1449
- const reply = buildUseReply(rec, id, place.text);
1450
- const batch = buildBatch({ file: s.file, format: s.format, baseSha: sha256(b), note: "", comments: [{ ...c, durable }], edits: [] });
2299
+ const batch = buildBatch({ file: s.file, format: s.format, baseSha: sha, note: "", comments: [{ ...c, durable }], edits: [] });
1451
2300
  if (!reply || batch.instructions.comments[0]?.reply_with !== "find_replace_or_text") throw new HttpError(409, LOCAL_HINT, { code: "not_here" });
1452
2301
  plan = { durable, reply, batch };
1453
2302
  };
2303
+ if (!this.#fits(s, { comments: s.comments, edits: [...s.edits, ...this.#editsToTake(s, edits)] })) throw tooLarge("This request"); // before anything is written
1454
2304
  let taken;
1455
2305
  try {
1456
2306
  taken = this.#takePage(s, { baseSha, bytes }, check);
@@ -1459,15 +2309,16 @@ export class Sessions extends EventEmitter {
1459
2309
  if (["not_here", "not_eligible", "protected"].includes(err?.extra?.code)) {
1460
2310
  c.localHint = LOCAL_HINT;
1461
2311
  this.#changed(s);
1462
- }
2312
+ } else this.emit("change", s, "draft"); // the drafts it stored
1463
2313
  throw err;
1464
2314
  }
1465
- for (const e of Array.isArray(edits) ? edits.slice(0, 2000) : []) s.edits.push({ kind: e.kind, before: e.before, after: e.after, label: e.label, location: e.location });
2315
+ s.edits.push(...this.#editsToTake(s, edits));
1466
2316
  c.durable = plan.durable;
1467
2317
  c.status = "sent";
1468
2318
  c.batchId = plan.batch.batch_id;
1469
2319
  delete c.localHint;
1470
2320
  s.lastWrite = taken.writeInfo;
2321
+ this.#sentRev(s, { rev, tab });
1471
2322
  try {
1472
2323
  this.startLocal(key, plan.batch, { reply: plan.reply, summary: USE_SUMMARY, model: true }); // saves, tells the page, starts the runner
1473
2324
  } catch (err) {
@@ -1526,13 +2377,13 @@ export class Sessions extends EventEmitter {
1526
2377
  * "voice" with a fixed summary, no history update and no analysis. A file that changed anyway (a direct save past voice_only) is
1527
2378
  * recorded as a withdrawn run would be: flagged, kept Revert-able.
1528
2379
  */
1529
- #doneVoice(s, results, run, post, postSha) {
2380
+ #doneVoice(s, results, run, post, postSha, prepared = null) {
1530
2381
  const flags = [];
1531
2382
  for (const r of (Array.isArray(results.results) ? results.results : []).slice(0, 1000)) flags.push({ code: "unknown_result", severity: "warn", comment_id: String(r?.comment_id ?? "").slice(0, 64) });
1532
2383
  const check = cleanVoiceDraft(results.voice_draft); // size first, then hidden characters, links and markup, the layout, the caps
1533
2384
  if (check.code) flags.push({ code: "bad_voice_draft", severity: "warn", reason: check.code });
1534
2385
  this.#guardFlags(run, flags);
1535
- const queuedPlan = this.#planQueued(s, post, postSha, flags);
2386
+ const queuedPlan = this.#planQueued(s, post, postSha, flags, prepared);
1536
2387
  const { list: ignored, more: ignoredMore } = ignoredInstructions(results.document_instructions_ignored);
1537
2388
  // ---- commit: the draft file first (a failed write keeps nothing and says so), then the session
1538
2389
  let kept = false;
@@ -1585,7 +2436,7 @@ export class Sessions extends EventEmitter {
1585
2436
  if (!batch) throw new HttpError(409, "nothing to take back");
1586
2437
  // the note travels back with the comments: the withdrawn batch's note, then a queued one, then whatever was typed since
1587
2438
  const notes = [batch.instructions?.overall_note, s.queuedNote, s.note].filter(Boolean);
1588
- s.note = [...new Set(notes)].join("\n\n");
2439
+ this.#giveNote(s, [...new Set(notes)].join("\n\n"));
1589
2440
  s.queuedNote = "";
1590
2441
  let event = "state";
1591
2442
  if (s.delivered) {
@@ -1636,7 +2487,27 @@ export class Sessions extends EventEmitter {
1636
2487
  return true;
1637
2488
  }
1638
2489
 
1639
- revert(key, activityId) {
2490
+ /** revert() as the server's route calls it: a waiting batch's anchors are moved onto the restored version in the anchor worker first. */
2491
+ async revertAsync(key, activityId) {
2492
+ return this.#serial(key, async () => {
2493
+ const s = this.get(key);
2494
+ const entry = s.activity.find((a) => a.id === activityId && a.kind === "agent");
2495
+ let prepared = null;
2496
+ if (entry && !entry.reverted && !s.delivered && s.pending && !readOnlyKind(s.pending) && !s.pending.instructions?.review && s.snapshots.has(entry.preSha)) {
2497
+ const tasks = this.#revertComments(s, entry).map(reanchorTask);
2498
+ if (tasks.length) prepared = await this.#offThread(s.format, { source: { path: s.snapshots.blob(entry.preSha), sha: entry.preSha }, tasks });
2499
+ }
2500
+ return this.revert(key, activityId, { prepared });
2501
+ });
2502
+ }
2503
+
2504
+ /** The comments of the waiting batch, as revert() moves them: the run's comments become drafts first, so only the others are left. */
2505
+ #revertComments(s, entry) {
2506
+ const ids = new Set(s.pending.instructions.comments.map((c) => c.id));
2507
+ return s.comments.filter((c) => ids.has(c.id) && !(c.batchId === entry.batchId && ["done", "declined", "asked"].includes(c.status)));
2508
+ }
2509
+
2510
+ revert(key, activityId, { prepared = null } = {}) {
1640
2511
  const s = this.get(key);
1641
2512
  if (s.delivered) throw new HttpError(423, s.delivered.local ? "a change is being made on this Mac" : "the agent is working", { code: "locked" });
1642
2513
  const entry = s.activity.find((a) => a.id === activityId && a.kind === "agent");
@@ -1648,7 +2519,15 @@ export class Sessions extends EventEmitter {
1648
2519
  if (sha256(disk) !== entry.postSha) throw new HttpError(409, mac ? "the file changed after the change was made; revert refused" : "the file changed after the agent finished; revert refused", { code: "changed_since" });
1649
2520
  if (!s.snapshots.has(entry.preSha)) throw new HttpError(410, mac ? "the version before this change is no longer stored" : "the pre-agent version is no longer stored");
1650
2521
  const pre = s.snapshots.read(entry.preSha);
1651
- this.#replace(s, pre, { expectSha: entry.postSha, label: "before revert" }); // a revert is a new version
2522
+ // an HTML file: a revert restores the version before the run, by identity (the stored bytes must hash to the run's preSha), never
2523
+ // through the write check: that version was the file before (opened by the person or written after its own check), and checking it
2524
+ // against the run's result would refuse undoing a run that removed active content (the revert would "add" it back)
2525
+ let htmlChecked = null;
2526
+ if (s.format === "html") {
2527
+ if (sha256(pre) !== entry.preSha) throw new HttpError(410, mac ? "the version before this change is no longer stored" : "the pre-agent version is no longer stored");
2528
+ htmlChecked = { from: entry.postSha, to: entry.preSha };
2529
+ }
2530
+ this.#replace(s, pre, { expectSha: entry.postSha, label: "before revert", htmlChecked }); // a revert is a new version
1652
2531
  s.snapshots.add(pre, "reverted");
1653
2532
  s.baseSha = entry.preSha;
1654
2533
  addUpdate(s, { kind: "revert", before: entry.postSha, after: entry.preSha, local: mac }, this.now());
@@ -1666,7 +2545,7 @@ export class Sessions extends EventEmitter {
1666
2545
  }
1667
2546
  for (const g of s.suggestions || []) if (g.status === "accepted" && g.acceptedBatch === entry.batchId) g.status = "open"; // the text is back: it can be accepted again
1668
2547
  if (entry.by === "local" && entry.localNote && !s.note) {
1669
- s.note = entry.localNote; // the request comes back to the box, and the same words go to the agent next time
2548
+ this.#giveNote(s, entry.localNote); // the request comes back to the box, and the same words go to the agent next time
1670
2549
  s.skipLocalNote = entry.localNote;
1671
2550
  }
1672
2551
  // a batch built against the reverted version is rebuilt against the restored bytes (a review: its outline is read again; a voice
@@ -1676,19 +2555,9 @@ export class Sessions extends EventEmitter {
1676
2555
  else if (s.pending) {
1677
2556
  const ids = new Set(s.pending.instructions.comments.map((c) => c.id));
1678
2557
  const cs = s.comments.filter((c) => ids.has(c.id));
1679
- let units = null;
1680
- try {
1681
- units = readUnits(s.format, pre);
1682
- } catch {
1683
- units = null;
1684
- }
1685
- for (const c of cs) {
1686
- try {
1687
- if (units) c.durable = reanchor(units, c);
1688
- } catch {
1689
- c.durable = null;
1690
- }
1691
- }
2558
+ // moved onto the restored version by the anchor worker (revertAsync) or here; one whose work did not finish goes out weaker
2559
+ const work = this.#results(s.format, pre, entry.preSha, cs.map(reanchorTask), prepared);
2560
+ if (work.read?.ok) cs.forEach((c, i) => (c.durable = usable(work.at(i)) ? work.at(i) : work.at(i)?.error ? null : weakDurable(s.format, c.anchor)));
1692
2561
  s.pending = buildBatch({ file: s.file, format: s.format, baseSha: entry.preSha, note: s.pending.instructions.overall_note, comments: cs, edits: [], style: styleNote(), guide: voiceGuide() });
1693
2562
  for (const c of cs) c.batchId = s.pending.batch_id;
1694
2563
  }
@@ -1700,6 +2569,7 @@ export class Sessions extends EventEmitter {
1700
2569
  close(key) {
1701
2570
  const s = this.get(key);
1702
2571
  s.closed = true;
2572
+ this.#views.delete(key); // the render of an HTML file (htmlView)
1703
2573
  this.editors.delete(key); // the closed tab may stay on screen with its event stream open: it must not hold the file against the tab that reopens it
1704
2574
  for (const p of s.pollers.splice(0)) p.resolve({ status: "closed" });
1705
2575
  fs.rmSync(path.join(stateDir(), "agents", s.key, "outline"), { recursive: true, force: true }); // the outline files of Onionskin's own agent
@@ -1725,9 +2595,41 @@ export class Sessions extends EventEmitter {
1725
2595
  this.#save(s);
1726
2596
  } catch {} // no space for the snapshot: nothing is recorded, the file still loads
1727
2597
  }
2598
+ this.watch?.taken(s); // src/watch.js: this version is Onionskin's own from now on
1728
2599
  return out;
1729
2600
  }
1730
2601
 
2602
+ // ---- the view of an HTML file (GET /api/html-view) ------------------------------------------------------------------------------
2603
+ // key -> {sha, full, plain}: the render of one version (promises, so two requests for it share one worker run); one version per file,
2604
+ // dropped when the file changes or the session closes. Ceiling: one entry per open HTML file, each up to ~43 MB (the render copy at the
2605
+ // limits): a person with many large pages open holds them all; an LRU over files would be the upgrade.
2606
+ #views = new Map();
2607
+ /**
2608
+ * The page's view of an HTML file (PLAN-html 3.2, R12): the disk bytes are taken as /api/file takes them (loadForPage: an outside change
2609
+ * is recorded, the watcher told), then the render copy of exactly those bytes is built in the render worker (src/offload.js
2610
+ * renderOffThread), or the plain copy with `plain`. Resolves to {sha, plain, reason, stats, texts, editable, ident, bytes}; throws HttpError
2611
+ * 422 with the encoding's sentence and code (and `copyable`) or a limit's, 503 when the worker was busy or did not finish.
2612
+ */
2613
+ async htmlView(key, { plain = false } = {}) {
2614
+ const s = this.get(key);
2615
+ if (s.format !== "html") throw new HttpError(400, "this file is not an HTML file", { code: "not_html" });
2616
+ const { bytes, diskSha } = this.loadForPage(key);
2617
+ let c = this.#views.get(key);
2618
+ if (!c || c.sha !== diskSha) this.#views.set(key, (c = { sha: diskSha, full: null, plain: null }));
2619
+ const k = plain ? "plain" : "full";
2620
+ if (!c[k]) {
2621
+ const p = renderOffThread({ bytes, fileDir: path.dirname(s.file), plain }).then((r) => {
2622
+ if (r.ok) return { sha: diskSha, plain: r.plain, reason: r.reason, stats: r.stats, texts: r.texts, editable: r.editable, ident: r.ident, bytes: r.bytes };
2623
+ if (c[k] === p) c[k] = null; // a failure is not kept: the next request tries again
2624
+ if (r.failed) throw new HttpError(503, `Onionskin could not prepare this page: ${r.why}. Try again in a moment.`, { code: r.failed === "busy" ? "busy" : "render_failed" });
2625
+ if (r.code === "not_utf8" || ["utf16", "invalid_utf8", "roundtrip"].includes(r.code)) throw encodingRefusal(s.file, { error: r.error, code: r.code, copyable: r.copyable });
2626
+ throw new HttpError(422, `Onionskin cannot read this file: ${r.error}.`, { code: "too_large" });
2627
+ });
2628
+ c[k] = p;
2629
+ }
2630
+ return c[k];
2631
+ }
2632
+
1731
2633
  /** The history list and one update (browser routes, read-only; see src/history.js). */
1732
2634
  history(key) {
1733
2635
  return historyList(this.get(key));
@@ -1842,25 +2744,58 @@ export class Sessions extends EventEmitter {
1842
2744
  */
1843
2745
  resolveAll(key) {
1844
2746
  const s = this.get(key);
2747
+ // an HTML file is read in workers only: on this thread every anchor is "moved or removed" (the server's route resolves in the worker)
2748
+ if (s.format === "html") return { baseSha: sha256(readCapped(s.file)), format: s.format, anchors: Object.fromEntries(this.#resolveTask(s).comments.map((c) => [c.id, { method: "none" }])), view: null };
1845
2749
  const bytes = readCapped(s.file);
1846
- const out = { baseSha: sha256(bytes), format: s.format, anchors: {}, view: null };
1847
- const units = readUnits(s.format, bytes);
1848
- for (const c of s.comments) {
1849
- if (!c.durable) continue;
1850
- let r = resolveDurableAny(units, c.durable);
1851
- if (r.method === "none" && units.format === "pptx" && c.durable.pinFallback) {
1852
- const f = resolveDurableAny(units, c.durable.pinFallback);
1853
- if (f.method !== "none") r = { ...f, tieLost: true };
2750
+ const baseSha = sha256(bytes);
2751
+ const r = anchorWork({ format: s.format, bytes, sha: baseSha, tasks: [this.#resolveTask(s)], budget: SCAN_BUDGET });
2752
+ if (!r.read.ok) throw r.read.cap ? new CapError(r.read.cap) : new Error(r.read.invalid);
2753
+ if (r.results[0]?.error) throw new Error(r.results[0].error);
2754
+ return { baseSha, format: s.format, ...r.results[0] };
2755
+ }
2756
+
2757
+ /**
2758
+ * resolveAll() as the server's route calls it: resolved in the anchor worker against the file on disk. Work that did not finish
2759
+ * answers as a spent budget does: every anchor "moved or removed" (the page keeps the comments in its list), the view unknown.
2760
+ */
2761
+ async resolveAllAsync(key) {
2762
+ const s = this.get(key);
2763
+ const task = this.#resolveTask(s);
2764
+ if (s.format === "html") await this.#htmlDrafts(s, task);
2765
+ const r = await anchorsOffThread({ format: s.format, source: { path: s.file }, tasks: [task] });
2766
+ if (r.ok && r.r.read.cap) throw new CapError(r.r.read.cap);
2767
+ if (r.ok && r.r.read.ok && usable(r.r.results[0])) return { baseSha: r.sha, format: s.format, ...r.r.results[0] };
2768
+ return { baseSha: r.ok ? r.sha : sha256(readCapped(s.file)), format: s.format, anchors: Object.fromEntries(task.comments.map((c) => [c.id, { method: "none" }])), view: null };
2769
+ }
2770
+
2771
+ /**
2772
+ * HTML: a draft (no durable yet) whose page anchor was made on another version than the file on disk (an outside change, a revert, a
2773
+ * save) would be drawn at a node id of that other version. It is made durable on its own version first (a stored snapshot) and resolved
2774
+ * with the others; one whose version is no longer stored resolves by its quote. Drafts made on the file's version need nothing: the page
2775
+ * draws them at their own place.
2776
+ */
2777
+ async #htmlDrafts(s, task) {
2778
+ let cur;
2779
+ try {
2780
+ cur = sha256(readCapped(s.file));
2781
+ } catch {
2782
+ return;
2783
+ }
2784
+ const other = s.comments.filter((c) => !c.durable && c.anchor?.location?.version && c.anchor.location.version !== cur);
2785
+ const byVersion = new Map();
2786
+ for (const c of other) (byVersion.get(c.anchor.location.version) || byVersion.set(c.anchor.location.version, []).get(c.anchor.location.version)).push(c);
2787
+ for (const [v, list] of byVersion) {
2788
+ const r = s.snapshots.has(v) ? await this.#offThread("html", { source: { path: s.snapshots.blob(v), sha: v }, tasks: list.map(buildTask) }) : null;
2789
+ for (const c of list) {
2790
+ const d = r?.ok && r.read?.ok ? r.map.get(taskKey(buildTask(c))) : undefined;
2791
+ const durable = usable(d) && d ? d : weakDurable("html", c.anchor);
2792
+ if (durable) task.comments.push({ id: c.id, durable });
1854
2793
  }
1855
- if (units.format === "docx" && r.method !== "none") r.heading = headingBefore(units.paragraphs, r.ordinal);
1856
- out.anchors[c.id] = r;
1857
- }
1858
- if (s.format === "pptx") out.slides = units.deck.slides.map((x) => ({ index: x.index, id: x.id }));
1859
- if (s.format === "docx") {
1860
- if (s.view?.durable) out.view = { ...resolveDurable(units.paragraphs, s.view.durable), offsetPx: s.view.offsetPx, zoom: s.view.zoom, fraction: s.view.fraction };
1861
- } else out.view = resolveView(units, s.view);
1862
- out.sheets = s.format === "xlsx" ? units.wb.sheets.map((x) => ({ index: x.index, name: x.name, sheetId: x.sheetId })) : undefined;
1863
- return out;
2794
+ }
2795
+ }
2796
+
2797
+ #resolveTask(s) {
2798
+ return { op: "resolveAll", comments: s.comments.filter((c) => c.durable).map((c) => ({ id: c.id, durable: c.durable })), view: s.view };
1864
2799
  }
1865
2800
  }
1866
2801
 
@@ -1901,7 +2836,7 @@ function exportInWorker(job) {
1901
2836
  };
1902
2837
  const timer = setTimeout(() => finish({ timeout: true }), EXPORT_TIMEOUT_MS());
1903
2838
  try {
1904
- worker = new Worker(new URL("./redline-worker.js", import.meta.url), { workerData: job });
2839
+ worker = startWorker("redline-worker.js", { workerData: job });
1905
2840
  } catch (err) {
1906
2841
  return finish({ check_failed: String(err?.message || err) });
1907
2842
  }