onionskin-cli 0.0.3 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CHANGELOG.md +198 -0
  2. package/INSTALL.md +5 -1
  3. package/NOTICE +34 -0
  4. package/README.md +102 -34
  5. package/docs/contracts/batch.schema.json +27 -4
  6. package/docs/contracts/ops.schema.json +117 -5
  7. package/package.json +4 -2
  8. package/scripts/checker/check.py +24 -5
  9. package/scripts/checker/requirements.txt +482 -5
  10. package/scripts/requirements.txt +482 -5
  11. package/scripts/rewriter/apple/main.swift +1 -1
  12. package/scripts/rewriter/apple/onionskin-fm +0 -0
  13. package/scripts/rewriter/apple/onionskin-fm.json +1 -1
  14. package/skills/onionskin/SKILL.md +12 -9
  15. package/skills/onionskin/reference/batch.md +5 -3
  16. package/skills/onionskin/reference/formats.md +5 -1
  17. package/skills/onionskin/reference/ops.md +48 -4
  18. package/skills/onionskin/reference/troubleshooting.md +31 -0
  19. package/src/agent-detect.js +28 -2
  20. package/src/agent-prompt.js +12 -1
  21. package/src/agent-runner.js +110 -20
  22. package/src/anchor-work.js +154 -0
  23. package/src/anchor-worker.js +38 -0
  24. package/src/anchors.js +61 -11
  25. package/src/apply.js +119 -47
  26. package/src/batch.js +32 -8
  27. package/src/check-worker.js +8 -5
  28. package/src/checker.js +238 -33
  29. package/src/cli.js +248 -42
  30. package/src/constants.js +34 -1
  31. package/src/diff.js +3 -1
  32. package/src/doctor.js +12 -3
  33. package/src/docx.js +464 -87
  34. package/src/edit-hash.js +19 -0
  35. package/src/fsutil.js +14 -6
  36. package/src/history-worker.js +8 -8
  37. package/src/history.js +144 -31
  38. package/src/html-active.js +384 -0
  39. package/src/html-envelope.js +51 -0
  40. package/src/html-parser.js +8 -0
  41. package/src/html-render-worker.js +25 -0
  42. package/src/html-render.js +939 -0
  43. package/src/html-utf8.js +65 -0
  44. package/src/html-write-worker.js +14 -0
  45. package/src/html-write.js +380 -0
  46. package/src/html.js +1259 -0
  47. package/src/local.js +68 -4
  48. package/src/merge-package.js +1166 -0
  49. package/src/merge-pptx.js +98 -38
  50. package/src/merge-proof.js +1019 -0
  51. package/src/merge-worker.js +60 -0
  52. package/src/merge-xlsx.js +84 -29
  53. package/src/merge.js +286 -83
  54. package/src/nfc.js +46 -0
  55. package/src/notes.js +122 -0
  56. package/src/offload.js +419 -1
  57. package/src/open-worker.js +27 -0
  58. package/src/ops/common.js +2 -1
  59. package/src/ops/docx-table.js +5 -3
  60. package/src/ops/docx.js +58 -29
  61. package/src/ops/html.js +394 -0
  62. package/src/ops/pptx.js +82 -23
  63. package/src/ops/run.js +23 -3
  64. package/src/ops/runs.js +215 -13
  65. package/src/ops/schema.js +8 -1
  66. package/src/ops/xlsx.js +121 -10
  67. package/src/ops-worker.js +17 -0
  68. package/src/package.js +43 -4
  69. package/src/person.js +2 -1
  70. package/src/pptx.js +323 -23
  71. package/src/question.js +10 -3
  72. package/src/quick-plan.js +13 -6
  73. package/src/quick.js +1 -1
  74. package/src/redline-check.js +12 -2
  75. package/src/redline-worker.js +10 -14
  76. package/src/reply.js +46 -19
  77. package/src/retain.js +223 -0
  78. package/src/review.js +32 -18
  79. package/src/rewrite-check.js +2 -2
  80. package/src/rewriter.js +38 -8
  81. package/src/sanitize.js +16 -3
  82. package/src/server-entry.js +4 -3
  83. package/src/server.js +175 -62
  84. package/src/session.js +1151 -210
  85. package/src/setup.js +20 -2
  86. package/src/stop-gate.js +34 -0
  87. package/src/suggest-reply.js +42 -7
  88. package/src/suggest.js +14 -29
  89. package/src/text.js +161 -11
  90. package/src/update.js +517 -0
  91. package/src/voice-guide.js +15 -9
  92. package/src/voice-learn.js +15 -4
  93. package/src/voice-sample-worker.js +7 -6
  94. package/src/voice-store.js +11 -4
  95. package/src/voice-worker.js +4 -6
  96. package/src/voice.js +2 -1
  97. package/src/worker-host.js +23 -0
  98. package/src/xlsx.js +378 -97
  99. package/src/xml-tree.js +6 -3
  100. package/src/xml.js +82 -34
  101. package/src/zip-write.js +10 -5
  102. package/src/zip.js +73 -18
  103. package/vendor/parse5/LICENSE.txt +32 -0
  104. package/vendor/parse5/parse5.mjs +7822 -0
  105. package/vendor/parse5.lock.json +23 -0
  106. package/web/dist/assets/{index-DS-WST1E.js → index-CjsTf5gj.js} +58 -52
  107. package/web/dist/assets/index-DnbLtHhG.css +1 -0
  108. package/web/dist/index.html +2 -2
  109. package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/session.js CHANGED
@@ -2,9 +2,11 @@ import crypto from "node:crypto";
2
2
  import { EventEmitter } from "node:events";
3
3
  import fs from "node:fs";
4
4
  import path from "node:path";
5
- import { REFUSED_EXTENSIONS, FORMATS, MAX_DOC_BYTES, PRODUCT, env } from "./constants.js";
6
- import { buildDurable, readUnits, resolveDurableAny, resolveView, viewDurable } from "./anchors.js";
7
- import { durableFromEngine, headingBefore, resolveDurable } from "./docx.js";
5
+ import { EDITS_OVERFLOW, ONLY_FORMATS, REFUSED_EXTENSIONS, REFUSED_PAGES, MAX_DOC_BYTES, PRODUCT, env, formatOfExt } from "./constants.js";
6
+ import { readAll, unitsFor } from "./anchors.js";
7
+ import { anchorWork, weakDurable } from "./anchor-work.js";
8
+ import { scanBudget } from "./text.js";
9
+ import { CapError } from "./xml.js";
8
10
  import { checkRegularOwned, readDoc, replaceGuarded, sha256, Snapshots, WriteConflict, writeSiblingCopy } from "./fsutil.js";
9
11
  import { buildBatch, compactBatch, withOutline } from "./batch.js";
10
12
  import { styleChangedOutside, styleNote } from "./style.js";
@@ -13,7 +15,10 @@ import { cleanVoiceDraft } from "./voice-guide.js";
13
15
  import { voiceJob } from "./voice-learn.js";
14
16
  import { clearTickets, saveTicket, ticketOf } from "./lease.js";
15
17
  import { analyze } from "./analyze.js";
18
+ import { anchorTimeoutMs, anchorsOffThread, htmlWriteOffThread, mergeOffThread, readOffThread, renderOffThread, startWorker } from "./offload.js";
16
19
  import { deepNotice, validatePackage } from "./package.js";
20
+ import { HtmlEncodingError } from "./html.js";
21
+ import { ACTIVE_REFUSED } from "./html-active.js";
17
22
  import { execFile } from "node:child_process";
18
23
  import { reopenCommand } from "./person.js";
19
24
  import { sessionKey, stateDir, sub, writeStateFile } from "./paths.js";
@@ -21,16 +26,17 @@ import { cleanFlag, hasControlChars, untrusted, userText } from "./sanitize.js";
21
26
  import { addUpdate, historyDetail, historyList, visibleUpdates } from "./history.js";
22
27
  import { BAD_QUESTION_NOTE, QUESTIONS_PER_BATCH, cleanQuestion } from "./question.js";
23
28
  import { validateOps } from "./ops/schema.js";
24
- import { Worker } from "node:worker_threads";
25
29
  import { AUTHORS, wordDate } from "./redline.js";
26
30
  import { deepRedline } from "./checker.js";
27
31
  import { TRACKED, formattingText, savedText, unsupportedText } from "./tracked-words.js";
28
32
  import { LOCAL_HOLDER } from "./local.js";
29
33
  import { bgHolder, isAgentHolder, isBgHolder, termHolder, TERM_PREFIX } from "./agent-detect.js";
30
- import { OUTLINE_FILE_MAX, REVIEW_ASKS, REVIEW_KINDS, SUGGESTIONS_MAX, UNITS_MAX, cardsFrom, outlineOf, reviewSummary, unitsOf } from "./review.js";
34
+ import { OUTLINE_FILE_MAX, REVIEW_ASKS, REVIEW_KINDS, SUGGESTIONS_MAX, UNITS_MAX, outlineOf, reviewSummary, unitsOf } from "./review.js";
31
35
  import { VOICE_BATCH_MAX_CHARS } from "./voice-samples.js";
32
36
  import { STAMP_WORDS } from "./stamp-words.js";
33
- import { USE_SUMMARY, buildUseReply, placeOf } from "./suggest-reply.js";
37
+ import { USE_SUMMARY } from "./suggest-reply.js";
38
+ import { mergeNote } from "./notes.js";
39
+ import { COMMENT_MAX, COMMENT_TEXT_MAX, COMMENTS_MAX, EDITS_MAX, NOTE_MAX, TOO_MANY_COMMENTS, REQUEST_MAX, SESSION_MAX, capComment, cleanEdit, cleanPageAnchor, cleanView, cutText, jsonSize } from "./retain.js";
34
40
 
35
41
  /** The line under a comment whose quick fixes could not be written on this Mac (docs/contracts/routes.md). */
36
42
  export const LOCAL_HINT = "This could not be changed on this Mac, so nothing was written. Press Send to ask the agent.";
@@ -59,8 +65,33 @@ export class HttpError extends Error {
59
65
  }
60
66
  }
61
67
 
68
+ /** One comment more than a file keeps (src/retain.js COMMENTS_MAX): 413 too_many_comments, in words the page shows as they are. */
69
+ const tooMany = () => new HttpError(413, TOO_MANY_COMMENTS, { code: "too_many_comments" });
70
+ /** A note longer than Onionskin keeps (src/retain.js NOTE_MAX): 413 note_too_long. `joined`: a waiting note and this one together. */
71
+ const noteTooLong = (joined) =>
72
+ new HttpError(413, joined ? `The note waiting for the agent and this one together are longer than Onionskin takes (${NOTE_MAX.toLocaleString("en-US")} characters), so nothing was sent. Shorten this note, then Send again.` : `This note is longer than Onionskin takes (${NOTE_MAX.toLocaleString("en-US")} characters), so it was not saved. Shorten it, then try again.`, { code: "note_too_long" });
73
+ /** The page's note as stored: refused (never cut) past NOTE_MAX. */
74
+ function noteOf(v) {
75
+ const t = userText(v, NOTE_MAX + 1);
76
+ if ([...t].length > NOTE_MAX) throw noteTooLong(false);
77
+ return t;
78
+ }
79
+ /** Two notes, the earlier first, as one (a waiting batch's or the queued ones, then the new one): refused past NOTE_MAX. */
80
+ function joinNotes(a, b) {
81
+ const t = [a, b].filter(Boolean).join("\n\n");
82
+ if ([...t].length > NOTE_MAX) throw noteTooLong(true);
83
+ return t;
84
+ }
85
+
86
+ /** A request whose own data is more than Onionskin keeps (src/retain.js): 413 too_large_request, in words the page shows as they are. */
87
+ export const tooLarge = (what) => new HttpError(413, `${what} is larger than Onionskin takes (${Math.round(REQUEST_MAX / 1024 / 1024)} MB), so nothing was saved. Remove some comments or shorten the longest ones, then try again.`, { code: "too_large_request" });
88
+
62
89
  const LEASE_MS = () => Number(env("LEASE_MS", 15 * 60 * 1000));
90
+ const SENT_TABS = 16; // tabs of one file whose newest committed Send a session remembers (s.sent)
91
+ const DELETED_KEPT = 200; // ids and words of deleted drafts a session remembers for a handover (Sessions.handOver)
92
+ const HANDOVER_MS = 60_000; // how long a tab that lost editing may hand over its unsaved words (Sessions.handOver)
63
93
  const MAX_BYTES = MAX_DOC_BYTES;
94
+ const TOO_BIG = `Onionskin cannot read this file: it is larger than ${Math.round(MAX_BYTES / 1024 / 1024)} MB, more than Onionskin can read. Make it smaller (pictures and videos are often most of a file) or split it, and open that.`;
64
95
  /** "Show in folder" exists on macOS (`open -R`); elsewhere the page shows the path. ONIONSKIN_NO_OPEN=1 turns it off, ONIONSKIN_OPEN_STUB=<file> records instead of launching. */
65
96
  const canReveal = () => !!env("OPEN_STUB") || (process.platform === "darwin" && !env("NO_OPEN"));
66
97
  /** readDoc with the size refusal as an HTTP error */
@@ -68,31 +99,81 @@ function readCapped(file) {
68
99
  try {
69
100
  return readDoc(file);
70
101
  } catch (err) {
71
- if (err.code === "TOO_LARGE") throw new HttpError(413, `document too large: ${err.message}`, { code: "too_large" });
102
+ if (err.code === "TOO_LARGE") throw new HttpError(413, `Onionskin cannot read this file: ${err.message}.`, { code: "too_large" });
72
103
  throw err;
73
104
  }
74
105
  }
75
106
 
76
- const num = (v) => (Number.isFinite(Number(v)) ? Math.round(Number(v) * 1000) / 1000 : 0);
107
+ /**
108
+ * Every part the server reads later, of bytes about to become the file (a Send's, an agent's write), so they are refused before anything is
109
+ * written: past a reader's cap with the reason (CapError: 422 too_large), damaged where the readers look as an invalid package (422), never
110
+ * as an internal error. The reading is kept for what reads the same version next (unitsFor).
111
+ */
112
+ function readWhole(format, bytes, sha, refusal = "refusing to write") {
113
+ try {
114
+ const units = readAll(unitsFor(format, bytes, sha));
115
+ readInCaps(format, sha);
116
+ return units;
117
+ } catch (err) {
118
+ if (err instanceof CapError || err instanceof HttpError) throw err;
119
+ throw new HttpError(422, `${refusal}: ${untrusted(err?.message || err, 200)}`, { code: "invalid_package" });
120
+ }
121
+ }
122
+
123
+ // "<format>:<sha256>" of the versions read whole within the readers' caps, on this thread (readWhole) or by the open worker, so a version
124
+ // read again is not read whole again. Ceiling: VERIFIED_KEPT entries, the oldest dropped first (one dropped is read whole again when needed).
125
+ const VERIFIED_KEPT = 256;
126
+ const verified = new Set();
127
+ function readInCaps(format, sha) {
128
+ const id = `${format}:${sha}`;
129
+ verified.delete(id);
130
+ verified.add(id);
131
+ if (verified.size > VERIFIED_KEPT) verified.delete(verified.values().next().value);
132
+ }
133
+
134
+ /**
135
+ * Bytes about to be handed to the engine on this thread (a quick preview, a merge, a change made on this Mac), which reads without the
136
+ * readers' caps: they are read whole within the caps first (readWhole) unless this version already was, so a file that changed on disk
137
+ * past a cap is refused with the reason (CapError: 422 too_large) before the engine parses any of it. Returns `bytes`.
138
+ */
139
+ export function readChecked(format, bytes, { sha = sha256(bytes), refusal = "cannot read the file" } = {}) {
140
+ if (!verified.has(`${format}:${sha}`)) readWhole(format, bytes, sha, refusal);
141
+ return bytes;
142
+ }
143
+
144
+ // bounded before it is rounded, so the result is always finite (1e308 * 1000 is Infinity, stored as null); anything that is not a finite
145
+ // number is 0, as src/retain.js drops what is not of its type
146
+ const num = (v, max) => {
147
+ const n = Number(v);
148
+ return Number.isFinite(n) ? Math.round(Math.min(max, Math.max(0, n)) * 1000) / 1000 : 0;
149
+ };
150
+ const frac = (v) => num(v, 1);
151
+ const PX_MAX = 1_000_000; // a paragraph taller than this is not drawn by any page
77
152
  /** A pin: x/y as fractions of the element it sits on (`yPx` is kept for DOCX paragraphs); `on` says what that element is. */
78
153
  function cleanPin(p) {
79
- const out = { xFrac: num(p.xFrac), yPx: num(p.yPx) };
80
- if (p.yFrac != null) out.yFrac = num(p.yFrac);
154
+ const out = { xFrac: frac(p.xFrac), yPx: num(p.yPx, PX_MAX) };
155
+ if (p.yFrac != null) out.yFrac = frac(p.yFrac);
81
156
  if (["paragraph", "cell", "shape", "slide"].includes(p.on)) out.on = p.on;
82
157
  if (p.sxFrac != null) {
83
- out.sxFrac = num(p.sxFrac);
84
- out.syFrac = num(p.syFrac);
158
+ out.sxFrac = frac(p.sxFrac);
159
+ out.syFrac = frac(p.syFrac);
85
160
  }
86
161
  return out;
87
162
  }
88
163
 
89
164
  const APPLY_STATUS = new Set(["refused", "skipped_protected", "anchor_not_found", "expect_mismatch", "not_applied"]);
90
- // the three-way merge of each format (loaded on first use: they pull in the engine)
91
- const MERGERS = { docx: { module: "./merge.js", fn: "mergeDocx" }, xlsx: { module: "./merge-xlsx.js", fn: "mergeXlsx" }, pptx: { module: "./merge-pptx.js", fn: "mergePptx" } };
92
165
  const HELD_SHOWN = 5; // items the page's "not in the document" message lists (web/src/Banners.jsx)
93
166
  const WRITES_KEPT = 20; // writes per file the merge can trace (src/merge-xlsx.js agentChanges); an older base finds no record and is handed back
94
- /** What the page says the user did during the run: kinds and places only, used to hold edits back or to name what is not carried, never to write. */
95
- const editKinds = (edits) => (Array.isArray(edits) ? edits.slice(0, 2000) : []).map((e) => ({ kind: String(e?.kind ?? "").slice(0, 40), sheet: typeof e?.sheet === "string" ? e.sheet.slice(0, 100) : null, slide: Number.isInteger(e?.slide) ? e.slide : null }));
167
+ /**
168
+ * What the page says the user did during the run: kinds and places only, used to hold edits back or to name what is not carried, never to
169
+ * write. The first EDITS_KEPT; past them one more entry of kind EDITS_OVERFLOW says the list was cut, so a planner that needs the whole list
170
+ * refuses the merge (src/merge-xlsx.js) instead of reading the cut list as complete. The proof of the merge never reads it.
171
+ */
172
+ const EDITS_KEPT = 2000;
173
+ export const editKinds = (edits) => {
174
+ const list = (Array.isArray(edits) ? edits.slice(0, EDITS_KEPT) : []).map((e) => ({ kind: String(e?.kind ?? "").slice(0, 40), sheet: typeof e?.sheet === "string" ? e.sheet.slice(0, 100) : null, slide: Number.isInteger(e?.slide) ? e.slide : null }));
175
+ return Array.isArray(edits) && edits.length > EDITS_KEPT ? [...list, { kind: EDITS_OVERFLOW, sheet: null, slide: null }] : list;
176
+ };
96
177
  /** One op `apply` did not apply, as the page shows it: codes and short reasons only (never document text). */
97
178
  function cleanProblem(p, run = "") {
98
179
  return {
@@ -101,7 +182,8 @@ function cleanProblem(p, run = "") {
101
182
  op: untrusted(p?.op, 40),
102
183
  status: APPLY_STATUS.has(p?.status) ? p.status : "refused",
103
184
  code: p?.code ? untrusted(p.code, 60) : null,
104
- reason: untrusted(p?.reason, 200),
185
+ // the R1 refusal in the server's own words, never the reason text an agent's report carries
186
+ reason: p?.code === "active_content" ? ACTIVE_REFUSED : untrusted(p?.reason, 200),
105
187
  };
106
188
  }
107
189
 
@@ -109,17 +191,47 @@ function shellQuote(s) {
109
191
  return `'${String(s).replace(/'/g, `'\\''`)}'`;
110
192
  }
111
193
 
112
- /** A stored durable anchor re-resolved against `units` (the bytes the agent will read next); null when the unit is gone. Without a durable, one is built from the browser's anchor. */
113
- function reanchor(units, c) {
114
- const d = c.durable;
115
- if (!d) return buildDurable(units, c.anchor);
116
- const r = resolveDurableAny(units, d);
117
- if (r.method === "none") return null;
118
- if (units.format === "docx") return { ...d, ...r, format: "docx", quote: d.quote, prefix: d.prefix, suffix: d.suffix, paraText: r.text, engineDelta: r.engineDelta };
119
- if (units.format === "xlsx") return buildDurable(units, { location: { sheetIndex: r.sheetIndex, sheetName: r.sheetName, range: r.range } });
120
- return buildDurable(units, { location: { slideIndex: r.slideIndex, slideId: r.slideId, shapeId: r.shapeId ?? undefined, shapeName: r.shapeName, cell: d.cell || undefined }, quote: d.quote, prefix: d.prefix, suffix: d.suffix, hitKind: d.quote ? undefined : "shape" });
194
+ /**
195
+ * An HTML file that is not saved as UTF-8 (PLAN-html R10, R18): 400 with the fixed sentence as it is and `code` (not_utf8 or the
196
+ * reason). The page shows the sentence; the UTF-8 copy it offers is a CLI action (`onionskin open` prints its command: src/html-utf8.js).
197
+ */
198
+ function encodingRefusal(real, r) {
199
+ return new HttpError(400, r.error, { code: r.code === "not_utf8" ? "not_utf8" : "encoding" });
121
200
  }
122
201
 
202
+ /** The person's pending HTML text edits a Send or "Save my edits" carries (web/src/html-view.js createEdits): checked fields only. */
203
+ const HTML_EDITS_MAX = 2000;
204
+ const HTML_EDIT_TEXT_MAX = 1_000_000; // one text node's new text; the whole request is held to REQUEST_MAX (src/server.js)
205
+ export function cleanHtmlEdits(list) {
206
+ if (list == null) return [];
207
+ if (!Array.isArray(list) || list.length > HTML_EDITS_MAX) throw new HttpError(400, `htmlEdits must be a list of at most ${HTML_EDITS_MAX} edits`, { code: "bad_edits" });
208
+ return list.map((e) => {
209
+ const ok = e && typeof e === "object" && Number.isSafeInteger(e.nodeId) && e.nodeId >= 0 && typeof e.versionSha === "string" && /^[0-9a-f]{64}$/.test(e.versionSha) && typeof e.baseTextHash === "string" && /^[0-9a-f]{14}$/.test(e.baseTextHash) && typeof e.newText === "string" && e.newText.length <= HTML_EDIT_TEXT_MAX;
210
+ if (!ok) throw new HttpError(400, "an HTML text edit must be {nodeId, versionSha, baseTextHash, newText}", { code: "bad_edits" });
211
+ return { nodeId: e.nodeId, versionSha: e.versionSha, baseTextHash: e.baseTextHash, newText: e.newText };
212
+ });
213
+ }
214
+ /** 409 version_changed: the page's edits were made on a version that is not the file on disk now (it reloads the view and places them again). */
215
+ const VERSION_CHANGED = "The file changed since this page loaded it, so your text edits were not written. The page shows the new version and puts your edits back where it can find their text; check them, then Send again.";
216
+ /** The write check refused the person's text edits: 409 for an edit made on other text (the page reloads), 422 for the others; nothing was written. */
217
+ function htmlEditRefusal(c) {
218
+ if (c.code === "edit_moved") return new HttpError(409, VERSION_CHANGED, { code: "version_changed" });
219
+ if (c.code === "check_failed" || c.failed) return new HttpError(503, `Your text edits could not be checked: ${c.reason || c.why || "the check did not finish"}. Nothing was written; your edits are still in this page. Try again in a moment.`, { code: "check_failed" });
220
+ return new HttpError(422, `Your text edits were not written: ${c.reason}.`, { code: c.code, ...(c.reasons ? { reasons: c.reasons } : {}) });
221
+ }
222
+
223
+ // The tasks the anchor work does for a request (src/anchor-work.js), and how a result is matched to its task: by the task's own content,
224
+ // so a comment changed while the worker ran finds no answer and goes out weaker instead of with another one's anchor.
225
+ const buildTask = (c) => ({ op: "build", anchor: c.anchor, pin: c.kind === "pin" });
226
+ const reanchorTask = (c) => ({ op: "reanchor", c: { durable: c.durable ?? null, anchor: c.durable ? null : c.anchor } });
227
+ const editAnchor = (e) => ({ location: e.location, quote: e.after, ...(e.cut ? { cut: true } : {}) });
228
+ const cardsTask = (results, ix, run) => ({ op: "cards", list: results.suggestions, units: ix.units, index: ix.index, reviewId: run.batch.batch_id });
229
+ // a quick fix's Use: the page anchor's location and text only (its kind of selection does not change where the sentence is)
230
+ const useTask = (a, rec, id) => ({ op: "use", anchor: { location: a.location, quote: a.quote, prefix: a.prefix, suffix: a.suffix }, rec: { quote: rec.quote, before: rec.before, after: rec.after }, id });
231
+ const taskKey = (t) => JSON.stringify(t);
232
+ /** A task's result that can be used: not missing, not an error of its own. A null anchor (the unit is not in the file) is a result. */
233
+ const usable = (r) => r !== undefined && !(r && typeof r === "object" && "error" in r && Object.keys(r).length === 1);
234
+
123
235
  /**
124
236
  * All session state and transitions, with no HTTP in it.
125
237
  * States (derived): idle, listening, working, queued, stranded, closed. poll results add timeout and superseded.
@@ -128,17 +240,31 @@ function reanchor(units, c) {
128
240
  * - Send while working: comments only, kept as `queued`; released to the next poller after `done`.
129
241
  * - Lease expiry or Take back: `delivered` is withdrawn, the document unlocks.
130
242
  */
243
+ // What one request may spend scanning a whole reading for moved anchors (src/text.js scanBudget): about 0.1 to 0.3 s here. Fifty moved
244
+ // comments on a Word document of 26,000 paragraphs took 10 s before. Past it, the rest resolve as moved or removed, and an anchor being
245
+ // made is made without the text it cannot pay for (src/xlsx.js xlsxDurable). The server's routes make and resolve anchors in the anchor
246
+ // worker (src/anchor-worker.js, ten times this budget, within a time limit); the synchronous methods (Send, done, revert, resolve, Use of
247
+ // a quick fix) do it on the caller's thread under this budget.
248
+ export const SCAN_BUDGET = 40_000_000;
249
+ const REFUSED_KEPT = 64;
250
+ const REFUSED_MS = 2 * 60_000;
251
+
131
252
  export class Sessions extends EventEmitter {
132
253
  // delivered run -> styleChangedOutside() when it went out; memory only, so a run reloaded after a restart has none and is not flagged
133
254
  #styleSeen = new WeakMap();
134
255
  // delivered run -> guideChangedOutside() when it went out (the same idea for voice-guide.json); taking it also arms the guide's guard
135
256
  #guideSeen = new WeakMap();
136
- constructor({ now = Date.now } = {}) {
257
+ // the last outline built for an agent Onionskin started: {key, sha, all} (one entry; ceiling: two such agents in turn build twice)
258
+ #outline = null;
259
+ // `anchors`: the anchor worker's call (src/offload.js anchorsOffThread); a test passes its own to control when the work answers
260
+ constructor({ now = Date.now, anchors = anchorsOffThread } = {}) {
137
261
  super();
138
262
  this.now = now;
263
+ this.anchors = anchors;
139
264
  this.map = new Map(); // key -> session
140
265
  this.createdCopies = new Set(); // absolute paths of conflict copies THIS server wrote: the only paths `showCopy` will reveal
141
266
  this.editors = new Map(); // key -> id of the one browser tab that may edit (in memory only: a tab's id lives as long as its page)
267
+ this.handovers = new Map(); // key -> {tab, until}: the tab another tab just took editing from (memory only), see handOver()
142
268
  this.briefed = new Map(); // key -> Set of holders that already received a complete batch in this session (memory only: a restarted server briefs again)
143
269
  // key -> how the file got from one version to the next, oldest first: {from, to, route: "apply", ops, partial, anchors} for an agent's
144
270
  // apply, {from, to, route: "commit"} for a commit, {from, to, route: "merge", stable} for a pass of the user's merge. The merge reads it
@@ -157,12 +283,69 @@ export class Sessions extends EventEmitter {
157
283
  if (s.closed) return false; // a closed session has no editor; the tab that reopens it claims it
158
284
  if (cur && cur !== tab && !takeover) return false;
159
285
  if (cur !== tab) {
286
+ if (cur) this.handovers.set(key, { tab: cur, until: this.now() + HANDOVER_MS }); // the tab that lost editing may hand over its unsaved words once
160
287
  this.editors.set(key, tab);
161
288
  this.emit("change", s, "state");
162
289
  }
163
290
  return true;
164
291
  }
165
292
 
293
+ /**
294
+ * The tab that just lost editing to another ("Edit here instead") hands over the words it had not saved yet: once, within HANDOVER_MS, and
295
+ * only that tab. Nothing another tab saved is replaced: a comment it made is added as a draft; words it typed into a draft nobody changed since
296
+ * it last saved it (`base`, per comment) replace it; words typed into one the store holds otherwise become a new draft next to it; its note is merged into the stored one from the note it started from (`base`, src/notes.js mergeNote),
297
+ * and the tab that edits now gets it as a note the server gave back (its stale save cannot overwrite it). Refused (409 `no_handover`) for
298
+ * any other tab or later, and like a draft save past a cap (413, nothing changes). The page that cannot hand over keeps the words visible
299
+ * with Copy my comments (web/src/App.jsx).
300
+ */
301
+ // key -> (tab -> {sent, seq}): the note the newest draft save of each tab carried, and the session `seq` it stored at (memory only, the last 8 tabs). A tab that hands over a note the server already took
302
+ // from it (its save's answer was lost, so it did not know) hands over nothing new: whatever happened to that note since is not its loss
303
+ #noteFrom = new Map();
304
+ handOver(key, tab, { comments, note, base, baseSeq, noteRev } = {}) {
305
+ const s = this.get(key);
306
+ const h = this.handovers.get(key);
307
+ if (!h || h.tab !== tab || h.until < this.now()) throw new HttpError(409, "This tab no longer edits the file.", { code: "no_handover" });
308
+ if (!Array.isArray(comments)) throw new HttpError(400, "comments must be an array");
309
+ const byId = new Map(s.comments.map((c) => [c.id, c]));
310
+ const next = [...s.comments];
311
+ for (const c of comments.filter((x) => x && typeof x === "object")) {
312
+ const id = cutText(String(c.id), 64);
313
+ const old = byId.get(id);
314
+ const text = userText(c.text, COMMENT_TEXT_MAX);
315
+ const del = s.deletedDrafts?.[id];
316
+ if (!old && del === text) continue; // deleted in the tab that edits now, with these very words: it stays deleted
317
+ if (!old) next.push(this.#draft(c, del === undefined ? id : cutText(`${id}~h${crypto.randomBytes(3).toString("hex")}`, 64), null)); // deleted there, with other words: these come back as a new draft
318
+ else if (text === old.text) continue;
319
+ else if (old.status === "draft" && !old.answer && typeof c.base === "string" && userText(c.base, COMMENT_TEXT_MAX) === old.text) next[next.indexOf(old)] = this.#draft(c, id, old); // nobody changed it since: the newer words replace it
320
+ else next.push(this.#draft(c, cutText(`${id}~h${crypto.randomBytes(3).toString("hex")}`, 64), null));
321
+ }
322
+ if (next.length > COMMENTS_MAX) throw tooMany();
323
+ for (const c of next) if (c.status === "draft" && jsonSize(c) > COMMENT_MAX) throw tooLarge("This draft");
324
+ let text = s.note;
325
+ const from = this.#noteFrom.get(key);
326
+ const last = from?.get(tab);
327
+ const already = typeof note === "string" && (last?.sent === userText(note, NOTE_MAX + 1) || this.#noteTaken(s, noteRev, tab)); // or a Send of that tab took it
328
+ if (typeof note === "string" && note !== s.note && !already) {
329
+ // what it started from: the note the page's box is based on (`base`) when that came from a state at or after that tab's last save
330
+ // (`baseSeq`: the page took that save's answer, or something newer, as a Take back's); else the note that save carried (its answer
331
+ // was lost, so the box is that note and what was typed since); with no save of that tab, what the page says
332
+ const fresh = typeof base === "string" && Number.isSafeInteger(baseSeq) && (!last || baseSeq >= last.seq);
333
+ const from = fresh ? base : last ? last.sent : typeof base === "string" ? base : "";
334
+ const mine = noteOf(note);
335
+ // the box was emptied here since (a Send took the note that tab started from): only what that tab added after it is handed over
336
+ text = !s.note && from && mine.startsWith(from) ? mine.slice(from.length).replace(/^\s+/, "") : mergeNote(from, s.note, mine);
337
+ }
338
+ if ([...text].length > NOTE_MAX && [...text].length > [...s.note].length) throw noteTooLong(true);
339
+ const before = s.note;
340
+ this.#commitDraft(s, { comments: next, note: before }, null, tab, true);
341
+ this.handovers.delete(key);
342
+ if (text !== before) {
343
+ this.#giveNote(s, text);
344
+ this.#changed(s, "draft");
345
+ } else this.emit("change", s, "draft");
346
+ return s;
347
+ }
348
+
166
349
  /** The tab's event stream ended (closed, reloaded): the file has no editor until a tab claims it. */
167
350
  releaseEditor(key, tab) {
168
351
  if (this.editors.get(key) !== tab) return;
@@ -178,8 +361,50 @@ export class Sessions extends EventEmitter {
178
361
  }
179
362
 
180
363
  // ---- open ------------------------------------------------------------------------------------------------
181
- /** `opts.agent === "background"`: the person opened it from a terminal, so the server's background agent works on it (stored on the session). */
182
- open(input, opts = {}) {
364
+ /**
365
+ * open() as the server's routes call it: the file is first read in a worker thread with a time and memory limit (src/offload.js
366
+ * readOffThread: the package check and every part the server reads later), so a slow or hostile file never stalls the server, and a
367
+ * file no reader accepts is refused here, with the reason, instead of failing later at Send.
368
+ */
369
+ async openAsync(input, opts = {}) {
370
+ const { real, format } = this.#checkInput(input);
371
+ // A file that changes between the worker's reading and open()'s own (a swap, a save in progress) is read in the worker again, once:
372
+ // open() never takes bytes the worker has not read, so nothing on this thread reads a version past the readers' caps.
373
+ for (let attempt = 0; ; attempt++) {
374
+ const bytes = readCapped(real);
375
+ const sha = sha256(bytes);
376
+ const known = this.#refused.get(`${format}:${sha}`);
377
+ const r = known && this.now() - known.at < REFUSED_MS ? known : await readOffThread(bytes, format);
378
+ if (r.ok) readInCaps(format, r.sha);
379
+ if (!r.ok) {
380
+ // the same bytes are refused again at once (an agent's polls fall back to opening a file that is not open, again and again)
381
+ if (r.stage !== "busy") this.#refuse(`${format}:${sha}`, r);
382
+ if (r.stage === "encoding") throw encodingRefusal(real, r);
383
+ if (r.stage === "invalid") throw new HttpError(400, `Not a valid .${format}: ${r.error}`);
384
+ if (r.stage === "busy") throw new HttpError(503, `Onionskin is busy opening other files: ${r.error}. Try again in a minute.`, { code: "busy" });
385
+ const hint = r.stage === "timeout" || r.stage === "memory" ? " It may be damaged, or made to stall programs that read it." : "";
386
+ throw new HttpError(400, `This file could not be opened: ${r.error}.${hint}`, { code: r.stage === "timeout" ? "open_timeout" : "unreadable" });
387
+ }
388
+ try {
389
+ return this.open(input, { ...opts, checkedSha: r.sha, onlyChecked: true });
390
+ } catch (err) {
391
+ if (err?.extra?.code !== "changed_while_opening" || attempt >= 1) throw err;
392
+ }
393
+ }
394
+ }
395
+
396
+ // "<format>:<sha256>" -> what the open worker said about those bytes when it refused them, for REFUSED_MS (a refusal for time may not hold once the
397
+ // Mac is less busy); at most REFUSED_KEPT, oldest dropped first
398
+ #refused = new Map();
399
+ #refuse(id, r) {
400
+ if (r.at) return; // already a remembered answer
401
+ this.#refused.delete(id);
402
+ this.#refused.set(id, { ok: false, stage: r.stage, error: r.error, code: r.code, copyable: r.copyable, at: this.now() });
403
+ if (this.#refused.size > REFUSED_KEPT) this.#refused.delete(this.#refused.keys().next().value);
404
+ }
405
+
406
+ /** The path checks of open(): the real path, its format and its lstat. Throws HttpError. */
407
+ #checkInput(input) {
183
408
  let real;
184
409
  try {
185
410
  real = fs.realpathSync(path.resolve(input));
@@ -190,23 +415,36 @@ export class Sessions extends EventEmitter {
190
415
  if (hasControlChars(real)) throw new HttpError(400, "This file's path contains control or bidirectional-text characters. Rename or move the file and open it again.", { code: "unsafe_path" });
191
416
  const ext = path.extname(real).toLowerCase();
192
417
  if (REFUSED_EXTENSIONS.has(ext)) throw new HttpError(400, `${ext} (macro-enabled) files are refused`);
193
- const format = ext.slice(1);
194
- if (!FORMATS[format]) throw new HttpError(400, "Only .docx, .xlsx and .pptx files are supported");
418
+ if (Object.hasOwn(REFUSED_PAGES, ext)) throw new HttpError(400, REFUSED_PAGES[ext], { code: "refused_kind" });
419
+ const format = formatOfExt(ext);
420
+ if (!format) throw new HttpError(400, ONLY_FORMATS);
195
421
  let st;
196
422
  try {
197
423
  st = checkRegularOwned(real);
198
424
  } catch (err) {
199
425
  throw new HttpError(400, err.message);
200
426
  }
427
+ return { real, format, st };
428
+ }
429
+
430
+ /** `opts.agent === "background"`: the person opened it from a terminal, so the server's background agent works on it (stored on the session). */
431
+ open(input, opts = {}) {
432
+ const { real, format, st } = this.#checkInput(input);
201
433
  const bytes = readCapped(real);
202
- try {
203
- validatePackage(bytes, format);
204
- } catch (err) {
205
- throw new HttpError(400, `Not a valid .${format}: ${err.message}`);
434
+ const sha = sha256(bytes);
435
+ // openAsync already read these exact bytes in a worker; for it, other bytes (the file changed since) go back to the worker. Any other
436
+ // caller (tests, tools) gets the package check here.
437
+ if (opts.onlyChecked && opts.checkedSha !== sha) throw new HttpError(409, "The file kept changing while Onionskin opened it. Open it again once it is saved.", { code: "changed_while_opening" });
438
+ if (opts.checkedSha !== sha) {
439
+ try {
440
+ validatePackage(bytes, format);
441
+ } catch (err) {
442
+ if (err instanceof HtmlEncodingError) throw encodingRefusal(real, { error: err.message, code: err.code, copyable: err.copyable });
443
+ throw new HttpError(400, `Not a valid .${format}: ${err.message}`);
444
+ }
206
445
  }
207
446
  const key = sessionKey(real);
208
447
  let s = this.map.get(key);
209
- const sha = sha256(bytes);
210
448
  if (!s) {
211
449
  s = this.#load(key) || this.#fresh(key, real, format, sha);
212
450
  s.file = real;
@@ -245,7 +483,7 @@ export class Sessions extends EventEmitter {
245
483
  const j = JSON.parse(fs.readFileSync(f, "utf8"));
246
484
  const s = { ...this.#fresh(key, j.file, j.format, j.baseSha), ...j, pollers: [] };
247
485
  // a hard kill can leave work in flight that no result will ever settle
248
- if (s.delivered?.stage === "checking") s.delivered.stage = "writing";
486
+ if (s.delivered?.stage === "checking") (s.delivered.stage = "writing"), (s.delivered.checkingAt = null);
249
487
  if (s.delivered?.local) {
250
488
  // the local runner died with the server. Not yet writing: the batch waits for an agent, as if handed back. Writing: the lease
251
489
  // expires at the first tick, which records what was written (it needs the snapshots, which open() sets after this)
@@ -269,8 +507,128 @@ export class Sessions extends EventEmitter {
269
507
  }
270
508
 
271
509
  #save(s) {
510
+ const { raw, cut } = this.#encode(s);
511
+ if (cut) Object.assign(s, cut); // in memory too
512
+ writeStateFile(path.join(sub("sessions"), `${s.key}.json`), raw);
513
+ }
514
+
515
+ /**
516
+ * The session file's text for `s`: {raw, cut}. The one cap on a session file (src/retain.js SESSION_MAX, in bytes): past it every
517
+ * comment's page anchor and every typed edit are cut further (`cut`: those lists, for the caller to keep); nothing is dropped. Each part
518
+ * is cut where it is stored, so this is a last line. `raw` can still be past the cap; a change the page asked for is then refused (#fits).
519
+ */
520
+ #encode(s) {
272
521
  const { pollers, snapshots, ...rest } = s;
273
- writeStateFile(path.join(sub("sessions"), `${s.key}.json`), JSON.stringify(rest));
522
+ const raw = JSON.stringify(rest);
523
+ if (Buffer.byteLength(raw) <= SESSION_MAX) return { raw, cut: null };
524
+ const cut = {
525
+ comments: rest.comments.map((c) => capComment({ ...c, anchor: c.anchor && { ...c.anchor, cut: true, quote: cutText(c.anchor.quote, 2000), paraText: cutText(c.anchor.paraText, 2000) } })),
526
+ edits: rest.edits.map((e) => ({ ...e, before: cutText(e.before, 2000), after: cutText(e.after, 2000), cut: true })),
527
+ };
528
+ return { raw: JSON.stringify({ ...rest, ...cut }), cut };
529
+ }
530
+
531
+ /** Whether the session with `patch` fits its file once cut (#encode): checked before a change the page asked for touches anything. */
532
+ #fits(s, patch) {
533
+ return Buffer.byteLength(this.#encode({ ...s, ...patch }).raw) <= SESSION_MAX;
534
+ }
535
+
536
+ /** The page's typed edits since the last Send, as they are kept for the next batch: each field cut (src/retain.js), at most EDITS_MAX in all. */
537
+ #editsToTake(s, edits) {
538
+ return (Array.isArray(edits) ? edits.slice(0, Math.max(0, EDITS_MAX - s.edits.length)) : []).map(cleanEdit);
539
+ }
540
+
541
+ // ---- anchor work: the anchor worker, or this thread (src/anchor-work.js) ------------------------------------------------
542
+ /**
543
+ * The results of `tasks` on the version `sha` (`bytes`: a Buffer, or a function that returns it). With `prepared` (what the anchor
544
+ * worker answered for the same tasks, src/session.js #offThread) they are taken from it, and nothing is read here: a task it has no
545
+ * answer for (its work did not finish, or it was for another version) reads as not done. Without, they are worked out on this thread
546
+ * under the request's budget, as the synchronous methods always did. Returns {read, outline, at(i)}.
547
+ */
548
+ #results(format, bytes, sha, tasks, prepared, { check = false } = {}) {
549
+ if (prepared) {
550
+ const ok = prepared.ok && prepared.sha === sha;
551
+ return { read: ok ? prepared.read : null, outline: ok ? prepared.outline : null, at: (i) => (ok ? prepared.map.get(taskKey(tasks[i])) : undefined) };
552
+ }
553
+ // an HTML file is read in workers only (the parser is not on this thread): without the worker's answer the work is not done, and each
554
+ // caller makes its anchors weaker, as when the worker did not finish
555
+ if (format === "html") return { read: null, outline: null, at: () => undefined };
556
+ const r = anchorWork({ format, bytes, sha, check, tasks, budget: SCAN_BUDGET });
557
+ return { read: r.read, outline: null, at: (i) => r.results[i] };
558
+ }
559
+
560
+ /** Bytes about to become the file: what their reading found. Past a cap 422 (CapError); not read in time and never read whole before: 503. */
561
+ #readVerdict(work, format, sha, refusal) {
562
+ const read = work.read;
563
+ if (read?.ok) return readInCaps(format, sha);
564
+ if (read?.cap) throw new CapError(read.cap);
565
+ if (read?.invalid) throw new HttpError(422, `${refusal}: ${untrusted(read.invalid, 200)}`, { code: "invalid_package" });
566
+ if (verified.has(`${format}:${sha}`)) return; // the work did not finish, but this version was read whole within the caps before
567
+ throw new HttpError(503, "Onionskin could not finish reading the file in time, so nothing was sent. Your comments are kept. Try again in a moment.", { code: "read_timeout" });
568
+ }
569
+
570
+ /** `job` ({source, check, tasks, outline}) in the anchor worker: {ok: true, sha, read, outline, map: task key -> result} or {ok: false, why}. */
571
+ async #offThread(format, job) {
572
+ const r = await this.anchors({ format, ...job });
573
+ if (!r.ok) return { ok: false, why: r.failed };
574
+ const map = new Map();
575
+ job.tasks.forEach((t, i) => map.set(taskKey(t), r.r.results[i]));
576
+ return { ok: true, sha: r.sha, read: r.r.read, outline: r.r.outline ?? null, map };
577
+ }
578
+
579
+ // the async methods of one file run in turn (another file's never wait for them); synchronous ones keep their own order
580
+ #chains = new Map();
581
+ #serial(key, fn) {
582
+ const prev = this.#chains.get(key) || Promise.resolve();
583
+ const run = prev.then(fn, fn);
584
+ const tail = run.then(() => {}, () => {});
585
+ this.#chains.set(key, tail);
586
+ tail.then(() => this.#chains.get(key) === tail && this.#chains.delete(key));
587
+ return run;
588
+ }
589
+
590
+ /**
591
+ * `fn` (a synchronous change the page asks for: a draft save, Take back, an answer) in this file's turn, as the server's routes call it.
592
+ * One that arrives while a Send waits for the anchor worker is applied after that Send commits, against what it committed, as when Send
593
+ * was synchronous. Never called from inside a turn (it would wait for itself).
594
+ */
595
+ inTurn(key, fn) {
596
+ this.get(key);
597
+ return this.#serial(key, () => fn());
598
+ }
599
+
600
+ /** setDraft() in this file's turn (the route POST /api/draft). */
601
+ setDraftAsync(key, args) {
602
+ return this.inTurn(key, () => this.setDraft(key, args));
603
+ }
604
+
605
+ /**
606
+ * Resolves once every request of this file that arrived before now is answered, so a page that loads (or reloads) during a Send starts
607
+ * from what it committed, never from the middle of it. Bounded: a Send plans its worker job at most twice, each within the worker's time
608
+ * limit; past that the page gets the state as it is then.
609
+ */
610
+ async settled(key, ms = 2 * anchorTimeoutMs() + 5_000) {
611
+ const tail = this.#chains.get(key);
612
+ if (!tail) return;
613
+ let timer;
614
+ await Promise.race([tail, new Promise((r) => (timer = setTimeout(r, ms)))]);
615
+ clearTimeout(timer);
616
+ }
617
+
618
+ /**
619
+ * An agent Onionskin started polls while a batch waits for it (src/server.js /cli/poll): the outline its delivery carries is built in
620
+ * the anchor worker first, so the delivery finds it (#withOutline's memo) instead of reading the file on this thread.
621
+ */
622
+ async outlineAhead(key, holder) {
623
+ const s = this.get(key);
624
+ if (!isAgentHolder(holder) || !s.pending || s.delivered || s.pending.instructions?.review || s.pending.instructions?.voice_request) return;
625
+ const r = await anchorsOffThread({ format: s.format, source: { path: s.file }, tasks: [], outline: true });
626
+ if (r.ok && r.r.read.ok && r.r.outline) this.#outline = { key: s.key, sha: r.sha, all: r.r.outline };
627
+ }
628
+
629
+ /** An agent Onionskin started is waiting for this file: its delivery carries the outline, which the anchor worker then builds too. */
630
+ #wantsOutline(s) {
631
+ return s.pollers.some((p) => isAgentHolder(p.holder));
274
632
  }
275
633
 
276
634
  get(key) {
@@ -325,6 +683,9 @@ export class Sessions extends EventEmitter {
325
683
  strandedCommand: state === "stranded" ? `${PRODUCT.bin} poll ${shellQuote(s.file)}` : null,
326
684
  startCommand: state === "stranded" ? `${PRODUCT.bin} ${shellQuote(s.file)}` : null, // what a person types in a terminal to start an agent on this file
327
685
  seq: s.seq,
686
+ // per tab, the newest Send (or quick fix's Use) of it that committed: {tab: {rev, noteRev}}. A page whose Send got no answer learns
687
+ // from it whether that Send went through (web/src/App.jsx checkSent)
688
+ sent: s.sent || {},
328
689
  editor: this.editors.get(s.key) || null,
329
690
  conflict: s.conflict || null,
330
691
  canReveal: canReveal(),
@@ -343,31 +704,198 @@ export class Sessions extends EventEmitter {
343
704
  }
344
705
 
345
706
  // ---- drafts ----------------------------------------------------------------------------------------------
346
- /** Browser replaces its list of comments/pins and the overall note. Server-owned fields (status, results) are kept. */
347
- setDraft(key, { comments, note }) {
707
+ // key -> {tab, rev}: the newest save or Send a tab numbered (memory only: a reloaded page is a new tab and counts again). Saves and Sends
708
+ // run in turn (inTurn), but two requests of one tab can still arrive out of order: an autosave sent just before Send was pressed can land
709
+ // after the Send committed, and would put the note the Send took back in the box. One numbered at or below what the tab already sent
710
+ // changes nothing (test/send-race.test.js).
711
+ #revs = new Map();
712
+ // s.sent = {tab: {rev, noteRev}}: per tab (the last SENT_TABS), the newest numbered Send (or Use) of it that committed, and the revision
713
+ // of its note box the newest such Send took. Stored with the session, so a page still asking whether its Send went through gets the right
714
+ // answer after the server restarts. A save of that tab whose note is that revision or older (its save on the way out while the Send was in
715
+ // flight) leaves the note alone, and so does its handover: a note a Send delivered is never put back.
716
+ #sentRev(s, { rev, tab, noteRev }, took = false) {
717
+ if (!Number.isSafeInteger(rev) || typeof tab !== "string") return;
718
+ const all = s.sent && typeof s.sent === "object" ? s.sent : {};
719
+ const last = all[tab];
720
+ if (last && last.rev > rev) return;
721
+ const { [tab]: _, ...rest } = all;
722
+ const entry = { rev, noteRev: took && Number.isSafeInteger(noteRev) ? noteRev : last?.noteRev ?? null };
723
+ s.sent = Object.fromEntries([...Object.entries(rest), [tab, entry]].slice(-SENT_TABS));
724
+ }
725
+ /** Whether `noteRev` of `tab` is a note a committed Send of that tab already took (see s.sent). */
726
+ #noteTaken(s, noteRev, tab) {
727
+ const last = typeof tab === "string" ? s.sent?.[tab] : null;
728
+ return Number.isSafeInteger(noteRev) && Number.isSafeInteger(last?.noteRev) && noteRev <= last.noteRev;
729
+ }
730
+
731
+ /**
732
+ * Browser replaces its list of comments/pins and the overall note. Server-owned fields (status, results) are kept: a sent, queued or
733
+ * finished comment is as the server holds it, whatever words the page sends for it. `note` absent (not a string): the stored note is
734
+ * kept (the page's save while a Send is in flight, which took the note). `rev` (optional): see #revs. Every field is rebuilt from what
735
+ * Onionskin reads (src/retain.js). Refused (413) before anything changes: more than COMMENTS_MAX comments, a note past NOTE_MAX, a
736
+ * comment past COMMENT_MAX after its cuts, a session past SESSION_MAX after the writer's. Nothing is dropped or cut without a word.
737
+ */
738
+ setDraft(key, { comments, note, rev, tab, noteRev, seen, noteSeen } = {}) {
348
739
  const s = this.get(key);
349
740
  if (!Array.isArray(comments)) throw new HttpError(400, "comments must be an array");
741
+ if (this.#overtaken(s, rev, tab, true)) return s;
742
+ this.#commitDraft(s, this.#draftPlan(s, { comments, note: this.#noteTaken(s, noteRev, tab) ? undefined : note, seen, noteSeen, tab }), rev, tab);
743
+ if (typeof note === "string" && tab) {
744
+ const m = this.#noteFrom.get(key) || new Map();
745
+ m.delete(tab);
746
+ m.set(tab, { sent: userText(note, NOTE_MAX + 1), seq: s.seq, used: !this.#noteTaken(s, noteRev, tab) }); // see handOver and #draftPlan
747
+ if (m.size > 8) m.delete(m.keys().next().value); // the tabs of one file: a handful
748
+ this.#noteFrom.set(key, m);
749
+ }
750
+ return s;
751
+ }
752
+
753
+ /** Whether this tab already numbered a newer save or Send than `rev` (`orSame`: one numbered the same too). See #revs. */
754
+ #overtaken(s, rev, tab, orSame = false) {
755
+ if (!Number.isSafeInteger(rev)) return false;
756
+ const last = this.#revs.get(s.key);
757
+ return !!last && last.tab === tab && (orSame ? rev <= last.rev : rev < last.rev);
758
+ }
759
+
760
+ /**
761
+ * What a draft save of `comments` and `note` would store, {comments, note}, with every limit checked and nothing changed. `note` absent:
762
+ * the stored note. A note past NOTE_MAX is refused, except one not longer than the stored note (Take back can join notes past it: the
763
+ * page saves the box it was given, and may shorten it).
764
+ */
765
+ #draftPlan(s, { comments, note, seen, noteSeen, tab }) {
766
+ if (!Array.isArray(comments)) throw new HttpError(400, "comments must be an array");
767
+ const list = comments.filter((c) => c && typeof c === "object");
768
+ if (list.length > COMMENTS_MAX) throw tooMany();
769
+ const long = typeof note === "string" && note.length > NOTE_MAX && note.length <= s.note.length;
770
+ let text = typeof note !== "string" ? s.note : long ? userText(note, s.note.length) : noteOf(note);
771
+ // the server gave a note back to the box (#giveNote) that the page had not seen when it sent this (`seen`: the newest session `seq` it
772
+ // took): the page's note is merged into it, never put over it
773
+ // (`noteSeen`: the newest state whose note the page's box took or is based on; a page can see a state and keep its own words)
774
+ const given = s.noteGiven;
775
+ const ns = Number.isSafeInteger(noteSeen) ? noteSeen : seen;
776
+ if (typeof note === "string" && given && Number.isSafeInteger(ns) && ns <= given.seq) {
777
+ // from the note the server held before the first give the page had not seen (`steps`: each give; a session stored before them has
778
+ // the first one's only); or, when a save of this tab was merged in since (its answer still on its way), from the note that save
779
+ // carried: the box is that note and what was typed since, and the stored note already has that save's words
780
+ // (ceiling: the last 50 gives; past that, a page that saw none of the kept ones merges from the oldest note, as before `steps`)
781
+ const hit = given.steps?.find((g) => g.seq >= ns);
782
+ const first = hit && !(given.cut && hit === given.steps[0]) ? hit : given;
783
+ const last = this.#noteFrom.get(s.key)?.get(tab);
784
+ text = mergeNote(last?.used && last.seq > first.seq ? last.sent : first.before, s.note, text);
785
+ }
350
786
  const prev = new Map(s.comments.map((c) => [c.id, c]));
351
- s.comments = comments.slice(0, 500).map((c) => {
352
- const old = prev.get(String(c.id));
353
- const base = {
354
- id: String(c.id).slice(0, 64),
355
- kind: ["text", "element", "pin"].includes(c.kind) ? c.kind : "element",
356
- text: userText(c.text),
357
- anchor: c.anchor && typeof c.anchor === "object" ? c.anchor : null,
358
- pin: c.pin && typeof c.pin === "object" ? cleanPin(c.pin) : null,
359
- pinNo: Number.isInteger(c.pinNo) ? c.pinNo : null,
360
- };
361
- if (old && old.status !== "draft") return { ...old, text: old.text, kind: old.kind }; // sent/done comments are frozen
362
- // an answered question: the words, the question and the answer stay as the person left them (the page cannot change them)
363
- const keep = old?.skipLocal ? { skipLocal: true } : {}; // a reverted local run: the next Send of this comment goes to the agent
364
- if (old?.localHint && userText(c.text) === old.text) keep.localHint = old.localHint; // the words changed: the hint no longer applies
365
- if (old?.answer) return { ...base, text: old.text, status: "draft", durable: old.durable || null, question: old.question, answer: old.answer, ...keep };
366
- return { ...base, status: "draft", durable: old?.durable || null, ...keep };
787
+ const next = list.map((c) => {
788
+ const id = cutText(String(c.id), 64);
789
+ const old = prev.get(id);
790
+ return old && old.status !== "draft" ? old : this.#draft(c, id, old); // sent/queued/finished comments are frozen
367
791
  });
368
- s.note = userText(typeof note === "string" ? note : "");
369
- this.#changed(s, "draft");
370
- return s;
792
+ // a comment on its way to the agent (in a waiting batch, or queued) stays whatever list a save carries: a save never changes what a
793
+ // Send delivers (a page that had not seen it yet, after a reload, does not take it out of the batch). Nor does a save delete a draft
794
+ // another tab wrote after the state its page took (`seen`): the page did not know of it (a window that handed over its words, the save
795
+ // of a page that was left landing after the reloaded page loaded)
796
+ const named = new Set(next.map((c) => c.id));
797
+ const unseen = (c) => c.status === "draft" && Number.isSafeInteger(seen) && c.wroteBy && c.wroteBy.tab !== tab && c.wroteBy.seq > seen;
798
+ for (const c of s.comments) if ((c.status === "sent" || c.status === "queued" || unseen(c)) && !named.has(c.id)) next.push(c);
799
+ if (next.length > COMMENTS_MAX) throw tooMany();
800
+ for (const c of next) if (c.status === "draft" && jsonSize(c) > COMMENT_MAX) throw tooLarge("This draft");
801
+ return { comments: next, note: text, seen: Number.isSafeInteger(noteSeen) ? noteSeen : seen };
802
+ }
803
+
804
+ /**
805
+ * The server puts `text` in the note box itself (Take back, a run that stopped with comments queued behind it, the Revert of a run on this
806
+ * Mac). Until a save shows the page has seen it, a save of the note from the page is merged into it (#draftPlan): the note it had then is kept.
807
+ */
808
+ #giveNote(s, text) {
809
+ if (text === s.note) return;
810
+ const step = { seq: s.seq, before: s.note };
811
+ if (s.noteGiven) {
812
+ const steps = [...(s.noteGiven.steps || []), step];
813
+ s.noteGiven = { ...s.noteGiven, seq: s.seq, steps: steps.slice(-50), ...(steps.length > 50 || !s.noteGiven.steps ? { cut: true } : {}) };
814
+ } else s.noteGiven = { ...step, steps: [step] };
815
+ s.note = text;
816
+ }
817
+
818
+ /** Stores a plan of #draftPlan as a draft save: refused (413) past the session cap before anything changes. */
819
+ #commitDraft(s, plan, rev, tab, quiet = false) {
820
+ // each draft remembers which tab last wrote its words, and when (#draftPlan: a save does not delete one its page has not seen)
821
+ const prev = new Map(s.comments.map((c) => [c.id, c]));
822
+ for (const c of plan.comments) {
823
+ if (c.status !== "draft") continue;
824
+ const old = prev.get(c.id);
825
+ if (!old || old.status !== "draft" || old.text !== c.text) c.wroteBy = { tab: tab ?? null, seq: s.seq + 1 };
826
+ else if (old.wroteBy && c !== old) c.wroteBy = old.wroteBy;
827
+ }
828
+ // the words of drafts this save deletes, by id (the last DELETED_KEPT): a window handing over words afterwards does not bring them back
829
+ const kept = new Set(plan.comments.map((c) => c.id));
830
+ const gone = [...prev.values()].filter((c) => c.status === "draft" && !kept.has(c.id));
831
+ const deleted = gone.length ? Object.fromEntries([...Object.entries(s.deletedDrafts || {}), ...gone.map((c) => [c.id, c.text])].slice(-DELETED_KEPT)) : s.deletedDrafts;
832
+ const patch = { comments: plan.comments, note: plan.note, seq: s.seq + 1, ...(deleted ? { deletedDrafts: deleted } : {}) };
833
+ const { raw, cut } = this.#encode({ ...s, ...patch });
834
+ if (Buffer.byteLength(raw) > SESSION_MAX) throw tooLarge("This draft");
835
+ if (Number.isSafeInteger(rev) && !this.#overtaken(s, rev, tab)) this.#revs.set(s.key, { tab, rev }); // never lowered
836
+ Object.assign(s, patch, cut || {});
837
+ if (s.noteGiven && Number.isSafeInteger(plan.seen) && plan.seen > s.noteGiven.seq) delete s.noteGiven; // the page has seen it
838
+ writeStateFile(path.join(sub("sessions"), `${s.key}.json`), raw);
839
+ // `quiet`: a Send's (or a Use's) save, which its commit announces; an event for the moment between would show its note still in the box
840
+ if (!quiet) this.emit("change", s, "draft");
841
+ }
842
+
843
+ /** One draft as stored, from the page's comment `c` (`old`: the stored draft of the same id, if any). */
844
+ #draft(c, id, old) {
845
+ const base = {
846
+ id,
847
+ kind: ["text", "element", "pin"].includes(c.kind) ? c.kind : "element",
848
+ text: userText(c.text, COMMENT_TEXT_MAX),
849
+ anchor: cleanPageAnchor(c.anchor), // the fields the server and the page read, each cut (src/retain.js)
850
+ pin: c.pin && typeof c.pin === "object" ? cleanPin(c.pin) : null,
851
+ pinNo: Number.isInteger(c.pinNo) ? c.pinNo : null,
852
+ };
853
+ // an answered question: the words, the question and the answer stay as the person left them (the page cannot change them)
854
+ const keep = old?.skipLocal ? { skipLocal: true } : {}; // a reverted local run: the next Send of this comment goes to the agent
855
+ if (old?.localHint && base.text === old.text) keep.localHint = old.localHint; // the words changed: the hint no longer applies
856
+ if (old?.answer) return capComment({ ...base, text: old.text, status: "draft", durable: old.durable || null, question: old.question, answer: old.answer, ...keep });
857
+ return capComment({ ...base, status: "draft", durable: old?.durable || null, ...keep });
858
+ }
859
+
860
+ /**
861
+ * What a Send (or a quick fix's Use: `useId`, the stamp it delivers) stores before it commits, as #draftPlan: nothing changed.
862
+ * Invariant: a Send commits exactly its own payload, whatever draft save is numbered before or after it. Normally its list and note
863
+ * replace the stored ones, as a draft save does. When this tab already numbered a newer save (one sent after the Send was pressed whose
864
+ * body arrived first, as a page's visibility-change save can), that save is the stored list and the Send is authoritative only for what
865
+ * it delivers: the drafts it names (their words as the Send carried them) and its note. Everything else stays as the newer save left it:
866
+ * a comment it added is a draft, one it deleted stays deleted (the page cannot delete what a Send in flight carries), its note (none:
867
+ * it leaves the note to the Send). A Use delivers only its stamp, so then it changes nothing else, not the note either.
868
+ */
869
+ #sentPlan(s, { comments, note, rev, tab, seen, noteSeen, sends }, useId = null) {
870
+ const list = Array.isArray(comments) ? comments : s.comments;
871
+ // `sends` (optional): the ids the page holds read only for this Send, the drafts it means to deliver; a comment the page still showed as
872
+ // queued or sent when Send was pressed (and could be typed into after) is not delivered by it
873
+ const named = Array.isArray(sends) ? new Set(sends.slice(0, COMMENTS_MAX).map((id) => cutText(String(id), 64))) : null;
874
+ if (!this.#overtaken(s, rev, tab)) {
875
+ const plan = this.#draftPlan(s, { comments: list, note: typeof note === "string" ? note : s.note, seen, noteSeen, tab });
876
+ return named && useId == null ? { ...plan, only: named } : plan;
877
+ }
878
+ const own = this.#draftPlan(s, { comments: list.filter((c) => c && typeof c === "object" && (useId == null || String(c.id) === useId)), note: useId == null && typeof note === "string" ? note : s.note, seen, noteSeen, tab });
879
+ const mine = new Map(own.comments.map((c) => [c.id, c]));
880
+ // `only`: the drafts this Send delivers (the others are the newer save's, and stay drafts)
881
+ return { comments: s.comments.map((c) => (c.status === "draft" && mine.has(c.id) ? mine.get(c.id) : c)), note: own.note, only: new Set([...mine.keys()].filter((id) => !named || useId != null || named.has(id))) };
882
+ }
883
+
884
+ /** The drafts of `comments` a Send with `plan` delivers: all of them, or those it names when a newer save was stored first. */
885
+ #delivers(comments, plan) {
886
+ return comments.filter((c) => c.status === "draft" && (!plan.only || plan.only.has(c.id)));
887
+ }
888
+
889
+ /**
890
+ * The 413 refusals of a Send, on the state its plan would make (nothing changed yet): the bytes past the cap, a note that joined with
891
+ * the one waiting (the batch an agent has not taken, or the queued note during a run) would be past NOTE_MAX, a session that would not
892
+ * fit its file. Run before the worker is asked and again at the commit, against the state then.
893
+ */
894
+ #sendLimits(s, plan, payload) {
895
+ if (payload.bytes?.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
896
+ if (s.delivered) joinNotes(s.queuedNote, plan.note);
897
+ else joinNotes(s.pending && !readOnlyKind(s.pending) ? s.pending.instructions?.overall_note || "" : "", plan.note);
898
+ if (!this.#fits(s, { comments: plan.comments, edits: [...s.edits, ...this.#editsToTake(s, payload.edits)] })) throw tooLarge("This Send");
371
899
  }
372
900
 
373
901
  // ---- send ------------------------------------------------------------------------------------------------
@@ -376,58 +904,211 @@ export class Sessions extends EventEmitter {
376
904
  * Compare-and-swap on the disk hash. Returns the session.
377
905
  */
378
906
  send(key, payload) {
907
+ return scanBudget(SCAN_BUDGET, () => this.#send(key, payload));
908
+ }
909
+
910
+ /**
911
+ * send() as the server's route calls it: the reading of the bytes that become the file and every anchor of the batch are worked out
912
+ * first, in the warm anchor worker with a time limit (src/offload.js anchorsOffThread), then send() commits with what came back and
913
+ * reads nothing itself. Work that did not finish leaves its comments weaker (src/anchor-work.js weakDurable), never lost; bytes it
914
+ * could not read in time are refused with "try again" unless that version was read whole before.
915
+ * Invariant: a Send and the page's draft saves, Take back and answers of one file run in turn (inTurn), as when Send was synchronous.
916
+ * The Send saves its list and note when its turn comes, then commits exactly that; whatever arrives meanwhile waits and is applied after
917
+ * the commit, against what it committed. A page loaded meanwhile waits for it too (settled).
918
+ */
919
+ async sendAsync(key, payload) {
920
+ const s = this.get(key);
921
+ const hasBytes = !!payload.bytes?.length;
922
+ if (s.closed || (s.delivered && hasBytes)) return this.send(key, payload); // refused there, before anything changes
923
+ return this.#serial(key, async () => {
924
+ // refused there, before anything changes: closed, or locked
925
+ if (s.closed || (s.delivered && hasBytes)) return this.send(key, payload);
926
+ // nothing is stored before the commit: the worker works on what the Send would store, every limit is checked on it first (a refusal
927
+ // here changes nothing) and again at the commit
928
+ let prepared = null;
929
+ let html = {};
930
+ for (let attempt = 0; attempt < 2; attempt++) {
931
+ const queued = !!s.delivered;
932
+ const plan = this.#sentPlan(s, payload);
933
+ this.#sendLimits(s, plan, payload);
934
+ if (s.format === "html") this.#htmlSendChecks(s, payload);
935
+ const job = this.#sendJob(s, payload, plan);
936
+ // an HTML file that is not queued: the person's text edits are written through the write check, and every anchor is made on the
937
+ // version it was made on, then moved onto the bytes this Send writes (#htmlSendWork)
938
+ if (s.format === "html" && !s.delivered) ({ prepared, html } = await this.#htmlSendWork(s, payload, job));
939
+ else prepared = job ? await this.#offThread(s.format, job) : null;
940
+ if (!!s.delivered === queued) break; // a run started or ended while it worked: plan again once, for the other case
941
+ }
942
+ return this.send(key, { ...payload, ...html, prepared });
943
+ });
944
+ }
945
+
946
+ /** An HTML Send's own refusals, before any work: no bytes from the browser ever (PLAN-html 3.1), no text edits during a run (D5). */
947
+ #htmlSendChecks(s, payload) {
948
+ if (payload.bytes?.length && !payload.htmlChecked) throw new HttpError(400, "an HTML file's bytes never come from the page; send its text edits", { code: "no_bytes_for_html" });
949
+ if (payload.htmlEdits?.length && s.delivered) throw new HttpError(423, s.delivered.local ? "the document is locked while a change is made on this Mac" : "the document is locked while the agent works", { code: "locked" });
950
+ }
951
+
952
+ /**
953
+ * The work of an HTML Send that is not queued, off this thread (the parser exists only in workers):
954
+ * 1. the person's text edits (R9), when there are any: on the version the page loaded, which must be the file on disk (else 409
955
+ * version_changed: the page reloads the view and places its edits again), applied by the write check in a worker (src/html-write.js
956
+ * checkedHtmlWrite "edits": one write, the proof, the inventory). Its bytes become the file at the commit (#takePage), which checks
957
+ * the disk hash again after these awaits (R12).
958
+ * 2. the anchors: a page anchor names a node of ONE version (`location.version`, a node id means nothing in another). An anchor made on
959
+ * the version this Send writes is built there; one made on another version (a draft that outlived a reload, the typed edits'
960
+ * places when this Send changes the bytes) is built on that version first (the file on disk or a stored snapshot), then moved onto
961
+ * the bytes this Send writes (reanchor), as a queued comment is moved onto the agent's file at done.
962
+ * Returns {prepared (what #offThread returns, keyed by this Send's own tasks), html: {bytes, htmlChecked} or {}}.
963
+ */
964
+ async #htmlSendWork(s, payload, job) {
965
+ let disk;
966
+ try {
967
+ checkRegularOwned(s.file);
968
+ disk = readCapped(s.file);
969
+ } catch (err) {
970
+ throw new HttpError(409, `cannot read the file: ${err.message}`, { code: "gone" });
971
+ }
972
+ const diskSha = sha256(disk);
973
+ let target = { bytes: disk, sha: diskSha };
974
+ let html = {};
975
+ const edits = cleanHtmlEdits(payload.htmlEdits);
976
+ if (edits.length) {
977
+ if (payload.baseSha !== diskSha || edits.some((e) => e.versionSha !== diskSha)) throw new HttpError(409, VERSION_CHANGED, { code: "version_changed", diskSha });
978
+ const c = await htmlWriteOffThread({ before: disk, edits: edits.map((e) => ({ nodeId: e.nodeId, text: e.newText, baseTextHash: e.baseTextHash })) });
979
+ if (!c.ok) throw htmlEditRefusal(c);
980
+ const bytes = Buffer.from(c.bytes.buffer, c.bytes.byteOffset, c.bytes.byteLength);
981
+ target = { bytes, sha: sha256(bytes) };
982
+ html = { bytes, htmlChecked: { from: diskSha, to: target.sha } };
983
+ }
984
+ const prepared = await this.#htmlAnchorWork(s, job.tasks, target, { disk, diskSha, check: true, outline: job.outline });
985
+ return { prepared, html };
986
+ }
987
+
988
+ /**
989
+ * `tasks` (build, reanchor and view tasks of one request) on the version `target` ({bytes, sha}), in the anchor worker, with every build
990
+ * task whose page anchor was made on another version (`location.version`) built on that version first (`known.disk` when it is the file
991
+ * on disk, else a stored snapshot) and moved onto `target`. One made on a version no longer stored is moved as a weak anchor (its quote
992
+ * searched). Returns what #offThread returns, its map keyed by the tasks given.
993
+ */
994
+ async #htmlAnchorWork(s, tasks, target, { disk = null, diskSha = null, check = false, outline = false } = {}) {
995
+ const elsewhere = new Map(); // version -> [task index]
996
+ tasks.forEach((t, i) => {
997
+ const v = t.op === "build" ? t.anchor?.location?.version : null;
998
+ if (v && v !== target.sha) (elsewhere.get(v) || elsewhere.set(v, []).get(v)).push(i);
999
+ });
1000
+ const moved = new Map(); // task index -> the durable made on its own version (null: not there; undefined: not worked out)
1001
+ for (const [v, list] of elsewhere) {
1002
+ const source = v === diskSha && disk ? { bytes: disk } : s.snapshots.has(v) ? { path: s.snapshots.blob(v), sha: v } : null;
1003
+ if (!source) continue;
1004
+ const r = await this.#offThread("html", { source, tasks: list.map((i) => tasks[i]) });
1005
+ if (r.ok && r.read?.ok) list.forEach((i) => moved.set(i, r.map.get(taskKey(tasks[i]))));
1006
+ }
1007
+ const asked = tasks.map((t, i) => {
1008
+ if (!elsewhere.size || !(t.op === "build" && t.anchor?.location?.version && t.anchor.location.version !== target.sha)) return t;
1009
+ const d = moved.get(i);
1010
+ return { op: "reanchor", c: { durable: usable(d) && d ? d : weakDurable("html", t.anchor), anchor: null } };
1011
+ });
1012
+ const r = await this.#offThread("html", { source: { bytes: target.bytes }, check, tasks: asked, outline });
1013
+ if (!r.ok) return r;
1014
+ const map = new Map();
1015
+ asked.forEach((t, i) => map.set(taskKey(tasks[i]), r.map.get(taskKey(t))));
1016
+ return { ...r, sha: target.sha, map };
1017
+ }
1018
+
1019
+ /** What a Send asks of the anchor worker (the same tasks #send makes, for the list it would store), or null when it needs nothing read. */
1020
+ #sendJob(s, payload, plan) {
1021
+ if (s.delivered) {
1022
+ const need = this.#delivers(plan.comments, plan).filter((c) => !c.durable);
1023
+ if (!need.length || !s.snapshots.has(s.delivered.preSha)) return null;
1024
+ return { source: { path: s.snapshots.blob(s.delivered.preSha), sha: s.delivered.preSha }, tasks: need.map(buildTask) };
1025
+ }
1026
+ const { tasks } = this.#sendTasks(s, this.#editsToTake(s, payload.edits), payload.viewState, plan);
1027
+ const bytes = payload.bytes?.length ? payload.bytes : null;
1028
+ return { source: bytes ? { bytes } : { path: s.file }, check: true, tasks, outline: this.#wantsOutline(s) };
1029
+ }
1030
+
1031
+ /** The tasks of a Send that is not queued: every comment that goes out (drafts, and those of a batch still waiting), the typed edits without an anchor yet, the view. */
1032
+ #sendTasks(s, newEdits, viewState, plan = { comments: s.comments }) {
1033
+ const go = new Set(this.#delivers(plan.comments, plan));
1034
+ const eligible = plan.comments.filter((c) => go.has(c) || c.status === "sent");
1035
+ const edits = [...s.edits, ...newEdits].filter((e) => !e.durable);
1036
+ const vs = cleanView(viewState);
1037
+ return { eligible, edits, vs, tasks: [...eligible.map(buildTask), ...edits.map((e) => ({ op: "build", anchor: editAnchor(e), pin: false })), ...(vs ? [{ op: "view", vs }] : [])] };
1038
+ }
1039
+
1040
+ /**
1041
+ * The commit of a Send: the Send's own list and note are saved first (#sentPlan), then every draft goes out with the note. A batch still
1042
+ * waiting for an agent (not delivered) is replaced by one that carries its comments and its note, then this one's; during a run the
1043
+ * drafts and the note are queued after those already queued. Every 413 (#sendLimits) is refused before anything changes.
1044
+ */
1045
+ #send(key, payload) {
379
1046
  const s = this.get(key);
380
1047
  if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
1048
+ if (s.format === "html") this.#htmlSendChecks(s, payload);
381
1049
  const hasBytes = !!payload.bytes?.length;
382
1050
  if (s.delivered && hasBytes) throw new HttpError(423, s.delivered.local ? "the document is locked while a change is made on this Mac" : "the document is locked while the agent works", { code: "locked" }); // before touching any state
383
- this.setDraft(key, { comments: payload.comments ?? s.comments, note: payload.note ?? s.note });
1051
+ // the Send's own list and note (#sentPlan), every limit checked on them before anything changes; then stored as a draft save is, so a
1052
+ // refusal after this (the file changed, could not be read in time) leaves them as editable drafts
1053
+ const plan = this.#sentPlan(s, payload);
1054
+ this.#sendLimits(s, plan, payload);
1055
+ this.#commitDraft(s, plan, payload.rev, payload.tab, true);
1056
+ try {
1057
+ return this.#sendCommit(s, payload, plan);
1058
+ } catch (err) {
1059
+ this.emit("change", s, "draft"); // refused after the save: the page's tabs see the drafts it stored (a Send that commits says so once, at the end)
1060
+ throw err;
1061
+ }
1062
+ }
1063
+
1064
+ /** The rest of #send, once its list and note are stored: the batch (or the queue) and the commit. */
1065
+ #sendCommit(s, payload, plan) {
384
1066
  if (s.delivered) {
385
1067
  // queue: comments and note only. A new comment gets its durable anchor from the bytes the page shows (the pre-agent snapshot),
386
1068
  // never from the disk, which the agent may already have changed; done() then moves it onto the post-agent file.
387
- const fresh = s.comments.filter((c) => c.status === "draft");
388
- if (fresh.some((c) => !c.durable)) {
389
- let units = null;
390
- try {
391
- units = readUnits(s.format, s.snapshots.read(s.delivered.preSha));
392
- } catch {
393
- units = null; // no snapshot: done() falls back to building from the post-agent file
394
- }
395
- if (units) {
396
- for (const c of fresh) {
397
- if (c.durable) continue;
398
- c.durable = buildDurable(units, c.anchor);
399
- if (units.format === "pptx" && c.kind === "pin" && c.durable?.shapeId != null) c.durable.pinFallback = buildDurable(units, { location: { slideIndex: c.durable.slideIndex, slideId: c.durable.slideId } });
400
- }
401
- }
1069
+ const queuedNote = joinNotes(s.queuedNote, s.note); // before anything changes
1070
+ const fresh = this.#delivers(s.comments, plan);
1071
+ const need = fresh.filter((c) => !c.durable);
1072
+ if (need.length) {
1073
+ // no snapshot, or work that did not finish: the anchor stays unset, and done() builds it from the post-agent file
1074
+ const work = this.#results(s.format, () => s.snapshots.read(s.delivered.preSha), s.delivered.preSha, need.map(buildTask), payload.prepared);
1075
+ if (work.read?.ok) need.forEach((c, i) => (c.durable = usable(work.at(i)) ? work.at(i) : null));
402
1076
  }
403
1077
  for (const c of fresh) c.status = "queued";
404
- if (s.note) {
405
- // the note belongs to the queued batch now; the stored note is cleared so the next Send does not repeat it
406
- s.queuedNote = [s.queuedNote, s.note].filter(Boolean).join("\n\n");
407
- s.note = "";
408
- }
1078
+ // the note belongs to the queued batch now; it leaves the box so the next Send does not repeat it
1079
+ s.queuedNote = queuedNote;
1080
+ s.note = "";
409
1081
  if (fresh.length || s.queuedNote) s.queued = true; // a Send with nothing new queues nothing (no empty batch after this one)
1082
+ this.#sentRev(s, payload, true);
410
1083
  this.#changed(s);
411
1084
  this.#voice(s, { sent: fresh });
412
1085
  return s;
413
1086
  }
414
- const { diskSha, written, writtenSha, writeInfo, lastUpdate } = this.#takePage(s, payload);
1087
+ // a batch still waiting for an agent is replaced: its comments go again (they are "sent"), and so does its note, first
1088
+ const waiting = s.pending && !readOnlyKind(s.pending) ? s.pending.instructions?.overall_note || "" : "";
1089
+ const note = joinNotes(waiting, s.note); // before anything is written
1090
+ // the bytes that become the file are read whole before anything is written (one past a reader's cap is refused: CapError, nothing
1091
+ // changes), and every anchor of the batch is built from them, by the anchor worker (sendAsync) or here
1092
+ const newEdits = this.#editsToTake(s, payload.edits);
1093
+ const { eligible, edits, vs, tasks } = this.#sendTasks(s, newEdits, payload.viewState, { ...plan, comments: s.comments });
1094
+ // what is stored must fit its file once cut (src/retain.js SESSION_MAX): checked before the file is written or anything changes
1095
+ if (!this.#fits(s, { comments: s.comments, edits: [...s.edits, ...newEdits] })) throw tooLarge("This Send");
1096
+ let work = null;
1097
+ const { diskSha, writtenSha, writeInfo, lastUpdate } = this.#takePage(s, payload, (b, sha) => {
1098
+ work = this.#results(s.format, b, sha, tasks, payload.prepared, { check: true });
1099
+ this.#readVerdict(work, s.format, sha, "refusing to write");
1100
+ });
415
1101
 
416
1102
  // user edits since the last Send (document-derived text) accumulate until a batch is delivered
417
- for (const e of Array.isArray(payload.edits) ? payload.edits.slice(0, 2000) : []) s.edits.push({ kind: e.kind, before: e.before, after: e.after, label: e.label, location: e.location });
418
-
419
- const units = readUnits(s.format, written);
420
- const resolve = (anchor) => buildDurable(units, anchor);
421
- const eligible = s.comments.filter((c) => c.status === "draft" || c.status === "sent");
422
- for (const c of eligible) {
423
- c.durable = resolve(c.anchor);
424
- // a PPTX pin dropped on a shape also remembers its slide: if the shape is deleted the pin stays on the slide
425
- if (units.format === "pptx" && c.kind === "pin" && c.durable?.shapeId != null) c.durable.pinFallback = buildDurable(units, { location: { slideIndex: c.durable.slideIndex, slideId: c.durable.slideId } });
426
- }
427
- for (const e of s.edits) e.durable = e.durable || resolve({ location: e.location, quote: e.after });
428
- s.view = this.#resolveView(s, units, payload.viewState);
429
-
430
- const batch = buildBatch({ file: s.file, format: s.format, baseSha: writtenSha, note: s.note, comments: eligible, edits: s.edits, style: styleNote(), guide: voiceGuide() });
1103
+ s.edits.push(...newEdits);
1104
+ // a comment whose anchor work did not finish goes out weaker (src/anchor-work.js weakDurable); a PPTX pin on a shape also remembers its slide
1105
+ eligible.forEach((c, i) => (c.durable = usable(work.at(i)) ? work.at(i) : weakDurable(s.format, c.anchor)));
1106
+ edits.forEach((e, i) => (e.durable = usable(work.at(eligible.length + i)) ? work.at(eligible.length + i) : weakDurable(s.format, editAnchor(e))));
1107
+ const v = vs ? work.at(eligible.length + edits.length) : null;
1108
+ s.view = !vs ? null : usable(v) ? v : { format: s.format, raw: vs };
1109
+ if (work.outline) this.#outline = { key: s.key, sha: writtenSha, all: work.outline };
1110
+
1111
+ const batch = buildBatch({ file: s.file, format: s.format, baseSha: writtenSha, note, comments: eligible, edits: s.edits, style: styleNote(), guide: voiceGuide() });
431
1112
  const sent = eligible.filter((c) => c.status === "draft");
432
1113
  for (const c of eligible) {
433
1114
  c.status = "sent";
@@ -444,6 +1125,7 @@ export class Sessions extends EventEmitter {
444
1125
  this.#deliverIfPolled(s);
445
1126
  delete s.quickConfirm;
446
1127
  delete s.skipLocalNote; // it held for this Send only (the decision for its batch is made)
1128
+ this.#sentRev(s, payload, true);
447
1129
  this.#changed(s);
448
1130
  this.#voice(s, { last: lastUpdate, diskSha, writtenSha, sent });
449
1131
  if (this.state(s) === "stranded" && !s.pollers.length) this.emit("stranded", s); // nobody took it: the server may start its background agent
@@ -453,8 +1135,8 @@ export class Sessions extends EventEmitter {
453
1135
  /**
454
1136
  * The page's bytes become the file, as at Send (and at a quick fix's Use): the disk read, the compare-and-swap against the version
455
1137
  * the page loaded, the package check, the guarded replace, the "sent" snapshot, the new base and the person's own history update.
456
- * `check(bytes)` (optional) sees the bytes that are about to be the file, after the package check and before anything is written, and
457
- * may throw to refuse. Returns {disk, diskSha, written, writtenSha, writeInfo, lastUpdate}.
1138
+ * `check(bytes, sha)` (optional) sees the bytes that are about to be the file and their sha256, after the package check and before
1139
+ * anything is written, and may throw to refuse. Returns {disk, diskSha, written, writtenSha, writeInfo, lastUpdate}.
458
1140
  */
459
1141
  #takePage(s, payload, check = null) {
460
1142
  const hasBytes = !!payload.bytes?.length;
@@ -466,25 +1148,33 @@ export class Sessions extends EventEmitter {
466
1148
  throw new HttpError(409, `cannot read the file: ${err.message}`, { code: "gone" });
467
1149
  }
468
1150
  const diskSha = sha256(disk);
469
- if (diskSha !== payload.baseSha) {
470
- throw this.#outsideChange(s, { side: "user", bytes: payload.bytes, diskSha, baseSha: payload.baseSha });
1151
+ // an HTML file's text edits were applied by its write check (#htmlSendWork) to the version the page loaded: no bytes come from the page
1152
+ const htmlChecked = s.format === "html" ? payload.htmlChecked ?? null : null;
1153
+ if (diskSha !== payload.baseSha || (htmlChecked && htmlChecked.from !== diskSha)) {
1154
+ // an HTML page sent no bytes: nothing of its to keep as a copy (its edits stay in the page, which reloads and places them again)
1155
+ throw this.#outsideChange(s, { side: "user", bytes: s.format === "html" ? null : payload.bytes, diskSha, baseSha: payload.baseSha });
471
1156
  }
472
1157
  let written = disk;
473
1158
  let writeInfo = null;
474
1159
  if (hasBytes) {
475
- if (payload.bytes.length > MAX_BYTES) throw new HttpError(413, "document too large");
476
- try {
477
- validatePackage(payload.bytes, s.format);
478
- } catch (err) {
479
- throw new HttpError(422, `refusing to write: ${err.message}`, { code: "invalid_package" });
1160
+ if (payload.bytes.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
1161
+ if (s.format === "html") {
1162
+ if (htmlChecked?.to !== sha256(payload.bytes)) throw new HttpError(500, "an HTML file is written only after its write check", { code: "unchecked_write" });
1163
+ } else {
1164
+ try {
1165
+ validatePackage(payload.bytes, s.format);
1166
+ } catch (err) {
1167
+ throw new HttpError(422, `refusing to write: ${err.message}`, { code: "invalid_package" });
1168
+ }
480
1169
  }
481
1170
  }
482
- if (check) check(hasBytes ? payload.bytes : disk);
1171
+ const writtenSha = hasBytes ? sha256(payload.bytes) : diskSha;
1172
+ if (htmlChecked && hasBytes) readInCaps("html", writtenSha); // the write check read these bytes whole within the limits
1173
+ if (check) check(hasBytes ? payload.bytes : disk, writtenSha);
483
1174
  if (hasBytes) {
484
- writeInfo = this.#replace(s, payload.bytes, { expectSha: diskSha, label: "before send", side: "user" });
1175
+ writeInfo = this.#replace(s, payload.bytes, { expectSha: diskSha, label: "before send", side: "user", htmlChecked });
485
1176
  written = payload.bytes;
486
1177
  }
487
- const writtenSha = sha256(written);
488
1178
  s.snapshots.add(written, "sent");
489
1179
  s.baseSha = writtenSha;
490
1180
  const lastUpdate = s.history?.at(-1); // learning pairs an agent run with the person's edits right after it
@@ -508,16 +1198,21 @@ export class Sessions extends EventEmitter {
508
1198
  * (DOCX paragraphs, formatting and moves: src/merge.js; XLSX cells: src/merge-xlsx.js; PPTX shape text: src/merge-pptx.js) is carried onto it as ops (same engine, expect, protected and package checks
509
1199
  * as an agent's apply) and written as the user's own update. Refused while a batch is delivered (423): the page waits and asks
510
1200
  * again. Returns {merged, unplaced, sha, same, mineSha, copy_name}.
511
- * Ceiling: the engine runs in the server process (about 0.1 s on a 300-paragraph file); move it to a worker if large files stall polls.
1201
+ * The reading of the three versions within the readers' caps, the plan and the engine's run happen in a worker thread with a time and
1202
+ * memory limit (src/offload.js mergeOffThread): the server's thread stays free whatever the document. A merge that fails or runs out of
1203
+ * time writes nothing: the person's version is saved as a copy next to the file and the page keeps their edits (503 merge_failed).
1204
+ * One merge per session at a time (another is 423 locked; the page has only one in flight).
512
1205
  */
513
1206
  async mergeUserEdits(key, { baseSha, bytes, edits = [] }) {
514
1207
  const s = this.get(key);
515
1208
  if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
516
1209
  if (s.delivered) throw new HttpError(423, s.delivered.local ? "a change is being made on this Mac" : "the agent is working", { code: "locked" });
517
- const merger = MERGERS[s.format];
1210
+ if (s.merging) throw new HttpError(423, "your edits are being merged already", { code: "locked" });
1211
+ // an HTML view is read only during a run (PLAN-html D5): nothing is typed to merge, and its bytes never come from the page
1212
+ if (s.format === "html") throw new HttpError(400, "an HTML file's edits are never merged: its view is read only while the agent works", { code: "not_for_html" });
518
1213
  if (!/^[0-9a-f]{64}$/.test(baseSha)) throw new HttpError(400, "baseSha must be a sha256"); // it names a snapshot file
519
1214
  if (!bytes?.length) throw new HttpError(400, "no document bytes");
520
- if (bytes.length > MAX_BYTES) throw new HttpError(413, "document too large");
1215
+ if (bytes.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
521
1216
  try {
522
1217
  validatePackage(bytes, s.format);
523
1218
  } catch (err) {
@@ -539,8 +1234,19 @@ export class Sessions extends EventEmitter {
539
1234
  } catch {
540
1235
  throw new HttpError(409, "the version this page started from is no longer stored", { code: "no_base" });
541
1236
  }
542
- const [mod, { runOps }] = await Promise.all([import(merger.module), import("./ops/run.js")]);
543
- const r = await mod[merger.fn]({ base, mine: bytes, theirs: disk, name: path.basename(s.file), runOps, edits: editKinds(edits), agent: s.format === "xlsx" ? this.#writesBetween(s, baseSha, diskSha, disk) : null });
1237
+ // the three versions are read whole within the readers' caps in the worker first (the page's version, a file swapped on disk and an
1238
+ // old snapshot alike): past one, the merge is refused with the reason and nothing is written
1239
+ let r;
1240
+ s.merging = true;
1241
+ try {
1242
+ r = await mergeOffThread({ format: s.format, base, mine: bytes, theirs: disk, name: path.basename(s.file), edits: editKinds(edits), agent: s.format === "xlsx" ? this.#writesBetween(s, baseSha, diskSha, disk) : null });
1243
+ } finally {
1244
+ s.merging = false;
1245
+ }
1246
+ if (r.cap) throw new CapError(r.cap);
1247
+ if (r.invalid) throw new HttpError(422, `refusing to merge: ${untrusted(r.invalid, 200)}`, { code: "invalid_package" });
1248
+ if (!r.ok) throw this.#mergeFailed(s, bytes, r);
1249
+ if (r.bytes) r.bytes = Buffer.from(r.bytes.buffer, r.bytes.byteOffset, r.bytes.byteLength); // a Uint8Array once it crossed threads
544
1250
  if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
545
1251
  if (s.delivered) throw new HttpError(423, s.delivered.local ? "a change is being made on this Mac" : "the agent is working", { code: "locked" }); // a batch was delivered while the engine ran
546
1252
  // The page shows the first HELD_SHOWN items and drops its own version when it reloads. When the list stands for more than it
@@ -570,6 +1276,24 @@ export class Sessions extends EventEmitter {
570
1276
  return { same: false, merged: r.merged, unplaced: r.unplaced.slice(0, 50), sha, mineSha: sha256(bytes), copy_name: copy?.name ?? null };
571
1277
  }
572
1278
 
1279
+ /**
1280
+ * A merge that did not finish (out of time or memory, busy, crashed) or whose result the proof refused: nothing was written. The person's version is saved next to the
1281
+ * file (except when Onionskin was only busy) and the answer says so in words the page shows inside its own sentence ("could not be
1282
+ * merged into its version (...). Your edits are still in this page: download them before you reload.").
1283
+ */
1284
+ #mergeFailed(s, bytes, r) {
1285
+ if (r.failed === "busy") return new HttpError(503, "Onionskin is merging other files right now; nothing was written, so try again in a moment", { code: "merge_busy" });
1286
+ let copy = null;
1287
+ try {
1288
+ copy = writeSiblingCopy(s.file, bytes);
1289
+ this.createdCopies.add(copy.path);
1290
+ } catch {}
1291
+ // refused: the proof of the merge (src/merge-proof.js) could not account for every edit of the person's in the result; the reason
1292
+ // holds counts and kinds of edits, never their text
1293
+ const why = r.failed === "timeout" || r.failed === "memory" ? r.why : r.refused ? `the merged file could not be checked to hold every one of your edits (${r.refused})` : "the merge stopped with an error";
1294
+ return new HttpError(503, `${why}; nothing was written${copy ? `, and a copy of your version was saved as ${copy.name} next to the file` : ""}`, { code: "merge_failed", copy_name: copy?.name ?? null });
1295
+ }
1296
+
573
1297
  #logWrite(s, entry) {
574
1298
  const list = this.writes.get(s.key) || [];
575
1299
  list.push(entry);
@@ -610,7 +1334,9 @@ export class Sessions extends EventEmitter {
610
1334
  * replacement (see replaceGuarded). `expectSha` is the hash the caller compared against; a file that is different now, or becomes
611
1335
  * different just before the rename, is not overwritten (the newer bytes are snapshotted) and the caller gets the 409 conflict.
612
1336
  */
613
- #replace(s, bytes, { expectSha, label, side = null }) {
1337
+ #replace(s, bytes, { expectSha, label, side = null, htmlChecked = null }) {
1338
+ // R1, R12: an HTML file is written only with bytes its write check passed in a worker (agentWriteChecked); every other path refuses
1339
+ if (s.format === "html" && htmlChecked?.to !== sha256(bytes)) throw new HttpError(500, "an HTML file is written only after its write check", { code: "unchecked_write" });
614
1340
  try {
615
1341
  return replaceGuarded(s.file, bytes, {
616
1342
  expectSha,
@@ -684,14 +1410,6 @@ export class Sessions extends EventEmitter {
684
1410
  return { stale: !s.delivered && !!loadedSha && diskSha !== loadedSha, diskSha };
685
1411
  }
686
1412
 
687
- #resolveView(s, units, vs) {
688
- if (!vs) return null;
689
- if (units.format === "docx" && vs.format !== "docx") return { format: s.format, raw: vs };
690
- if (units.format !== "docx") return viewDurable(units, vs) || { format: s.format, raw: vs };
691
- const d = durableFromEngine(units.paragraphs, { blockPath: vs.blockPath }, { quote: vs.quote });
692
- return { format: "docx", durable: d, offsetPx: Number(vs.offsetPx) || 0, zoom: vs.zoom ?? "fit", fraction: vs.fraction };
693
- }
694
-
695
1413
  // ---- poll / delivery -------------------------------------------------------------------------------------
696
1414
  /**
697
1415
  * Registers a poller. Returns a handle {promise, cancel}. Resolves with the batch object, or {status:'superseded'|'closed'}.
@@ -776,17 +1494,30 @@ export class Sessions extends EventEmitter {
776
1494
  * written to <state>/agents/<key>/outline/outline-<batch id>.jsonl (0600, atomic, one unit per line), the only folder that agent may read,
777
1495
  * and the batch names it. Older outline files of the session go at each such delivery, and with the folder on Close.
778
1496
  * Any other holder, a review (it has `units`) and a voice request (it quotes only the sources the person ticked) get the batch as stored.
779
- * A file that cannot be read gets no outline (the agent declines what it cannot find); an outline file that cannot be written leaves the
780
- * inline part with `outline_more`. Ceiling: read and parsed on the main thread at each such delivery, as a review is; upgrade path as in unitsOf.
1497
+ * A file that cannot be read gets no outline, and `outline_unavailable` says why (the agent declines what it cannot find); an outline file that cannot be written leaves the
1498
+ * inline part with `outline_more`. The outline is built in the anchor worker at Send, at done and before a poll's delivery (outlineAhead),
1499
+ * and found here by the file's hash. Ceiling: a delivery that finds none (a synchronous caller, or the file changed in between) reads and
1500
+ * parses the file on this thread, as a review does; upgrade path as in unitsOf.
781
1501
  */
782
1502
  #withOutline(s, batch, holder) {
783
1503
  if (!isAgentHolder(holder) || batch.instructions?.review || batch.instructions?.voice_request) return batch;
784
1504
  let all;
785
1505
  try {
786
1506
  checkRegularOwned(s.file);
787
- all = outlineOf(s.format, readCapped(s.file), OUTLINE_FILE_MAX);
788
- } catch {
789
- return batch;
1507
+ const bytes = readCapped(s.file);
1508
+ const sha = sha256(bytes);
1509
+ // the same version's outline is built once (an agent's polls and a Send's delivery read the same file), on the memo's reading of it
1510
+ if (this.#outline?.key === s.key && this.#outline.sha === sha) all = this.#outline.all;
1511
+ // an HTML file is never parsed on this thread: its outline comes from the anchor worker (Send, done, outlineAhead) or not at all
1512
+ else if (s.format === "html") return { ...batch, untrusted_document_evidence: { ...batch.untrusted_document_evidence, outline_unavailable: "Onionskin did not finish reading this page's outline in time. Use the anchors and their quotes; poll again for an outline." } };
1513
+ else {
1514
+ all = outlineOf(s.format, bytes, OUTLINE_FILE_MAX, unitsFor(s.format, bytes, sha));
1515
+ this.#outline = { key: s.key, sha, all };
1516
+ }
1517
+ } catch (err) {
1518
+ // the agent is told why it has no outline (a file changed past a reader's cap behind Onionskin, or gone), in the reader's plain words
1519
+ const why = err instanceof CapError ? `Onionskin cannot read this file: ${err.message}.` : "Onionskin cannot read this file as it is now.";
1520
+ return { ...batch, untrusted_document_evidence: { ...batch.untrusted_document_evidence, outline_unavailable: why } };
790
1521
  }
791
1522
  const inline = withOutline(batch, all, VOICE_BATCH_MAX_CHARS, { inlineMax: UNITS_MAX });
792
1523
  const name = `outline-${batch.batch_id}.jsonl`;
@@ -848,7 +1579,7 @@ export class Sessions extends EventEmitter {
848
1579
  #queuedToDrafts(s) {
849
1580
  if (s.queued) {
850
1581
  for (const c of s.comments) if (c.status === "queued") c.status = "draft";
851
- s.note = [s.queuedNote, s.note].filter(Boolean).join("\n\n");
1582
+ this.#giveNote(s, [s.queuedNote, s.note].filter(Boolean).join("\n\n"));
852
1583
  s.queuedNote = "";
853
1584
  }
854
1585
  s.queued = false;
@@ -943,7 +1674,16 @@ export class Sessions extends EventEmitter {
943
1674
 
944
1675
  /** Called on a timer: a delivered batch whose lease expired goes back to pending (stranded) and the document unlocks. */
945
1676
  tick() {
946
- for (const s of this.map.values()) if (s.delivered && s.delivered.lease.expiresAt <= this.now()) this.#expire(s);
1677
+ const now = this.now();
1678
+ const checkMax = Math.max(LEASE_MS(), 5 * 60_000);
1679
+ for (const s of this.map.values()) {
1680
+ if (!s.delivered || s.delivered.lease.expiresAt > now) continue;
1681
+ if (s.delivered.stage === "checking") {
1682
+ const since = s.delivered.checkingAt ?? s.delivered.lease.expiresAt;
1683
+ if (now - since < checkMax) continue;
1684
+ }
1685
+ this.#expire(s);
1686
+ }
947
1687
  }
948
1688
 
949
1689
  /**
@@ -979,6 +1719,7 @@ export class Sessions extends EventEmitter {
979
1719
  const s = this.get(key);
980
1720
  if (!s.delivered || s.delivered.stage === stage) return;
981
1721
  s.delivered.stage = stage;
1722
+ s.delivered.checkingAt = stage === "checking" ? this.now() : null;
982
1723
  this.#changed(s);
983
1724
  }
984
1725
 
@@ -999,11 +1740,22 @@ export class Sessions extends EventEmitter {
999
1740
  if (token != null && token !== d.lease.token) throw stale();
1000
1741
  }
1001
1742
 
1743
+ /**
1744
+ * The delivered batch, for its lease holder only (`GET /cli/batch` with a claim): holder, batch id and token are all required and checked
1745
+ * before anything of the batch is handed out, so `reply` and `apply` read no anchors, comments or file text for a delivery that is not theirs.
1746
+ */
1747
+ deliveredBatch(key, claim) {
1748
+ const s = this.get(key);
1749
+ if (!s.delivered) throw new HttpError(409, "no batch is delivered for this file, so the ticket is void", { code: "no_batch" });
1750
+ this.#assertLease(s, { ...claim, strict: true });
1751
+ return s.delivered.batch;
1752
+ }
1753
+
1002
1754
  /**
1003
1755
  * `apply` through a live session: the single authority for the write. Accepted only while a batch is delivered (state working or
1004
1756
  * queued: both hold a live lease) and only for the lease holder; compare-and-swap on the disk hash; renews the lease.
1005
1757
  */
1006
- agentWrite(key, { baseSha, bytes, holder, token, batchId, label = "before agent write", route = "commit", ops = null, partial = false }) {
1758
+ agentWrite(key, { baseSha, bytes, holder, token, batchId, label = "before agent write", route = "commit", ops = null, partial = false, htmlChecked = null }) {
1007
1759
  const s = this.get(key);
1008
1760
  if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
1009
1761
  const state = this.state(s);
@@ -1014,11 +1766,20 @@ export class Sessions extends EventEmitter {
1014
1766
  // `commit` does not know the hash its candidate was edited from: the base is the last hash this server wrote or served for the delivery
1015
1767
  if (baseSha == null) baseSha = s.delivered.knownSha ?? s.delivered.preSha;
1016
1768
  if (!bytes?.length) throw new HttpError(400, "no bytes");
1017
- if (bytes.length > MAX_BYTES) throw new HttpError(413, "document too large");
1018
- try {
1019
- validatePackage(bytes, s.format);
1020
- } catch (err) {
1021
- throw new HttpError(422, `refusing to write: ${err.message}`, { code: "invalid_package" });
1769
+ if (bytes.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
1770
+ const newSha = sha256(bytes);
1771
+ if (s.format === "html") {
1772
+ // checked in a worker (agentWriteChecked) for exactly these bytes; the disk hash it was checked against is compared below
1773
+ if (htmlChecked?.to !== newSha) throw new HttpError(500, "an HTML file is written only after its write check", { code: "unchecked_write" });
1774
+ } else {
1775
+ try {
1776
+ validatePackage(bytes, s.format);
1777
+ } catch (err) {
1778
+ throw new HttpError(422, `refusing to write: ${err.message}`, { code: "invalid_package" });
1779
+ }
1780
+ // read whole before anything is written, as a Send's bytes are: bytes past a reader's cap are refused with the reason (CapError, 422
1781
+ // too_large), never written to leave the next outline, review and done without a reading. The reading is kept for them (unitsFor).
1782
+ readWhole(s.format, bytes, newSha);
1022
1783
  }
1023
1784
  let disk;
1024
1785
  try {
@@ -1029,9 +1790,10 @@ export class Sessions extends EventEmitter {
1029
1790
  }
1030
1791
  const diskSha = sha256(disk);
1031
1792
  if (diskSha !== baseSha) throw this.#outsideChange(s, { side: "agent", bytes, diskSha, baseSha, message: "the file changed on disk since apply read it" });
1793
+ // R12: the compare-and-swap again after the check's await: the version it checked against must still be the one on disk
1794
+ if (s.format === "html" && htmlChecked.from !== diskSha) throw this.#outsideChange(s, { side: "agent", bytes, diskSha, baseSha: htmlChecked.from, message: "the file changed on disk while the write was being checked" });
1032
1795
  this.setStage(key, "writing"); // only once the checks passed: a refused write does not claim to be writing
1033
- const info = this.#replace(s, bytes, { expectSha: diskSha, label, side: "agent" });
1034
- const newSha = sha256(bytes);
1796
+ const info = this.#replace(s, bytes, { expectSha: diskSha, label, side: "agent", htmlChecked });
1035
1797
  // what the merge may rely on later: the ops `apply` says it ran (checked against the bytes when a merge needs them), or that it was a commit
1036
1798
  const checked = route === "apply" && s.format === "xlsx" && Array.isArray(ops) && validateOps({ schema: "onionskin.ops/1", ops }).ok; // only the XLSX merge reads them
1037
1799
  this.#logWrite(s, checked ? { from: diskSha, to: newSha, route, ops, partial: partial === true, anchors: s.delivered.batch.untrusted_document_evidence?.anchors || [] } : { from: diskSha, to: newSha, route: route === "apply" ? "apply" : "commit", ops: null });
@@ -1043,6 +1805,41 @@ export class Sessions extends EventEmitter {
1043
1805
  return { ok: true, sha256: newSha, lease_expires_at: s.delivered.lease.expiresAt };
1044
1806
  }
1045
1807
 
1808
+ /**
1809
+ * The agent's write (`/cli/agent-write`, `/cli/commit`) for every format. An HTML file's bytes first go through the write check
1810
+ * (src/html-write.js checkedHtmlWrite: decoding, limits, a parse, the charset rule, the inventory of active content) in a worker; the
1811
+ * bytes it returns (the charset declaration may be added) are then written by agentWrite, which checks the lease, the state and the
1812
+ * disk hash again after that await (R12). Other formats: agentWrite as it is.
1813
+ */
1814
+ async agentWriteChecked(key, args, check = htmlWriteOffThread) {
1815
+ const s = this.get(key);
1816
+ if (s.format !== "html") return this.agentWrite(key, args);
1817
+ // the cheap refusals first, so a write that cannot land costs no worker; agentWrite makes every one of them again after the await
1818
+ if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
1819
+ if (!s.delivered) throw new HttpError(409, `no batch is delivered to an agent (state: ${this.state(s)}); run poll and wait for the user's Send`, { code: "not_working", state: this.state(s) });
1820
+ this.#assertLease(s, { holder: args.holder, token: args.token, batchId: args.batchId, strict: true });
1821
+ if (!args.bytes?.length) throw new HttpError(400, "no bytes");
1822
+ let disk;
1823
+ try {
1824
+ checkRegularOwned(s.file);
1825
+ disk = readCapped(s.file);
1826
+ } catch (err) {
1827
+ throw new HttpError(409, `cannot read the file: ${err.message}`, { code: "gone" });
1828
+ }
1829
+ // the bytes checked are the ones whose hash agentWrite compares with the base and the disk after this await (from === disk === base)
1830
+ const c = await check({ before: disk, after: args.bytes });
1831
+ if (!c.ok) {
1832
+ // a refused write is part of the run's result: the page lists it with the run (the R1 sentence for active content), nothing was written
1833
+ if (c.code !== "check_failed" && s.delivered && !s.closed) {
1834
+ const p = cleanProblem({ key: `write:${c.code}`, op: args.route === "apply" ? "apply" : "commit", status: "refused", code: c.code, reason: c.reason });
1835
+ s.delivered.applyProblems = [...(s.delivered.applyProblems || []).filter((x) => x.key !== p.key), p].slice(0, 40);
1836
+ this.#changed(s);
1837
+ }
1838
+ throw new HttpError(c.code === "check_failed" ? 503 : 422, c.reason, { code: c.code, ...(c.reasons ? { reasons: c.reasons } : {}) });
1839
+ }
1840
+ return this.agentWrite(key, { ...args, bytes: c.bytes, htmlChecked: { from: sha256(disk), to: sha256(c.bytes) } });
1841
+ }
1842
+
1046
1843
  /**
1047
1844
  * What `apply` could not do (refused by the engine, protected, anchor or expectation mismatch), told by the lease holder after each run
1048
1845
  * (also when nothing was written). Shown to the user with the activity entry at `done`.
@@ -1067,6 +1864,7 @@ export class Sessions extends EventEmitter {
1067
1864
  if (!s.delivered) throw new HttpError(409, "no batch is in progress for this file (taken back or already finished)", { code: "no_batch" });
1068
1865
  this.#assertLease(s, claim, { staleOnHolder: true });
1069
1866
  this.setStage(key, "checking");
1867
+ this.#renew(s);
1070
1868
  let post;
1071
1869
  let pre;
1072
1870
  try {
@@ -1082,7 +1880,25 @@ export class Sessions extends EventEmitter {
1082
1880
  return { pre, post, format: s.format, postSha: sha256(post), batchId: s.delivered.batch.batch_id, deferSoffice: env("SOFFICE_SYNC", "") !== "1" };
1083
1881
  }
1084
1882
 
1085
- done(key, results, claim = {}, analysis = null) {
1883
+ /**
1884
+ * done() as the server calls it (src/server.js finishRun): the anchors of the comments queued during the run (moved onto the file the
1885
+ * agent leaves), a review's cards and the outline for a waiting agent are worked out first in the anchor worker, then done() commits
1886
+ * with them and reads none of it on this thread. Work that did not finish leaves those comments weaker, with a flag; never lost.
1887
+ */
1888
+ async doneAsync(key, results, claim = {}, analysis = null, opts = {}) {
1889
+ return this.#serial(key, async () => {
1890
+ const s = this.get(key);
1891
+ const run = s.delivered;
1892
+ const tasks = [];
1893
+ if (run && s.queued) tasks.push(...s.comments.filter((c) => c.status === "queued").map(reanchorTask));
1894
+ if (run?.batch.instructions?.review && s.review?.batchId === run.batch.batch_id && results && typeof results === "object") tasks.push(cardsTask(results, s.review, run));
1895
+ const prepared = run && (tasks.length || this.#wantsOutline(s)) ? await this.#offThread(s.format, { source: { path: s.file }, tasks, outline: this.#wantsOutline(s) }) : null;
1896
+ return this.done(key, results, claim, analysis, { ...opts, prepared });
1897
+ });
1898
+ }
1899
+
1900
+ /** `inline: false` (the server): never analyze on this thread; an analysis for other bytes (the file changed while it ran) is a stated failure. */
1901
+ done(key, results, claim = {}, analysis = null, { inline = true, prepared = null } = {}) {
1086
1902
  const s = this.get(key);
1087
1903
  if (!s.delivered) throw new HttpError(409, "no batch is in progress for this file (taken back or already finished)", { code: "no_batch" });
1088
1904
  this.#assertLease(s, claim, { staleOnHolder: true });
@@ -1096,10 +1912,10 @@ export class Sessions extends EventEmitter {
1096
1912
  throw new HttpError(409, `cannot read the file: ${err.message}`);
1097
1913
  }
1098
1914
  const postSha = sha256(post);
1099
- if (readOnlyKind(run.batch) === "voice") return this.#doneVoice(s, results, run, post, postSha);
1915
+ if (readOnlyKind(run.batch) === "voice") return this.#doneVoice(s, results, run, post, postSha, prepared);
1100
1916
  // a review that wrote nothing ends with cards. One whose file changed anyway (a direct save past review_only) is finished as a run below:
1101
1917
  // its write is flagged and can be reverted, and no card is kept
1102
- if (run.batch.instructions?.review && postSha === run.preSha) return this.#doneReview(s, results, run, post);
1918
+ if (run.batch.instructions?.review && postSha === run.preSha) return this.#doneReview(s, results, run, post, prepared);
1103
1919
  s.snapshots.add(post, "after agent");
1104
1920
 
1105
1921
  // ---- plan: everything that can throw happens here, on locals; the session is not touched until the commit below ------
@@ -1113,7 +1929,14 @@ export class Sessions extends EventEmitter {
1113
1929
  let found;
1114
1930
  if (!pre) found = { flags: [{ code: "no_pre_snapshot", severity: "warn" }], checks: { node: "skipped", deep: { status: "skipped", reason: "no pre-agent snapshot" }, notice: "Checks skipped: the pre-agent version is no longer stored." }, changes: null };
1115
1931
  else if (analysis && analysis.postSha === postSha && analysis.batchId === run.batch.batch_id) found = analysis;
1116
- else found = analyze(pre, post, s.format);
1932
+ else if (!inline) {
1933
+ const why = "the file changed while it was being checked";
1934
+ found = { flags: [{ code: "checks_failed", severity: "warn", detail: why }], checks: { node: "failed", deep: { status: "skipped", reason: why }, notice: `The package checks did not finish: ${why}. The file was not verified.` }, changes: null };
1935
+ } else if (s.format === "html") {
1936
+ // an HTML file is read in workers only: a caller without the worker's analysis gets it stated as not run, never as clean
1937
+ const why = "the checks of an HTML file run in a worker";
1938
+ found = { flags: [{ code: "checks_failed", severity: "warn", detail: why }], checks: { node: "failed", deep: { status: "skipped", reason: why }, notice: `The checks did not run: ${why}. The file was not verified.` }, changes: null };
1939
+ } else found = analyze(pre, post, s.format);
1117
1940
  const flags = [...found.flags];
1118
1941
  const { checks, changes } = found;
1119
1942
  // the file must be what this server last wrote (apply/commit) or delivered; anything else changed it behind the lease (a direct save by
@@ -1154,7 +1977,8 @@ export class Sessions extends EventEmitter {
1154
1977
  // style.json changed by something else while this batch was out (an agent with a shell, say): already put back by src/style.js
1155
1978
  this.#guardFlags(run, flags);
1156
1979
  if (results.voice_draft !== undefined) flags.push({ code: "voice_draft_not_allowed", severity: "warn" }); // only a voice batch takes a draft; nothing is stored
1157
- const queuedPlan = this.#planQueued(s, post, postSha, flags);
1980
+ const queuedPlan = this.#planQueued(s, post, postSha, flags, prepared);
1981
+ if (prepared?.ok && prepared.sha === postSha && prepared.outline) this.#outline = { key: s.key, sha: postSha, all: prepared.outline };
1158
1982
  const cleanFlags = flags.map(cleanFlag);
1159
1983
  const { list: ignored, more: ignoredMore } = ignoredInstructions(results.document_instructions_ignored);
1160
1984
  const entry = {
@@ -1280,27 +2104,21 @@ export class Sessions extends EventEmitter {
1280
2104
  }
1281
2105
 
1282
2106
  /** Comments queued while the agent worked become the next pending batch, anchored on the file the agent leaves. Pure planning: touches no session state; re-anchoring problems are pushed onto `flags`. */
1283
- #planQueued(s, post, postSha, flags) {
2107
+ // `prepared`: the anchor worker's answer for the same tasks (doneAsync); without it the anchors are moved on this thread.
2108
+ #planQueued(s, post, postSha, flags, prepared = null) {
1284
2109
  let plan = null;
1285
2110
  if (s.queued) {
1286
2111
  const queued = s.comments.filter((c) => c.status === "queued");
1287
2112
  if (queued.length || s.queuedNote) {
1288
- let units = null;
1289
- try {
1290
- units = readUnits(s.format, post);
1291
- } catch (err) {
1292
- flags.push({ code: "queued_reanchor_failed", severity: "warn", detail: err.message });
1293
- }
2113
+ const work = this.#results(s.format, post, postSha, queued.map(reanchorTask), prepared);
2114
+ const read = work.read;
2115
+ if (!read?.ok) flags.push({ code: "queued_reanchor_failed", severity: "warn", detail: read?.invalid || read?.cap || "the anchors could not be worked out in time" });
1294
2116
  const durables = new Map();
1295
- for (const c of queued) {
1296
- let d = null;
1297
- try {
1298
- if (units) d = reanchor(units, c);
1299
- } catch (err) {
1300
- flags.push({ code: "queued_reanchor_failed", severity: "warn", comment_id: c.id, detail: err.message });
1301
- }
1302
- durables.set(c.id, d);
1303
- }
2117
+ queued.forEach((c, i) => {
2118
+ const r = work.at(i);
2119
+ if (read?.ok && !usable(r)) flags.push({ code: "queued_reanchor_failed", severity: "warn", comment_id: c.id, detail: r?.error ?? "the anchor could not be worked out in time" });
2120
+ durables.set(c.id, read?.ok && usable(r) ? r : weakDurable(s.format, c.anchor));
2121
+ });
1304
2122
  const batch = buildBatch({ file: s.file, format: s.format, baseSha: postSha, note: s.queuedNote, comments: queued.map((c) => ({ ...c, durable: durables.get(c.id) })), edits: [], style: styleNote(), guide: voiceGuide() });
1305
2123
  plan = { queued, durables, batch };
1306
2124
  }
@@ -1318,6 +2136,7 @@ export class Sessions extends EventEmitter {
1318
2136
  const s = this.get(key);
1319
2137
  if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
1320
2138
  if (!REVIEW_KINDS.includes(kind)) throw new HttpError(400, `kind must be one of ${REVIEW_KINDS.join(", ")}`, { code: "bad_kind" });
2139
+ if (s.format === "html") throw new HttpError(409, "Review first is not available for HTML files in this version. Comment on the page and press Send.", { code: "not_for_html" }); // PLAN-html 2a: the page hides the button
1321
2140
  if (s.delivered || s.pending) throw new HttpError(409, s.delivered?.local ? "Wait for the change on this Mac to finish, then try again." : "Wait for the agent to finish, then try again.", { code: "busy" });
1322
2141
  let disk;
1323
2142
  try {
@@ -1334,7 +2153,7 @@ export class Sessions extends EventEmitter {
1334
2153
  }
1335
2154
 
1336
2155
  #reviewBatch(s, kind, bytes, baseSha) {
1337
- const { units, more, index } = unitsOf(s.format, bytes);
2156
+ const { units, more, index } = unitsOf(s.format, bytes, unitsFor(s.format, bytes, baseSha));
1338
2157
  const batch = buildBatch({ file: s.file, format: s.format, baseSha, note: "", comments: [], edits: [], review: { kind, ask: REVIEW_ASKS[kind] }, units, unitsMore: more });
1339
2158
  s.review = { batchId: batch.batch_id, kind, units: units.map((u) => ({ id: u.id, protected: u.protected })), index };
1340
2159
  s.decidedBatch = batch.batch_id; // never the local runner's: only an agent reads the file
@@ -1342,13 +2161,19 @@ export class Sessions extends EventEmitter {
1342
2161
  }
1343
2162
 
1344
2163
  /** `done` of a review that wrote nothing: the suggestions become cards (checked against the file), an activity entry of kind "review", no history update. */
1345
- #doneReview(s, results, run, post) {
2164
+ #doneReview(s, results, run, post, prepared = null) {
1346
2165
  const flags = [];
1347
2166
  const ix = s.review?.batchId === run.batch.batch_id ? s.review : null;
1348
- const { cards, dropped } = ix ? cardsFrom(results.suggestions, { bytes: post, format: s.format, units: ix.units, index: ix.index, reviewId: run.batch.batch_id }) : { cards: [], dropped: (Array.isArray(results.suggestions) ? results.suggestions : []).map((_, i) => ({ index: i, reason: "the review's outline is gone" })) };
2167
+ const gone = (why) => ({ cards: [], dropped: (Array.isArray(results.suggestions) ? results.suggestions : []).map((_, i) => ({ index: i, reason: why })) });
2168
+ let checked = gone("the review's outline is gone");
2169
+ if (ix) {
2170
+ const r = this.#results(s.format, post, run.preSha, [cardsTask(results, ix, run)], prepared).at(0);
2171
+ checked = usable(r) && r ? r : gone("the suggestions could not be checked against the file in time");
2172
+ }
2173
+ const { cards, dropped } = checked;
1349
2174
  for (const d of dropped) flags.push({ code: "bad_suggestion", severity: "warn", ...(d.index != null ? { index: d.index } : {}), reason: d.reason });
1350
2175
  if (results.voice_draft !== undefined) flags.push({ code: "voice_draft_not_allowed", severity: "warn" });
1351
- const queuedPlan = this.#planQueued(s, post, run.preSha, flags);
2176
+ const queuedPlan = this.#planQueued(s, post, run.preSha, flags, prepared);
1352
2177
  const cleanFlags = flags.map(cleanFlag);
1353
2178
  const { list: ignored, more: ignoredMore } = ignoredInstructions(results.document_instructions_ignored); // a review reads the whole file: its report counts most here
1354
2179
  const entry = {
@@ -1428,8 +2253,25 @@ export class Sessions extends EventEmitter {
1428
2253
  * and Revert, as for any local run. The text written is the stored suggestion, never text from the page. Every refusal happens before
1429
2254
  * anything is written. Other comments and the overall note stay for the next Send; typed edits ride with it too.
1430
2255
  */
1431
- useSuggestion(key, { suggestion: rec, commentId, comments, note, baseSha, bytes = null, edits = [] }) {
2256
+ /**
2257
+ * useSuggestion() as the server's route calls it (src/suggest.js): where the suggestion goes in the bytes about to be the file, and
2258
+ * the one reply that writes it, are worked out in the anchor worker first. Work that did not finish writes nothing: the comment stays a
2259
+ * draft and says it can go to the agent, as when the sentence is not found.
2260
+ */
2261
+ async useSuggestionAsync(key, args) {
1432
2262
  const s = this.get(key);
2263
+ this.#useChecks(s, args); // refused before it waits its turn
2264
+ return this.#serial(key, async () => {
2265
+ const { id } = this.#useChecks(s, args);
2266
+ const { c } = this.#usePlan(s, args, id); // nothing is stored before the commit, as a Send (sendAsync)
2267
+ const bytes = args.bytes?.length && args.bytes.length <= MAX_BYTES ? args.bytes : null;
2268
+ const prepared = await this.#offThread(s.format, { source: bytes ? { bytes } : { path: s.file }, tasks: [useTask(c.anchor, args.suggestion, id)] });
2269
+ return this.useSuggestion(key, { ...args, prepared });
2270
+ });
2271
+ }
2272
+
2273
+ /** The refusals of a quick fix's Use that need nothing read (useSuggestion runs them first). Returns {id}. */
2274
+ #useChecks(s, { suggestion: rec, commentId, comments }) {
1433
2275
  if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
1434
2276
  if (s.delivered || s.pending) throw new HttpError(409, "The agent started working on this file, so nothing was written. Ask the agent instead, or try again when it is done.", { code: "busy" });
1435
2277
  const words = rec && Object.hasOwn(STAMP_WORDS, rec.task) && rec.task !== "translate" ? STAMP_WORDS[rec.task] : null;
@@ -1439,22 +2281,37 @@ export class Sessions extends EventEmitter {
1439
2281
  const old = s.comments.find((c) => c.id === id);
1440
2282
  const sameAnchor = (a) => a && typeof a === "object" && a.quote === rec.quote && JSON.stringify(a.location ?? null) === JSON.stringify(rec.location ?? null);
1441
2283
  if (!mine || userText(mine.text) !== words || !sameAnchor(mine.anchor) || (old && old.status !== "draft")) throw new HttpError(409, "This comment is not the suggestion's stamp.", { code: "not_this_comment" });
2284
+ return { id };
2285
+ }
1442
2286
 
1443
- this.setDraft(key, { comments, note: note ?? s.note });
2287
+ /** What a Use stores before it commits (#sentPlan with its stamp), the stamp as planned, and every limit checked: nothing changed. */
2288
+ #usePlan(s, { comments, note, edits = [], rev, tab, seen, noteSeen, bytes = null }, id) {
2289
+ if (bytes?.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" }); // before anything is stored, as a Send
2290
+ const saved = this.#sentPlan(s, { comments, note, rev, tab, seen, noteSeen }, id);
2291
+ const c = saved.comments.find((x) => x.id === id);
2292
+ if (c?.status !== "draft") throw new HttpError(409, "This comment is not the suggestion's stamp.", { code: "not_this_comment" });
2293
+ if (!this.#fits(s, { comments: saved.comments, edits: [...s.edits, ...this.#editsToTake(s, edits)] })) throw tooLarge("This request");
2294
+ return { saved, c };
2295
+ }
2296
+
2297
+ useSuggestion(key, { suggestion: rec, commentId, comments, note, baseSha, bytes = null, edits = [], prepared = null, rev, tab, seen, noteSeen }) {
2298
+ const s = this.get(key);
2299
+ const { id } = this.#useChecks(s, { suggestion: rec, commentId, comments });
2300
+ // the page's list and note are saved first (only the stamp goes out), after every limit is checked on them
2301
+ this.#commitDraft(s, this.#usePlan(s, { comments, note, edits, rev, tab, seen, noteSeen, bytes }, id).saved, rev, tab, true);
1444
2302
  const c = s.comments.find((x) => x.id === id);
1445
- const anchor = { location: c.anchor.location, quote: c.anchor.quote, prefix: c.anchor.prefix, suffix: c.anchor.suffix };
1446
2303
  let plan = null;
1447
- // the place and the reply are worked out on the very bytes that are about to be the file; any refusal here writes nothing
1448
- const check = (b) => {
1449
- const units = readUnits(s.format, b);
1450
- const durable = buildDurable(units, anchor);
1451
- const place = placeOf(units, durable, rec.quote);
2304
+ // the place and the reply are worked out on the very bytes that are about to be the file (by the anchor worker, or here); any
2305
+ // refusal here writes nothing, and work that did not finish is "not here"
2306
+ const check = (b, sha) => {
2307
+ const r = this.#results(s.format, b, sha, [useTask(c.anchor, rec, id)], prepared).at(0);
2308
+ const { durable, place, reply } = usable(r) ? r : { durable: null, place: { ok: false, code: "not_here" }, reply: null };
1452
2309
  if (!place.ok) throw new HttpError(409, place.code === "protected" ? "This part of the file is protected, so it cannot be changed here." : LOCAL_HINT, { code: place.code });
1453
- const reply = buildUseReply(rec, id, place.text);
1454
- const batch = buildBatch({ file: s.file, format: s.format, baseSha: sha256(b), note: "", comments: [{ ...c, durable }], edits: [] });
2310
+ const batch = buildBatch({ file: s.file, format: s.format, baseSha: sha, note: "", comments: [{ ...c, durable }], edits: [] });
1455
2311
  if (!reply || batch.instructions.comments[0]?.reply_with !== "find_replace_or_text") throw new HttpError(409, LOCAL_HINT, { code: "not_here" });
1456
2312
  plan = { durable, reply, batch };
1457
2313
  };
2314
+ if (!this.#fits(s, { comments: s.comments, edits: [...s.edits, ...this.#editsToTake(s, edits)] })) throw tooLarge("This request"); // before anything is written
1458
2315
  let taken;
1459
2316
  try {
1460
2317
  taken = this.#takePage(s, { baseSha, bytes }, check);
@@ -1463,15 +2320,16 @@ export class Sessions extends EventEmitter {
1463
2320
  if (["not_here", "not_eligible", "protected"].includes(err?.extra?.code)) {
1464
2321
  c.localHint = LOCAL_HINT;
1465
2322
  this.#changed(s);
1466
- }
2323
+ } else this.emit("change", s, "draft"); // the drafts it stored
1467
2324
  throw err;
1468
2325
  }
1469
- for (const e of Array.isArray(edits) ? edits.slice(0, 2000) : []) s.edits.push({ kind: e.kind, before: e.before, after: e.after, label: e.label, location: e.location });
2326
+ s.edits.push(...this.#editsToTake(s, edits));
1470
2327
  c.durable = plan.durable;
1471
2328
  c.status = "sent";
1472
2329
  c.batchId = plan.batch.batch_id;
1473
2330
  delete c.localHint;
1474
2331
  s.lastWrite = taken.writeInfo;
2332
+ this.#sentRev(s, { rev, tab });
1475
2333
  try {
1476
2334
  this.startLocal(key, plan.batch, { reply: plan.reply, summary: USE_SUMMARY, model: true }); // saves, tells the page, starts the runner
1477
2335
  } catch (err) {
@@ -1530,13 +2388,13 @@ export class Sessions extends EventEmitter {
1530
2388
  * "voice" with a fixed summary, no history update and no analysis. A file that changed anyway (a direct save past voice_only) is
1531
2389
  * recorded as a withdrawn run would be: flagged, kept Revert-able.
1532
2390
  */
1533
- #doneVoice(s, results, run, post, postSha) {
2391
+ #doneVoice(s, results, run, post, postSha, prepared = null) {
1534
2392
  const flags = [];
1535
2393
  for (const r of (Array.isArray(results.results) ? results.results : []).slice(0, 1000)) flags.push({ code: "unknown_result", severity: "warn", comment_id: String(r?.comment_id ?? "").slice(0, 64) });
1536
2394
  const check = cleanVoiceDraft(results.voice_draft); // size first, then hidden characters, links and markup, the layout, the caps
1537
2395
  if (check.code) flags.push({ code: "bad_voice_draft", severity: "warn", reason: check.code });
1538
2396
  this.#guardFlags(run, flags);
1539
- const queuedPlan = this.#planQueued(s, post, postSha, flags);
2397
+ const queuedPlan = this.#planQueued(s, post, postSha, flags, prepared);
1540
2398
  const { list: ignored, more: ignoredMore } = ignoredInstructions(results.document_instructions_ignored);
1541
2399
  // ---- commit: the draft file first (a failed write keeps nothing and says so), then the session
1542
2400
  let kept = false;
@@ -1589,7 +2447,7 @@ export class Sessions extends EventEmitter {
1589
2447
  if (!batch) throw new HttpError(409, "nothing to take back");
1590
2448
  // the note travels back with the comments: the withdrawn batch's note, then a queued one, then whatever was typed since
1591
2449
  const notes = [batch.instructions?.overall_note, s.queuedNote, s.note].filter(Boolean);
1592
- s.note = [...new Set(notes)].join("\n\n");
2450
+ this.#giveNote(s, [...new Set(notes)].join("\n\n"));
1593
2451
  s.queuedNote = "";
1594
2452
  let event = "state";
1595
2453
  if (s.delivered) {
@@ -1640,7 +2498,27 @@ export class Sessions extends EventEmitter {
1640
2498
  return true;
1641
2499
  }
1642
2500
 
1643
- revert(key, activityId) {
2501
+ /** revert() as the server's route calls it: a waiting batch's anchors are moved onto the restored version in the anchor worker first. */
2502
+ async revertAsync(key, activityId) {
2503
+ return this.#serial(key, async () => {
2504
+ const s = this.get(key);
2505
+ const entry = s.activity.find((a) => a.id === activityId && a.kind === "agent");
2506
+ let prepared = null;
2507
+ if (entry && !entry.reverted && !s.delivered && s.pending && !readOnlyKind(s.pending) && !s.pending.instructions?.review && s.snapshots.has(entry.preSha)) {
2508
+ const tasks = this.#revertComments(s, entry).map(reanchorTask);
2509
+ if (tasks.length) prepared = await this.#offThread(s.format, { source: { path: s.snapshots.blob(entry.preSha), sha: entry.preSha }, tasks });
2510
+ }
2511
+ return this.revert(key, activityId, { prepared });
2512
+ });
2513
+ }
2514
+
2515
+ /** The comments of the waiting batch, as revert() moves them: the run's comments become drafts first, so only the others are left. */
2516
+ #revertComments(s, entry) {
2517
+ const ids = new Set(s.pending.instructions.comments.map((c) => c.id));
2518
+ return s.comments.filter((c) => ids.has(c.id) && !(c.batchId === entry.batchId && ["done", "declined", "asked"].includes(c.status)));
2519
+ }
2520
+
2521
+ revert(key, activityId, { prepared = null } = {}) {
1644
2522
  const s = this.get(key);
1645
2523
  if (s.delivered) throw new HttpError(423, s.delivered.local ? "a change is being made on this Mac" : "the agent is working", { code: "locked" });
1646
2524
  const entry = s.activity.find((a) => a.id === activityId && a.kind === "agent");
@@ -1652,7 +2530,15 @@ export class Sessions extends EventEmitter {
1652
2530
  if (sha256(disk) !== entry.postSha) throw new HttpError(409, mac ? "the file changed after the change was made; revert refused" : "the file changed after the agent finished; revert refused", { code: "changed_since" });
1653
2531
  if (!s.snapshots.has(entry.preSha)) throw new HttpError(410, mac ? "the version before this change is no longer stored" : "the pre-agent version is no longer stored");
1654
2532
  const pre = s.snapshots.read(entry.preSha);
1655
- this.#replace(s, pre, { expectSha: entry.postSha, label: "before revert" }); // a revert is a new version
2533
+ // an HTML file: a revert restores the version before the run, by identity (the stored bytes must hash to the run's preSha), never
2534
+ // through the write check: that version was the file before (opened by the person or written after its own check), and checking it
2535
+ // against the run's result would refuse undoing a run that removed active content (the revert would "add" it back)
2536
+ let htmlChecked = null;
2537
+ if (s.format === "html") {
2538
+ if (sha256(pre) !== entry.preSha) throw new HttpError(410, mac ? "the version before this change is no longer stored" : "the pre-agent version is no longer stored");
2539
+ htmlChecked = { from: entry.postSha, to: entry.preSha };
2540
+ }
2541
+ this.#replace(s, pre, { expectSha: entry.postSha, label: "before revert", htmlChecked }); // a revert is a new version
1656
2542
  s.snapshots.add(pre, "reverted");
1657
2543
  s.baseSha = entry.preSha;
1658
2544
  addUpdate(s, { kind: "revert", before: entry.postSha, after: entry.preSha, local: mac }, this.now());
@@ -1670,7 +2556,7 @@ export class Sessions extends EventEmitter {
1670
2556
  }
1671
2557
  for (const g of s.suggestions || []) if (g.status === "accepted" && g.acceptedBatch === entry.batchId) g.status = "open"; // the text is back: it can be accepted again
1672
2558
  if (entry.by === "local" && entry.localNote && !s.note) {
1673
- s.note = entry.localNote; // the request comes back to the box, and the same words go to the agent next time
2559
+ this.#giveNote(s, entry.localNote); // the request comes back to the box, and the same words go to the agent next time
1674
2560
  s.skipLocalNote = entry.localNote;
1675
2561
  }
1676
2562
  // a batch built against the reverted version is rebuilt against the restored bytes (a review: its outline is read again; a voice
@@ -1680,19 +2566,9 @@ export class Sessions extends EventEmitter {
1680
2566
  else if (s.pending) {
1681
2567
  const ids = new Set(s.pending.instructions.comments.map((c) => c.id));
1682
2568
  const cs = s.comments.filter((c) => ids.has(c.id));
1683
- let units = null;
1684
- try {
1685
- units = readUnits(s.format, pre);
1686
- } catch {
1687
- units = null;
1688
- }
1689
- for (const c of cs) {
1690
- try {
1691
- if (units) c.durable = reanchor(units, c);
1692
- } catch {
1693
- c.durable = null;
1694
- }
1695
- }
2569
+ // moved onto the restored version by the anchor worker (revertAsync) or here; one whose work did not finish goes out weaker
2570
+ const work = this.#results(s.format, pre, entry.preSha, cs.map(reanchorTask), prepared);
2571
+ if (work.read?.ok) cs.forEach((c, i) => (c.durable = usable(work.at(i)) ? work.at(i) : work.at(i)?.error ? null : weakDurable(s.format, c.anchor)));
1696
2572
  s.pending = buildBatch({ file: s.file, format: s.format, baseSha: entry.preSha, note: s.pending.instructions.overall_note, comments: cs, edits: [], style: styleNote(), guide: voiceGuide() });
1697
2573
  for (const c of cs) c.batchId = s.pending.batch_id;
1698
2574
  }
@@ -1704,6 +2580,7 @@ export class Sessions extends EventEmitter {
1704
2580
  close(key) {
1705
2581
  const s = this.get(key);
1706
2582
  s.closed = true;
2583
+ this.#views.delete(key); // the render of an HTML file (htmlView)
1707
2584
  this.editors.delete(key); // the closed tab may stay on screen with its event stream open: it must not hold the file against the tab that reopens it
1708
2585
  for (const p of s.pollers.splice(0)) p.resolve({ status: "closed" });
1709
2586
  fs.rmSync(path.join(stateDir(), "agents", s.key, "outline"), { recursive: true, force: true }); // the outline files of Onionskin's own agent
@@ -1733,6 +2610,37 @@ export class Sessions extends EventEmitter {
1733
2610
  return out;
1734
2611
  }
1735
2612
 
2613
+ // ---- the view of an HTML file (GET /api/html-view) ------------------------------------------------------------------------------
2614
+ // key -> {sha, full, plain}: the render of one version (promises, so two requests for it share one worker run); one version per file,
2615
+ // dropped when the file changes or the session closes. Ceiling: one entry per open HTML file, each up to ~43 MB (the render copy at the
2616
+ // limits): a person with many large pages open holds them all; an LRU over files would be the upgrade.
2617
+ #views = new Map();
2618
+ /**
2619
+ * The page's view of an HTML file (PLAN-html 3.2, R12): the disk bytes are taken as /api/file takes them (loadForPage: an outside change
2620
+ * is recorded, the watcher told), then the render copy of exactly those bytes is built in the render worker (src/offload.js
2621
+ * renderOffThread), or the plain copy with `plain`. Resolves to {sha, plain, reason, stats, texts, editable, ident, bytes}; throws HttpError
2622
+ * 422 with the encoding's sentence and code (and `copyable`) or a limit's, 503 when the worker was busy or did not finish.
2623
+ */
2624
+ async htmlView(key, { plain = false } = {}) {
2625
+ const s = this.get(key);
2626
+ if (s.format !== "html") throw new HttpError(400, "this file is not an HTML file", { code: "not_html" });
2627
+ const { bytes, diskSha } = this.loadForPage(key);
2628
+ let c = this.#views.get(key);
2629
+ if (!c || c.sha !== diskSha) this.#views.set(key, (c = { sha: diskSha, full: null, plain: null }));
2630
+ const k = plain ? "plain" : "full";
2631
+ if (!c[k]) {
2632
+ const p = renderOffThread({ bytes, fileDir: path.dirname(s.file), plain }).then((r) => {
2633
+ if (r.ok) return { sha: diskSha, plain: r.plain, reason: r.reason, stats: r.stats, texts: r.texts, editable: r.editable, ident: r.ident, bytes: r.bytes };
2634
+ if (c[k] === p) c[k] = null; // a failure is not kept: the next request tries again
2635
+ if (r.failed) throw new HttpError(503, `Onionskin could not prepare this page: ${r.why}. Try again in a moment.`, { code: r.failed === "busy" ? "busy" : "render_failed" });
2636
+ if (r.code === "not_utf8" || ["utf16", "invalid_utf8", "roundtrip"].includes(r.code)) throw encodingRefusal(s.file, { error: r.error, code: r.code, copyable: r.copyable });
2637
+ throw new HttpError(422, `Onionskin cannot read this file: ${r.error}.`, { code: "too_large" });
2638
+ });
2639
+ c[k] = p;
2640
+ }
2641
+ return c[k];
2642
+ }
2643
+
1736
2644
  /** The history list and one update (browser routes, read-only; see src/history.js). */
1737
2645
  history(key) {
1738
2646
  return historyList(this.get(key));
@@ -1847,25 +2755,58 @@ export class Sessions extends EventEmitter {
1847
2755
  */
1848
2756
  resolveAll(key) {
1849
2757
  const s = this.get(key);
2758
+ // an HTML file is read in workers only: on this thread every anchor is "moved or removed" (the server's route resolves in the worker)
2759
+ if (s.format === "html") return { baseSha: sha256(readCapped(s.file)), format: s.format, anchors: Object.fromEntries(this.#resolveTask(s).comments.map((c) => [c.id, { method: "none" }])), view: null };
1850
2760
  const bytes = readCapped(s.file);
1851
- const out = { baseSha: sha256(bytes), format: s.format, anchors: {}, view: null };
1852
- const units = readUnits(s.format, bytes);
1853
- for (const c of s.comments) {
1854
- if (!c.durable) continue;
1855
- let r = resolveDurableAny(units, c.durable);
1856
- if (r.method === "none" && units.format === "pptx" && c.durable.pinFallback) {
1857
- const f = resolveDurableAny(units, c.durable.pinFallback);
1858
- if (f.method !== "none") r = { ...f, tieLost: true };
2761
+ const baseSha = sha256(bytes);
2762
+ const r = anchorWork({ format: s.format, bytes, sha: baseSha, tasks: [this.#resolveTask(s)], budget: SCAN_BUDGET });
2763
+ if (!r.read.ok) throw r.read.cap ? new CapError(r.read.cap) : new Error(r.read.invalid);
2764
+ if (r.results[0]?.error) throw new Error(r.results[0].error);
2765
+ return { baseSha, format: s.format, ...r.results[0] };
2766
+ }
2767
+
2768
+ /**
2769
+ * resolveAll() as the server's route calls it: resolved in the anchor worker against the file on disk. Work that did not finish
2770
+ * answers as a spent budget does: every anchor "moved or removed" (the page keeps the comments in its list), the view unknown.
2771
+ */
2772
+ async resolveAllAsync(key) {
2773
+ const s = this.get(key);
2774
+ const task = this.#resolveTask(s);
2775
+ if (s.format === "html") await this.#htmlDrafts(s, task);
2776
+ const r = await anchorsOffThread({ format: s.format, source: { path: s.file }, tasks: [task] });
2777
+ if (r.ok && r.r.read.cap) throw new CapError(r.r.read.cap);
2778
+ if (r.ok && r.r.read.ok && usable(r.r.results[0])) return { baseSha: r.sha, format: s.format, ...r.r.results[0] };
2779
+ return { baseSha: r.ok ? r.sha : sha256(readCapped(s.file)), format: s.format, anchors: Object.fromEntries(task.comments.map((c) => [c.id, { method: "none" }])), view: null };
2780
+ }
2781
+
2782
+ /**
2783
+ * HTML: a draft (no durable yet) whose page anchor was made on another version than the file on disk (an outside change, a revert, a
2784
+ * save) would be drawn at a node id of that other version. It is made durable on its own version first (a stored snapshot) and resolved
2785
+ * with the others; one whose version is no longer stored resolves by its quote. Drafts made on the file's version need nothing: the page
2786
+ * draws them at their own place.
2787
+ */
2788
+ async #htmlDrafts(s, task) {
2789
+ let cur;
2790
+ try {
2791
+ cur = sha256(readCapped(s.file));
2792
+ } catch {
2793
+ return;
2794
+ }
2795
+ const other = s.comments.filter((c) => !c.durable && c.anchor?.location?.version && c.anchor.location.version !== cur);
2796
+ const byVersion = new Map();
2797
+ for (const c of other) (byVersion.get(c.anchor.location.version) || byVersion.set(c.anchor.location.version, []).get(c.anchor.location.version)).push(c);
2798
+ for (const [v, list] of byVersion) {
2799
+ const r = s.snapshots.has(v) ? await this.#offThread("html", { source: { path: s.snapshots.blob(v), sha: v }, tasks: list.map(buildTask) }) : null;
2800
+ for (const c of list) {
2801
+ const d = r?.ok && r.read?.ok ? r.map.get(taskKey(buildTask(c))) : undefined;
2802
+ const durable = usable(d) && d ? d : weakDurable("html", c.anchor);
2803
+ if (durable) task.comments.push({ id: c.id, durable });
1859
2804
  }
1860
- if (units.format === "docx" && r.method !== "none") r.heading = headingBefore(units.paragraphs, r.ordinal);
1861
- out.anchors[c.id] = r;
1862
- }
1863
- if (s.format === "pptx") out.slides = units.deck.slides.map((x) => ({ index: x.index, id: x.id }));
1864
- if (s.format === "docx") {
1865
- if (s.view?.durable) out.view = { ...resolveDurable(units.paragraphs, s.view.durable), offsetPx: s.view.offsetPx, zoom: s.view.zoom, fraction: s.view.fraction };
1866
- } else out.view = resolveView(units, s.view);
1867
- out.sheets = s.format === "xlsx" ? units.wb.sheets.map((x) => ({ index: x.index, name: x.name, sheetId: x.sheetId })) : undefined;
1868
- return out;
2805
+ }
2806
+ }
2807
+
2808
+ #resolveTask(s) {
2809
+ return { op: "resolveAll", comments: s.comments.filter((c) => c.durable).map((c) => ({ id: c.id, durable: c.durable })), view: s.view };
1869
2810
  }
1870
2811
  }
1871
2812
 
@@ -1906,7 +2847,7 @@ function exportInWorker(job) {
1906
2847
  };
1907
2848
  const timer = setTimeout(() => finish({ timeout: true }), EXPORT_TIMEOUT_MS());
1908
2849
  try {
1909
- worker = new Worker(new URL("./redline-worker.js", import.meta.url), { workerData: job });
2850
+ worker = startWorker("redline-worker.js", { workerData: job });
1910
2851
  } catch (err) {
1911
2852
  return finish({ check_failed: String(err?.message || err) });
1912
2853
  }