onionskin-cli 0.0.3 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +198 -0
- package/INSTALL.md +5 -1
- package/NOTICE +34 -0
- package/README.md +102 -34
- package/docs/contracts/batch.schema.json +27 -4
- package/docs/contracts/ops.schema.json +117 -5
- package/package.json +4 -2
- package/scripts/checker/check.py +24 -5
- package/scripts/checker/requirements.txt +482 -5
- package/scripts/requirements.txt +482 -5
- package/scripts/rewriter/apple/main.swift +1 -1
- package/scripts/rewriter/apple/onionskin-fm +0 -0
- package/scripts/rewriter/apple/onionskin-fm.json +1 -1
- package/skills/onionskin/SKILL.md +12 -9
- package/skills/onionskin/reference/batch.md +5 -3
- package/skills/onionskin/reference/formats.md +5 -1
- package/skills/onionskin/reference/ops.md +48 -4
- package/skills/onionskin/reference/troubleshooting.md +31 -0
- package/src/agent-detect.js +28 -2
- package/src/agent-prompt.js +12 -1
- package/src/agent-runner.js +110 -20
- package/src/anchor-work.js +154 -0
- package/src/anchor-worker.js +38 -0
- package/src/anchors.js +61 -11
- package/src/apply.js +119 -47
- package/src/batch.js +32 -8
- package/src/check-worker.js +8 -5
- package/src/checker.js +238 -33
- package/src/cli.js +248 -42
- package/src/constants.js +34 -1
- package/src/diff.js +3 -1
- package/src/doctor.js +12 -3
- package/src/docx.js +464 -87
- package/src/edit-hash.js +19 -0
- package/src/fsutil.js +14 -6
- package/src/history-worker.js +8 -8
- package/src/history.js +144 -31
- package/src/html-active.js +384 -0
- package/src/html-envelope.js +51 -0
- package/src/html-parser.js +8 -0
- package/src/html-render-worker.js +25 -0
- package/src/html-render.js +939 -0
- package/src/html-utf8.js +65 -0
- package/src/html-write-worker.js +14 -0
- package/src/html-write.js +380 -0
- package/src/html.js +1259 -0
- package/src/local.js +68 -4
- package/src/merge-package.js +1166 -0
- package/src/merge-pptx.js +98 -38
- package/src/merge-proof.js +1019 -0
- package/src/merge-worker.js +60 -0
- package/src/merge-xlsx.js +84 -29
- package/src/merge.js +286 -83
- package/src/nfc.js +46 -0
- package/src/notes.js +122 -0
- package/src/offload.js +419 -1
- package/src/open-worker.js +27 -0
- package/src/ops/common.js +2 -1
- package/src/ops/docx-table.js +5 -3
- package/src/ops/docx.js +58 -29
- package/src/ops/html.js +394 -0
- package/src/ops/pptx.js +82 -23
- package/src/ops/run.js +23 -3
- package/src/ops/runs.js +215 -13
- package/src/ops/schema.js +8 -1
- package/src/ops/xlsx.js +121 -10
- package/src/ops-worker.js +17 -0
- package/src/package.js +43 -4
- package/src/person.js +2 -1
- package/src/pptx.js +323 -23
- package/src/question.js +10 -3
- package/src/quick-plan.js +13 -6
- package/src/quick.js +1 -1
- package/src/redline-check.js +12 -2
- package/src/redline-worker.js +10 -14
- package/src/reply.js +46 -19
- package/src/retain.js +223 -0
- package/src/review.js +32 -18
- package/src/rewrite-check.js +2 -2
- package/src/rewriter.js +38 -8
- package/src/sanitize.js +16 -3
- package/src/server-entry.js +4 -3
- package/src/server.js +175 -62
- package/src/session.js +1151 -210
- package/src/setup.js +20 -2
- package/src/stop-gate.js +34 -0
- package/src/suggest-reply.js +42 -7
- package/src/suggest.js +14 -29
- package/src/text.js +161 -11
- package/src/update.js +517 -0
- package/src/voice-guide.js +15 -9
- package/src/voice-learn.js +15 -4
- package/src/voice-sample-worker.js +7 -6
- package/src/voice-store.js +11 -4
- package/src/voice-worker.js +4 -6
- package/src/voice.js +2 -1
- package/src/worker-host.js +23 -0
- package/src/xlsx.js +378 -97
- package/src/xml-tree.js +6 -3
- package/src/xml.js +82 -34
- package/src/zip-write.js +10 -5
- package/src/zip.js +73 -18
- package/vendor/parse5/LICENSE.txt +32 -0
- package/vendor/parse5/parse5.mjs +7822 -0
- package/vendor/parse5.lock.json +23 -0
- package/web/dist/assets/{index-DS-WST1E.js → index-CjsTf5gj.js} +58 -52
- package/web/dist/assets/index-DnbLtHhG.css +1 -0
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/session.js
CHANGED
|
@@ -2,9 +2,11 @@ import crypto from "node:crypto";
|
|
|
2
2
|
import { EventEmitter } from "node:events";
|
|
3
3
|
import fs from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
|
-
import { REFUSED_EXTENSIONS,
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
5
|
+
import { EDITS_OVERFLOW, ONLY_FORMATS, REFUSED_EXTENSIONS, REFUSED_PAGES, MAX_DOC_BYTES, PRODUCT, env, formatOfExt } from "./constants.js";
|
|
6
|
+
import { readAll, unitsFor } from "./anchors.js";
|
|
7
|
+
import { anchorWork, weakDurable } from "./anchor-work.js";
|
|
8
|
+
import { scanBudget } from "./text.js";
|
|
9
|
+
import { CapError } from "./xml.js";
|
|
8
10
|
import { checkRegularOwned, readDoc, replaceGuarded, sha256, Snapshots, WriteConflict, writeSiblingCopy } from "./fsutil.js";
|
|
9
11
|
import { buildBatch, compactBatch, withOutline } from "./batch.js";
|
|
10
12
|
import { styleChangedOutside, styleNote } from "./style.js";
|
|
@@ -13,7 +15,10 @@ import { cleanVoiceDraft } from "./voice-guide.js";
|
|
|
13
15
|
import { voiceJob } from "./voice-learn.js";
|
|
14
16
|
import { clearTickets, saveTicket, ticketOf } from "./lease.js";
|
|
15
17
|
import { analyze } from "./analyze.js";
|
|
18
|
+
import { anchorTimeoutMs, anchorsOffThread, htmlWriteOffThread, mergeOffThread, readOffThread, renderOffThread, startWorker } from "./offload.js";
|
|
16
19
|
import { deepNotice, validatePackage } from "./package.js";
|
|
20
|
+
import { HtmlEncodingError } from "./html.js";
|
|
21
|
+
import { ACTIVE_REFUSED } from "./html-active.js";
|
|
17
22
|
import { execFile } from "node:child_process";
|
|
18
23
|
import { reopenCommand } from "./person.js";
|
|
19
24
|
import { sessionKey, stateDir, sub, writeStateFile } from "./paths.js";
|
|
@@ -21,16 +26,17 @@ import { cleanFlag, hasControlChars, untrusted, userText } from "./sanitize.js";
|
|
|
21
26
|
import { addUpdate, historyDetail, historyList, visibleUpdates } from "./history.js";
|
|
22
27
|
import { BAD_QUESTION_NOTE, QUESTIONS_PER_BATCH, cleanQuestion } from "./question.js";
|
|
23
28
|
import { validateOps } from "./ops/schema.js";
|
|
24
|
-
import { Worker } from "node:worker_threads";
|
|
25
29
|
import { AUTHORS, wordDate } from "./redline.js";
|
|
26
30
|
import { deepRedline } from "./checker.js";
|
|
27
31
|
import { TRACKED, formattingText, savedText, unsupportedText } from "./tracked-words.js";
|
|
28
32
|
import { LOCAL_HOLDER } from "./local.js";
|
|
29
33
|
import { bgHolder, isAgentHolder, isBgHolder, termHolder, TERM_PREFIX } from "./agent-detect.js";
|
|
30
|
-
import { OUTLINE_FILE_MAX, REVIEW_ASKS, REVIEW_KINDS, SUGGESTIONS_MAX, UNITS_MAX,
|
|
34
|
+
import { OUTLINE_FILE_MAX, REVIEW_ASKS, REVIEW_KINDS, SUGGESTIONS_MAX, UNITS_MAX, outlineOf, reviewSummary, unitsOf } from "./review.js";
|
|
31
35
|
import { VOICE_BATCH_MAX_CHARS } from "./voice-samples.js";
|
|
32
36
|
import { STAMP_WORDS } from "./stamp-words.js";
|
|
33
|
-
import { USE_SUMMARY
|
|
37
|
+
import { USE_SUMMARY } from "./suggest-reply.js";
|
|
38
|
+
import { mergeNote } from "./notes.js";
|
|
39
|
+
import { COMMENT_MAX, COMMENT_TEXT_MAX, COMMENTS_MAX, EDITS_MAX, NOTE_MAX, TOO_MANY_COMMENTS, REQUEST_MAX, SESSION_MAX, capComment, cleanEdit, cleanPageAnchor, cleanView, cutText, jsonSize } from "./retain.js";
|
|
34
40
|
|
|
35
41
|
/** The line under a comment whose quick fixes could not be written on this Mac (docs/contracts/routes.md). */
|
|
36
42
|
export const LOCAL_HINT = "This could not be changed on this Mac, so nothing was written. Press Send to ask the agent.";
|
|
@@ -59,8 +65,33 @@ export class HttpError extends Error {
|
|
|
59
65
|
}
|
|
60
66
|
}
|
|
61
67
|
|
|
68
|
+
/** One comment more than a file keeps (src/retain.js COMMENTS_MAX): 413 too_many_comments, in words the page shows as they are. */
|
|
69
|
+
const tooMany = () => new HttpError(413, TOO_MANY_COMMENTS, { code: "too_many_comments" });
|
|
70
|
+
/** A note longer than Onionskin keeps (src/retain.js NOTE_MAX): 413 note_too_long. `joined`: a waiting note and this one together. */
|
|
71
|
+
const noteTooLong = (joined) =>
|
|
72
|
+
new HttpError(413, joined ? `The note waiting for the agent and this one together are longer than Onionskin takes (${NOTE_MAX.toLocaleString("en-US")} characters), so nothing was sent. Shorten this note, then Send again.` : `This note is longer than Onionskin takes (${NOTE_MAX.toLocaleString("en-US")} characters), so it was not saved. Shorten it, then try again.`, { code: "note_too_long" });
|
|
73
|
+
/** The page's note as stored: refused (never cut) past NOTE_MAX. */
|
|
74
|
+
function noteOf(v) {
|
|
75
|
+
const t = userText(v, NOTE_MAX + 1);
|
|
76
|
+
if ([...t].length > NOTE_MAX) throw noteTooLong(false);
|
|
77
|
+
return t;
|
|
78
|
+
}
|
|
79
|
+
/** Two notes, the earlier first, as one (a waiting batch's or the queued ones, then the new one): refused past NOTE_MAX. */
|
|
80
|
+
function joinNotes(a, b) {
|
|
81
|
+
const t = [a, b].filter(Boolean).join("\n\n");
|
|
82
|
+
if ([...t].length > NOTE_MAX) throw noteTooLong(true);
|
|
83
|
+
return t;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** A request whose own data is more than Onionskin keeps (src/retain.js): 413 too_large_request, in words the page shows as they are. */
|
|
87
|
+
export const tooLarge = (what) => new HttpError(413, `${what} is larger than Onionskin takes (${Math.round(REQUEST_MAX / 1024 / 1024)} MB), so nothing was saved. Remove some comments or shorten the longest ones, then try again.`, { code: "too_large_request" });
|
|
88
|
+
|
|
62
89
|
const LEASE_MS = () => Number(env("LEASE_MS", 15 * 60 * 1000));
|
|
90
|
+
const SENT_TABS = 16; // tabs of one file whose newest committed Send a session remembers (s.sent)
|
|
91
|
+
const DELETED_KEPT = 200; // ids and words of deleted drafts a session remembers for a handover (Sessions.handOver)
|
|
92
|
+
const HANDOVER_MS = 60_000; // how long a tab that lost editing may hand over its unsaved words (Sessions.handOver)
|
|
63
93
|
const MAX_BYTES = MAX_DOC_BYTES;
|
|
94
|
+
const TOO_BIG = `Onionskin cannot read this file: it is larger than ${Math.round(MAX_BYTES / 1024 / 1024)} MB, more than Onionskin can read. Make it smaller (pictures and videos are often most of a file) or split it, and open that.`;
|
|
64
95
|
/** "Show in folder" exists on macOS (`open -R`); elsewhere the page shows the path. ONIONSKIN_NO_OPEN=1 turns it off, ONIONSKIN_OPEN_STUB=<file> records instead of launching. */
|
|
65
96
|
const canReveal = () => !!env("OPEN_STUB") || (process.platform === "darwin" && !env("NO_OPEN"));
|
|
66
97
|
/** readDoc with the size refusal as an HTTP error */
|
|
@@ -68,31 +99,81 @@ function readCapped(file) {
|
|
|
68
99
|
try {
|
|
69
100
|
return readDoc(file);
|
|
70
101
|
} catch (err) {
|
|
71
|
-
if (err.code === "TOO_LARGE") throw new HttpError(413, `
|
|
102
|
+
if (err.code === "TOO_LARGE") throw new HttpError(413, `Onionskin cannot read this file: ${err.message}.`, { code: "too_large" });
|
|
72
103
|
throw err;
|
|
73
104
|
}
|
|
74
105
|
}
|
|
75
106
|
|
|
76
|
-
|
|
107
|
+
/**
|
|
108
|
+
* Every part the server reads later, of bytes about to become the file (a Send's, an agent's write), so they are refused before anything is
|
|
109
|
+
* written: past a reader's cap with the reason (CapError: 422 too_large), damaged where the readers look as an invalid package (422), never
|
|
110
|
+
* as an internal error. The reading is kept for what reads the same version next (unitsFor).
|
|
111
|
+
*/
|
|
112
|
+
function readWhole(format, bytes, sha, refusal = "refusing to write") {
|
|
113
|
+
try {
|
|
114
|
+
const units = readAll(unitsFor(format, bytes, sha));
|
|
115
|
+
readInCaps(format, sha);
|
|
116
|
+
return units;
|
|
117
|
+
} catch (err) {
|
|
118
|
+
if (err instanceof CapError || err instanceof HttpError) throw err;
|
|
119
|
+
throw new HttpError(422, `${refusal}: ${untrusted(err?.message || err, 200)}`, { code: "invalid_package" });
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// "<format>:<sha256>" of the versions read whole within the readers' caps, on this thread (readWhole) or by the open worker, so a version
|
|
124
|
+
// read again is not read whole again. Ceiling: VERIFIED_KEPT entries, the oldest dropped first (one dropped is read whole again when needed).
|
|
125
|
+
const VERIFIED_KEPT = 256;
|
|
126
|
+
const verified = new Set();
|
|
127
|
+
function readInCaps(format, sha) {
|
|
128
|
+
const id = `${format}:${sha}`;
|
|
129
|
+
verified.delete(id);
|
|
130
|
+
verified.add(id);
|
|
131
|
+
if (verified.size > VERIFIED_KEPT) verified.delete(verified.values().next().value);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Bytes about to be handed to the engine on this thread (a quick preview, a merge, a change made on this Mac), which reads without the
|
|
136
|
+
* readers' caps: they are read whole within the caps first (readWhole) unless this version already was, so a file that changed on disk
|
|
137
|
+
* past a cap is refused with the reason (CapError: 422 too_large) before the engine parses any of it. Returns `bytes`.
|
|
138
|
+
*/
|
|
139
|
+
export function readChecked(format, bytes, { sha = sha256(bytes), refusal = "cannot read the file" } = {}) {
|
|
140
|
+
if (!verified.has(`${format}:${sha}`)) readWhole(format, bytes, sha, refusal);
|
|
141
|
+
return bytes;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// bounded before it is rounded, so the result is always finite (1e308 * 1000 is Infinity, stored as null); anything that is not a finite
|
|
145
|
+
// number is 0, as src/retain.js drops what is not of its type
|
|
146
|
+
const num = (v, max) => {
|
|
147
|
+
const n = Number(v);
|
|
148
|
+
return Number.isFinite(n) ? Math.round(Math.min(max, Math.max(0, n)) * 1000) / 1000 : 0;
|
|
149
|
+
};
|
|
150
|
+
const frac = (v) => num(v, 1);
|
|
151
|
+
const PX_MAX = 1_000_000; // a paragraph taller than this is not drawn by any page
|
|
77
152
|
/** A pin: x/y as fractions of the element it sits on (`yPx` is kept for DOCX paragraphs); `on` says what that element is. */
|
|
78
153
|
function cleanPin(p) {
|
|
79
|
-
const out = { xFrac:
|
|
80
|
-
if (p.yFrac != null) out.yFrac =
|
|
154
|
+
const out = { xFrac: frac(p.xFrac), yPx: num(p.yPx, PX_MAX) };
|
|
155
|
+
if (p.yFrac != null) out.yFrac = frac(p.yFrac);
|
|
81
156
|
if (["paragraph", "cell", "shape", "slide"].includes(p.on)) out.on = p.on;
|
|
82
157
|
if (p.sxFrac != null) {
|
|
83
|
-
out.sxFrac =
|
|
84
|
-
out.syFrac =
|
|
158
|
+
out.sxFrac = frac(p.sxFrac);
|
|
159
|
+
out.syFrac = frac(p.syFrac);
|
|
85
160
|
}
|
|
86
161
|
return out;
|
|
87
162
|
}
|
|
88
163
|
|
|
89
164
|
const APPLY_STATUS = new Set(["refused", "skipped_protected", "anchor_not_found", "expect_mismatch", "not_applied"]);
|
|
90
|
-
// the three-way merge of each format (loaded on first use: they pull in the engine)
|
|
91
|
-
const MERGERS = { docx: { module: "./merge.js", fn: "mergeDocx" }, xlsx: { module: "./merge-xlsx.js", fn: "mergeXlsx" }, pptx: { module: "./merge-pptx.js", fn: "mergePptx" } };
|
|
92
165
|
const HELD_SHOWN = 5; // items the page's "not in the document" message lists (web/src/Banners.jsx)
|
|
93
166
|
const WRITES_KEPT = 20; // writes per file the merge can trace (src/merge-xlsx.js agentChanges); an older base finds no record and is handed back
|
|
94
|
-
/**
|
|
95
|
-
|
|
167
|
+
/**
|
|
168
|
+
* What the page says the user did during the run: kinds and places only, used to hold edits back or to name what is not carried, never to
|
|
169
|
+
* write. The first EDITS_KEPT; past them one more entry of kind EDITS_OVERFLOW says the list was cut, so a planner that needs the whole list
|
|
170
|
+
* refuses the merge (src/merge-xlsx.js) instead of reading the cut list as complete. The proof of the merge never reads it.
|
|
171
|
+
*/
|
|
172
|
+
const EDITS_KEPT = 2000;
|
|
173
|
+
export const editKinds = (edits) => {
|
|
174
|
+
const list = (Array.isArray(edits) ? edits.slice(0, EDITS_KEPT) : []).map((e) => ({ kind: String(e?.kind ?? "").slice(0, 40), sheet: typeof e?.sheet === "string" ? e.sheet.slice(0, 100) : null, slide: Number.isInteger(e?.slide) ? e.slide : null }));
|
|
175
|
+
return Array.isArray(edits) && edits.length > EDITS_KEPT ? [...list, { kind: EDITS_OVERFLOW, sheet: null, slide: null }] : list;
|
|
176
|
+
};
|
|
96
177
|
/** One op `apply` did not apply, as the page shows it: codes and short reasons only (never document text). */
|
|
97
178
|
function cleanProblem(p, run = "") {
|
|
98
179
|
return {
|
|
@@ -101,7 +182,8 @@ function cleanProblem(p, run = "") {
|
|
|
101
182
|
op: untrusted(p?.op, 40),
|
|
102
183
|
status: APPLY_STATUS.has(p?.status) ? p.status : "refused",
|
|
103
184
|
code: p?.code ? untrusted(p.code, 60) : null,
|
|
104
|
-
|
|
185
|
+
// the R1 refusal in the server's own words, never the reason text an agent's report carries
|
|
186
|
+
reason: p?.code === "active_content" ? ACTIVE_REFUSED : untrusted(p?.reason, 200),
|
|
105
187
|
};
|
|
106
188
|
}
|
|
107
189
|
|
|
@@ -109,17 +191,47 @@ function shellQuote(s) {
|
|
|
109
191
|
return `'${String(s).replace(/'/g, `'\\''`)}'`;
|
|
110
192
|
}
|
|
111
193
|
|
|
112
|
-
/**
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
if (units.format === "docx") return { ...d, ...r, format: "docx", quote: d.quote, prefix: d.prefix, suffix: d.suffix, paraText: r.text, engineDelta: r.engineDelta };
|
|
119
|
-
if (units.format === "xlsx") return buildDurable(units, { location: { sheetIndex: r.sheetIndex, sheetName: r.sheetName, range: r.range } });
|
|
120
|
-
return buildDurable(units, { location: { slideIndex: r.slideIndex, slideId: r.slideId, shapeId: r.shapeId ?? undefined, shapeName: r.shapeName, cell: d.cell || undefined }, quote: d.quote, prefix: d.prefix, suffix: d.suffix, hitKind: d.quote ? undefined : "shape" });
|
|
194
|
+
/**
|
|
195
|
+
* An HTML file that is not saved as UTF-8 (PLAN-html R10, R18): 400 with the fixed sentence as it is and `code` (not_utf8 or the
|
|
196
|
+
* reason). The page shows the sentence; the UTF-8 copy it offers is a CLI action (`onionskin open` prints its command: src/html-utf8.js).
|
|
197
|
+
*/
|
|
198
|
+
function encodingRefusal(real, r) {
|
|
199
|
+
return new HttpError(400, r.error, { code: r.code === "not_utf8" ? "not_utf8" : "encoding" });
|
|
121
200
|
}
|
|
122
201
|
|
|
202
|
+
/** The person's pending HTML text edits a Send or "Save my edits" carries (web/src/html-view.js createEdits): checked fields only. */
|
|
203
|
+
const HTML_EDITS_MAX = 2000;
|
|
204
|
+
const HTML_EDIT_TEXT_MAX = 1_000_000; // one text node's new text; the whole request is held to REQUEST_MAX (src/server.js)
|
|
205
|
+
export function cleanHtmlEdits(list) {
|
|
206
|
+
if (list == null) return [];
|
|
207
|
+
if (!Array.isArray(list) || list.length > HTML_EDITS_MAX) throw new HttpError(400, `htmlEdits must be a list of at most ${HTML_EDITS_MAX} edits`, { code: "bad_edits" });
|
|
208
|
+
return list.map((e) => {
|
|
209
|
+
const ok = e && typeof e === "object" && Number.isSafeInteger(e.nodeId) && e.nodeId >= 0 && typeof e.versionSha === "string" && /^[0-9a-f]{64}$/.test(e.versionSha) && typeof e.baseTextHash === "string" && /^[0-9a-f]{14}$/.test(e.baseTextHash) && typeof e.newText === "string" && e.newText.length <= HTML_EDIT_TEXT_MAX;
|
|
210
|
+
if (!ok) throw new HttpError(400, "an HTML text edit must be {nodeId, versionSha, baseTextHash, newText}", { code: "bad_edits" });
|
|
211
|
+
return { nodeId: e.nodeId, versionSha: e.versionSha, baseTextHash: e.baseTextHash, newText: e.newText };
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
/** 409 version_changed: the page's edits were made on a version that is not the file on disk now (it reloads the view and places them again). */
|
|
215
|
+
const VERSION_CHANGED = "The file changed since this page loaded it, so your text edits were not written. The page shows the new version and puts your edits back where it can find their text; check them, then Send again.";
|
|
216
|
+
/** The write check refused the person's text edits: 409 for an edit made on other text (the page reloads), 422 for the others; nothing was written. */
|
|
217
|
+
function htmlEditRefusal(c) {
|
|
218
|
+
if (c.code === "edit_moved") return new HttpError(409, VERSION_CHANGED, { code: "version_changed" });
|
|
219
|
+
if (c.code === "check_failed" || c.failed) return new HttpError(503, `Your text edits could not be checked: ${c.reason || c.why || "the check did not finish"}. Nothing was written; your edits are still in this page. Try again in a moment.`, { code: "check_failed" });
|
|
220
|
+
return new HttpError(422, `Your text edits were not written: ${c.reason}.`, { code: c.code, ...(c.reasons ? { reasons: c.reasons } : {}) });
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// The tasks the anchor work does for a request (src/anchor-work.js), and how a result is matched to its task: by the task's own content,
|
|
224
|
+
// so a comment changed while the worker ran finds no answer and goes out weaker instead of with another one's anchor.
|
|
225
|
+
const buildTask = (c) => ({ op: "build", anchor: c.anchor, pin: c.kind === "pin" });
|
|
226
|
+
const reanchorTask = (c) => ({ op: "reanchor", c: { durable: c.durable ?? null, anchor: c.durable ? null : c.anchor } });
|
|
227
|
+
const editAnchor = (e) => ({ location: e.location, quote: e.after, ...(e.cut ? { cut: true } : {}) });
|
|
228
|
+
const cardsTask = (results, ix, run) => ({ op: "cards", list: results.suggestions, units: ix.units, index: ix.index, reviewId: run.batch.batch_id });
|
|
229
|
+
// a quick fix's Use: the page anchor's location and text only (its kind of selection does not change where the sentence is)
|
|
230
|
+
const useTask = (a, rec, id) => ({ op: "use", anchor: { location: a.location, quote: a.quote, prefix: a.prefix, suffix: a.suffix }, rec: { quote: rec.quote, before: rec.before, after: rec.after }, id });
|
|
231
|
+
const taskKey = (t) => JSON.stringify(t);
|
|
232
|
+
/** A task's result that can be used: not missing, not an error of its own. A null anchor (the unit is not in the file) is a result. */
|
|
233
|
+
const usable = (r) => r !== undefined && !(r && typeof r === "object" && "error" in r && Object.keys(r).length === 1);
|
|
234
|
+
|
|
123
235
|
/**
|
|
124
236
|
* All session state and transitions, with no HTTP in it.
|
|
125
237
|
* States (derived): idle, listening, working, queued, stranded, closed. poll results add timeout and superseded.
|
|
@@ -128,17 +240,31 @@ function reanchor(units, c) {
|
|
|
128
240
|
* - Send while working: comments only, kept as `queued`; released to the next poller after `done`.
|
|
129
241
|
* - Lease expiry or Take back: `delivered` is withdrawn, the document unlocks.
|
|
130
242
|
*/
|
|
243
|
+
// What one request may spend scanning a whole reading for moved anchors (src/text.js scanBudget): about 0.1 to 0.3 s here. Fifty moved
|
|
244
|
+
// comments on a Word document of 26,000 paragraphs took 10 s before. Past it, the rest resolve as moved or removed, and an anchor being
|
|
245
|
+
// made is made without the text it cannot pay for (src/xlsx.js xlsxDurable). The server's routes make and resolve anchors in the anchor
|
|
246
|
+
// worker (src/anchor-worker.js, ten times this budget, within a time limit); the synchronous methods (Send, done, revert, resolve, Use of
|
|
247
|
+
// a quick fix) do it on the caller's thread under this budget.
|
|
248
|
+
export const SCAN_BUDGET = 40_000_000;
|
|
249
|
+
const REFUSED_KEPT = 64;
|
|
250
|
+
const REFUSED_MS = 2 * 60_000;
|
|
251
|
+
|
|
131
252
|
export class Sessions extends EventEmitter {
|
|
132
253
|
// delivered run -> styleChangedOutside() when it went out; memory only, so a run reloaded after a restart has none and is not flagged
|
|
133
254
|
#styleSeen = new WeakMap();
|
|
134
255
|
// delivered run -> guideChangedOutside() when it went out (the same idea for voice-guide.json); taking it also arms the guide's guard
|
|
135
256
|
#guideSeen = new WeakMap();
|
|
136
|
-
|
|
257
|
+
// the last outline built for an agent Onionskin started: {key, sha, all} (one entry; ceiling: two such agents in turn build twice)
|
|
258
|
+
#outline = null;
|
|
259
|
+
// `anchors`: the anchor worker's call (src/offload.js anchorsOffThread); a test passes its own to control when the work answers
|
|
260
|
+
constructor({ now = Date.now, anchors = anchorsOffThread } = {}) {
|
|
137
261
|
super();
|
|
138
262
|
this.now = now;
|
|
263
|
+
this.anchors = anchors;
|
|
139
264
|
this.map = new Map(); // key -> session
|
|
140
265
|
this.createdCopies = new Set(); // absolute paths of conflict copies THIS server wrote: the only paths `showCopy` will reveal
|
|
141
266
|
this.editors = new Map(); // key -> id of the one browser tab that may edit (in memory only: a tab's id lives as long as its page)
|
|
267
|
+
this.handovers = new Map(); // key -> {tab, until}: the tab another tab just took editing from (memory only), see handOver()
|
|
142
268
|
this.briefed = new Map(); // key -> Set of holders that already received a complete batch in this session (memory only: a restarted server briefs again)
|
|
143
269
|
// key -> how the file got from one version to the next, oldest first: {from, to, route: "apply", ops, partial, anchors} for an agent's
|
|
144
270
|
// apply, {from, to, route: "commit"} for a commit, {from, to, route: "merge", stable} for a pass of the user's merge. The merge reads it
|
|
@@ -157,12 +283,69 @@ export class Sessions extends EventEmitter {
|
|
|
157
283
|
if (s.closed) return false; // a closed session has no editor; the tab that reopens it claims it
|
|
158
284
|
if (cur && cur !== tab && !takeover) return false;
|
|
159
285
|
if (cur !== tab) {
|
|
286
|
+
if (cur) this.handovers.set(key, { tab: cur, until: this.now() + HANDOVER_MS }); // the tab that lost editing may hand over its unsaved words once
|
|
160
287
|
this.editors.set(key, tab);
|
|
161
288
|
this.emit("change", s, "state");
|
|
162
289
|
}
|
|
163
290
|
return true;
|
|
164
291
|
}
|
|
165
292
|
|
|
293
|
+
/**
|
|
294
|
+
* The tab that just lost editing to another ("Edit here instead") hands over the words it had not saved yet: once, within HANDOVER_MS, and
|
|
295
|
+
* only that tab. Nothing another tab saved is replaced: a comment it made is added as a draft; words it typed into a draft nobody changed since
|
|
296
|
+
* it last saved it (`base`, per comment) replace it; words typed into one the store holds otherwise become a new draft next to it; its note is merged into the stored one from the note it started from (`base`, src/notes.js mergeNote),
|
|
297
|
+
* and the tab that edits now gets it as a note the server gave back (its stale save cannot overwrite it). Refused (409 `no_handover`) for
|
|
298
|
+
* any other tab or later, and like a draft save past a cap (413, nothing changes). The page that cannot hand over keeps the words visible
|
|
299
|
+
* with Copy my comments (web/src/App.jsx).
|
|
300
|
+
*/
|
|
301
|
+
// key -> (tab -> {sent, seq}): the note the newest draft save of each tab carried, and the session `seq` it stored at (memory only, the last 8 tabs). A tab that hands over a note the server already took
|
|
302
|
+
// from it (its save's answer was lost, so it did not know) hands over nothing new: whatever happened to that note since is not its loss
|
|
303
|
+
#noteFrom = new Map();
|
|
304
|
+
handOver(key, tab, { comments, note, base, baseSeq, noteRev } = {}) {
|
|
305
|
+
const s = this.get(key);
|
|
306
|
+
const h = this.handovers.get(key);
|
|
307
|
+
if (!h || h.tab !== tab || h.until < this.now()) throw new HttpError(409, "This tab no longer edits the file.", { code: "no_handover" });
|
|
308
|
+
if (!Array.isArray(comments)) throw new HttpError(400, "comments must be an array");
|
|
309
|
+
const byId = new Map(s.comments.map((c) => [c.id, c]));
|
|
310
|
+
const next = [...s.comments];
|
|
311
|
+
for (const c of comments.filter((x) => x && typeof x === "object")) {
|
|
312
|
+
const id = cutText(String(c.id), 64);
|
|
313
|
+
const old = byId.get(id);
|
|
314
|
+
const text = userText(c.text, COMMENT_TEXT_MAX);
|
|
315
|
+
const del = s.deletedDrafts?.[id];
|
|
316
|
+
if (!old && del === text) continue; // deleted in the tab that edits now, with these very words: it stays deleted
|
|
317
|
+
if (!old) next.push(this.#draft(c, del === undefined ? id : cutText(`${id}~h${crypto.randomBytes(3).toString("hex")}`, 64), null)); // deleted there, with other words: these come back as a new draft
|
|
318
|
+
else if (text === old.text) continue;
|
|
319
|
+
else if (old.status === "draft" && !old.answer && typeof c.base === "string" && userText(c.base, COMMENT_TEXT_MAX) === old.text) next[next.indexOf(old)] = this.#draft(c, id, old); // nobody changed it since: the newer words replace it
|
|
320
|
+
else next.push(this.#draft(c, cutText(`${id}~h${crypto.randomBytes(3).toString("hex")}`, 64), null));
|
|
321
|
+
}
|
|
322
|
+
if (next.length > COMMENTS_MAX) throw tooMany();
|
|
323
|
+
for (const c of next) if (c.status === "draft" && jsonSize(c) > COMMENT_MAX) throw tooLarge("This draft");
|
|
324
|
+
let text = s.note;
|
|
325
|
+
const from = this.#noteFrom.get(key);
|
|
326
|
+
const last = from?.get(tab);
|
|
327
|
+
const already = typeof note === "string" && (last?.sent === userText(note, NOTE_MAX + 1) || this.#noteTaken(s, noteRev, tab)); // or a Send of that tab took it
|
|
328
|
+
if (typeof note === "string" && note !== s.note && !already) {
|
|
329
|
+
// what it started from: the note the page's box is based on (`base`) when that came from a state at or after that tab's last save
|
|
330
|
+
// (`baseSeq`: the page took that save's answer, or something newer, as a Take back's); else the note that save carried (its answer
|
|
331
|
+
// was lost, so the box is that note and what was typed since); with no save of that tab, what the page says
|
|
332
|
+
const fresh = typeof base === "string" && Number.isSafeInteger(baseSeq) && (!last || baseSeq >= last.seq);
|
|
333
|
+
const from = fresh ? base : last ? last.sent : typeof base === "string" ? base : "";
|
|
334
|
+
const mine = noteOf(note);
|
|
335
|
+
// the box was emptied here since (a Send took the note that tab started from): only what that tab added after it is handed over
|
|
336
|
+
text = !s.note && from && mine.startsWith(from) ? mine.slice(from.length).replace(/^\s+/, "") : mergeNote(from, s.note, mine);
|
|
337
|
+
}
|
|
338
|
+
if ([...text].length > NOTE_MAX && [...text].length > [...s.note].length) throw noteTooLong(true);
|
|
339
|
+
const before = s.note;
|
|
340
|
+
this.#commitDraft(s, { comments: next, note: before }, null, tab, true);
|
|
341
|
+
this.handovers.delete(key);
|
|
342
|
+
if (text !== before) {
|
|
343
|
+
this.#giveNote(s, text);
|
|
344
|
+
this.#changed(s, "draft");
|
|
345
|
+
} else this.emit("change", s, "draft");
|
|
346
|
+
return s;
|
|
347
|
+
}
|
|
348
|
+
|
|
166
349
|
/** The tab's event stream ended (closed, reloaded): the file has no editor until a tab claims it. */
|
|
167
350
|
releaseEditor(key, tab) {
|
|
168
351
|
if (this.editors.get(key) !== tab) return;
|
|
@@ -178,8 +361,50 @@ export class Sessions extends EventEmitter {
|
|
|
178
361
|
}
|
|
179
362
|
|
|
180
363
|
// ---- open ------------------------------------------------------------------------------------------------
|
|
181
|
-
/**
|
|
182
|
-
|
|
364
|
+
/**
|
|
365
|
+
* open() as the server's routes call it: the file is first read in a worker thread with a time and memory limit (src/offload.js
|
|
366
|
+
* readOffThread: the package check and every part the server reads later), so a slow or hostile file never stalls the server, and a
|
|
367
|
+
* file no reader accepts is refused here, with the reason, instead of failing later at Send.
|
|
368
|
+
*/
|
|
369
|
+
async openAsync(input, opts = {}) {
|
|
370
|
+
const { real, format } = this.#checkInput(input);
|
|
371
|
+
// A file that changes between the worker's reading and open()'s own (a swap, a save in progress) is read in the worker again, once:
|
|
372
|
+
// open() never takes bytes the worker has not read, so nothing on this thread reads a version past the readers' caps.
|
|
373
|
+
for (let attempt = 0; ; attempt++) {
|
|
374
|
+
const bytes = readCapped(real);
|
|
375
|
+
const sha = sha256(bytes);
|
|
376
|
+
const known = this.#refused.get(`${format}:${sha}`);
|
|
377
|
+
const r = known && this.now() - known.at < REFUSED_MS ? known : await readOffThread(bytes, format);
|
|
378
|
+
if (r.ok) readInCaps(format, r.sha);
|
|
379
|
+
if (!r.ok) {
|
|
380
|
+
// the same bytes are refused again at once (an agent's polls fall back to opening a file that is not open, again and again)
|
|
381
|
+
if (r.stage !== "busy") this.#refuse(`${format}:${sha}`, r);
|
|
382
|
+
if (r.stage === "encoding") throw encodingRefusal(real, r);
|
|
383
|
+
if (r.stage === "invalid") throw new HttpError(400, `Not a valid .${format}: ${r.error}`);
|
|
384
|
+
if (r.stage === "busy") throw new HttpError(503, `Onionskin is busy opening other files: ${r.error}. Try again in a minute.`, { code: "busy" });
|
|
385
|
+
const hint = r.stage === "timeout" || r.stage === "memory" ? " It may be damaged, or made to stall programs that read it." : "";
|
|
386
|
+
throw new HttpError(400, `This file could not be opened: ${r.error}.${hint}`, { code: r.stage === "timeout" ? "open_timeout" : "unreadable" });
|
|
387
|
+
}
|
|
388
|
+
try {
|
|
389
|
+
return this.open(input, { ...opts, checkedSha: r.sha, onlyChecked: true });
|
|
390
|
+
} catch (err) {
|
|
391
|
+
if (err?.extra?.code !== "changed_while_opening" || attempt >= 1) throw err;
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
// "<format>:<sha256>" -> what the open worker said about those bytes when it refused them, for REFUSED_MS (a refusal for time may not hold once the
|
|
397
|
+
// Mac is less busy); at most REFUSED_KEPT, oldest dropped first
|
|
398
|
+
#refused = new Map();
|
|
399
|
+
#refuse(id, r) {
|
|
400
|
+
if (r.at) return; // already a remembered answer
|
|
401
|
+
this.#refused.delete(id);
|
|
402
|
+
this.#refused.set(id, { ok: false, stage: r.stage, error: r.error, code: r.code, copyable: r.copyable, at: this.now() });
|
|
403
|
+
if (this.#refused.size > REFUSED_KEPT) this.#refused.delete(this.#refused.keys().next().value);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** The path checks of open(): the real path, its format and its lstat. Throws HttpError. */
|
|
407
|
+
#checkInput(input) {
|
|
183
408
|
let real;
|
|
184
409
|
try {
|
|
185
410
|
real = fs.realpathSync(path.resolve(input));
|
|
@@ -190,23 +415,36 @@ export class Sessions extends EventEmitter {
|
|
|
190
415
|
if (hasControlChars(real)) throw new HttpError(400, "This file's path contains control or bidirectional-text characters. Rename or move the file and open it again.", { code: "unsafe_path" });
|
|
191
416
|
const ext = path.extname(real).toLowerCase();
|
|
192
417
|
if (REFUSED_EXTENSIONS.has(ext)) throw new HttpError(400, `${ext} (macro-enabled) files are refused`);
|
|
193
|
-
|
|
194
|
-
|
|
418
|
+
if (Object.hasOwn(REFUSED_PAGES, ext)) throw new HttpError(400, REFUSED_PAGES[ext], { code: "refused_kind" });
|
|
419
|
+
const format = formatOfExt(ext);
|
|
420
|
+
if (!format) throw new HttpError(400, ONLY_FORMATS);
|
|
195
421
|
let st;
|
|
196
422
|
try {
|
|
197
423
|
st = checkRegularOwned(real);
|
|
198
424
|
} catch (err) {
|
|
199
425
|
throw new HttpError(400, err.message);
|
|
200
426
|
}
|
|
427
|
+
return { real, format, st };
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/** `opts.agent === "background"`: the person opened it from a terminal, so the server's background agent works on it (stored on the session). */
|
|
431
|
+
open(input, opts = {}) {
|
|
432
|
+
const { real, format, st } = this.#checkInput(input);
|
|
201
433
|
const bytes = readCapped(real);
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
434
|
+
const sha = sha256(bytes);
|
|
435
|
+
// openAsync already read these exact bytes in a worker; for it, other bytes (the file changed since) go back to the worker. Any other
|
|
436
|
+
// caller (tests, tools) gets the package check here.
|
|
437
|
+
if (opts.onlyChecked && opts.checkedSha !== sha) throw new HttpError(409, "The file kept changing while Onionskin opened it. Open it again once it is saved.", { code: "changed_while_opening" });
|
|
438
|
+
if (opts.checkedSha !== sha) {
|
|
439
|
+
try {
|
|
440
|
+
validatePackage(bytes, format);
|
|
441
|
+
} catch (err) {
|
|
442
|
+
if (err instanceof HtmlEncodingError) throw encodingRefusal(real, { error: err.message, code: err.code, copyable: err.copyable });
|
|
443
|
+
throw new HttpError(400, `Not a valid .${format}: ${err.message}`);
|
|
444
|
+
}
|
|
206
445
|
}
|
|
207
446
|
const key = sessionKey(real);
|
|
208
447
|
let s = this.map.get(key);
|
|
209
|
-
const sha = sha256(bytes);
|
|
210
448
|
if (!s) {
|
|
211
449
|
s = this.#load(key) || this.#fresh(key, real, format, sha);
|
|
212
450
|
s.file = real;
|
|
@@ -245,7 +483,7 @@ export class Sessions extends EventEmitter {
|
|
|
245
483
|
const j = JSON.parse(fs.readFileSync(f, "utf8"));
|
|
246
484
|
const s = { ...this.#fresh(key, j.file, j.format, j.baseSha), ...j, pollers: [] };
|
|
247
485
|
// a hard kill can leave work in flight that no result will ever settle
|
|
248
|
-
if (s.delivered?.stage === "checking") s.delivered.stage = "writing";
|
|
486
|
+
if (s.delivered?.stage === "checking") (s.delivered.stage = "writing"), (s.delivered.checkingAt = null);
|
|
249
487
|
if (s.delivered?.local) {
|
|
250
488
|
// the local runner died with the server. Not yet writing: the batch waits for an agent, as if handed back. Writing: the lease
|
|
251
489
|
// expires at the first tick, which records what was written (it needs the snapshots, which open() sets after this)
|
|
@@ -269,8 +507,128 @@ export class Sessions extends EventEmitter {
|
|
|
269
507
|
}
|
|
270
508
|
|
|
271
509
|
#save(s) {
|
|
510
|
+
const { raw, cut } = this.#encode(s);
|
|
511
|
+
if (cut) Object.assign(s, cut); // in memory too
|
|
512
|
+
writeStateFile(path.join(sub("sessions"), `${s.key}.json`), raw);
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
/**
|
|
516
|
+
* The session file's text for `s`: {raw, cut}. The one cap on a session file (src/retain.js SESSION_MAX, in bytes): past it every
|
|
517
|
+
* comment's page anchor and every typed edit are cut further (`cut`: those lists, for the caller to keep); nothing is dropped. Each part
|
|
518
|
+
* is cut where it is stored, so this is a last line. `raw` can still be past the cap; a change the page asked for is then refused (#fits).
|
|
519
|
+
*/
|
|
520
|
+
#encode(s) {
|
|
272
521
|
const { pollers, snapshots, ...rest } = s;
|
|
273
|
-
|
|
522
|
+
const raw = JSON.stringify(rest);
|
|
523
|
+
if (Buffer.byteLength(raw) <= SESSION_MAX) return { raw, cut: null };
|
|
524
|
+
const cut = {
|
|
525
|
+
comments: rest.comments.map((c) => capComment({ ...c, anchor: c.anchor && { ...c.anchor, cut: true, quote: cutText(c.anchor.quote, 2000), paraText: cutText(c.anchor.paraText, 2000) } })),
|
|
526
|
+
edits: rest.edits.map((e) => ({ ...e, before: cutText(e.before, 2000), after: cutText(e.after, 2000), cut: true })),
|
|
527
|
+
};
|
|
528
|
+
return { raw: JSON.stringify({ ...rest, ...cut }), cut };
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
/** Whether the session with `patch` fits its file once cut (#encode): checked before a change the page asked for touches anything. */
|
|
532
|
+
#fits(s, patch) {
|
|
533
|
+
return Buffer.byteLength(this.#encode({ ...s, ...patch }).raw) <= SESSION_MAX;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
/** The page's typed edits since the last Send, as they are kept for the next batch: each field cut (src/retain.js), at most EDITS_MAX in all. */
|
|
537
|
+
#editsToTake(s, edits) {
|
|
538
|
+
return (Array.isArray(edits) ? edits.slice(0, Math.max(0, EDITS_MAX - s.edits.length)) : []).map(cleanEdit);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// ---- anchor work: the anchor worker, or this thread (src/anchor-work.js) ------------------------------------------------
|
|
542
|
+
/**
|
|
543
|
+
* The results of `tasks` on the version `sha` (`bytes`: a Buffer, or a function that returns it). With `prepared` (what the anchor
|
|
544
|
+
* worker answered for the same tasks, src/session.js #offThread) they are taken from it, and nothing is read here: a task it has no
|
|
545
|
+
* answer for (its work did not finish, or it was for another version) reads as not done. Without, they are worked out on this thread
|
|
546
|
+
* under the request's budget, as the synchronous methods always did. Returns {read, outline, at(i)}.
|
|
547
|
+
*/
|
|
548
|
+
#results(format, bytes, sha, tasks, prepared, { check = false } = {}) {
|
|
549
|
+
if (prepared) {
|
|
550
|
+
const ok = prepared.ok && prepared.sha === sha;
|
|
551
|
+
return { read: ok ? prepared.read : null, outline: ok ? prepared.outline : null, at: (i) => (ok ? prepared.map.get(taskKey(tasks[i])) : undefined) };
|
|
552
|
+
}
|
|
553
|
+
// an HTML file is read in workers only (the parser is not on this thread): without the worker's answer the work is not done, and each
|
|
554
|
+
// caller makes its anchors weaker, as when the worker did not finish
|
|
555
|
+
if (format === "html") return { read: null, outline: null, at: () => undefined };
|
|
556
|
+
const r = anchorWork({ format, bytes, sha, check, tasks, budget: SCAN_BUDGET });
|
|
557
|
+
return { read: r.read, outline: null, at: (i) => r.results[i] };
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/** Bytes about to become the file: what their reading found. Past a cap 422 (CapError); not read in time and never read whole before: 503. */
|
|
561
|
+
#readVerdict(work, format, sha, refusal) {
|
|
562
|
+
const read = work.read;
|
|
563
|
+
if (read?.ok) return readInCaps(format, sha);
|
|
564
|
+
if (read?.cap) throw new CapError(read.cap);
|
|
565
|
+
if (read?.invalid) throw new HttpError(422, `${refusal}: ${untrusted(read.invalid, 200)}`, { code: "invalid_package" });
|
|
566
|
+
if (verified.has(`${format}:${sha}`)) return; // the work did not finish, but this version was read whole within the caps before
|
|
567
|
+
throw new HttpError(503, "Onionskin could not finish reading the file in time, so nothing was sent. Your comments are kept. Try again in a moment.", { code: "read_timeout" });
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
/** `job` ({source, check, tasks, outline}) in the anchor worker: {ok: true, sha, read, outline, map: task key -> result} or {ok: false, why}. */
|
|
571
|
+
async #offThread(format, job) {
|
|
572
|
+
const r = await this.anchors({ format, ...job });
|
|
573
|
+
if (!r.ok) return { ok: false, why: r.failed };
|
|
574
|
+
const map = new Map();
|
|
575
|
+
job.tasks.forEach((t, i) => map.set(taskKey(t), r.r.results[i]));
|
|
576
|
+
return { ok: true, sha: r.sha, read: r.r.read, outline: r.r.outline ?? null, map };
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
// the async methods of one file run in turn (another file's never wait for them); synchronous ones keep their own order
|
|
580
|
+
#chains = new Map();
|
|
581
|
+
#serial(key, fn) {
|
|
582
|
+
const prev = this.#chains.get(key) || Promise.resolve();
|
|
583
|
+
const run = prev.then(fn, fn);
|
|
584
|
+
const tail = run.then(() => {}, () => {});
|
|
585
|
+
this.#chains.set(key, tail);
|
|
586
|
+
tail.then(() => this.#chains.get(key) === tail && this.#chains.delete(key));
|
|
587
|
+
return run;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* `fn` (a synchronous change the page asks for: a draft save, Take back, an answer) in this file's turn, as the server's routes call it.
|
|
592
|
+
* One that arrives while a Send waits for the anchor worker is applied after that Send commits, against what it committed, as when Send
|
|
593
|
+
* was synchronous. Never called from inside a turn (it would wait for itself).
|
|
594
|
+
*/
|
|
595
|
+
inTurn(key, fn) {
|
|
596
|
+
this.get(key);
|
|
597
|
+
return this.#serial(key, () => fn());
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/** setDraft() in this file's turn (the route POST /api/draft). */
|
|
601
|
+
setDraftAsync(key, args) {
|
|
602
|
+
return this.inTurn(key, () => this.setDraft(key, args));
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
/**
|
|
606
|
+
* Resolves once every request of this file that arrived before now is answered, so a page that loads (or reloads) during a Send starts
|
|
607
|
+
* from what it committed, never from the middle of it. Bounded: a Send plans its worker job at most twice, each within the worker's time
|
|
608
|
+
* limit; past that the page gets the state as it is then.
|
|
609
|
+
*/
|
|
610
|
+
async settled(key, ms = 2 * anchorTimeoutMs() + 5_000) {
|
|
611
|
+
const tail = this.#chains.get(key);
|
|
612
|
+
if (!tail) return;
|
|
613
|
+
let timer;
|
|
614
|
+
await Promise.race([tail, new Promise((r) => (timer = setTimeout(r, ms)))]);
|
|
615
|
+
clearTimeout(timer);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
/**
|
|
619
|
+
* An agent Onionskin started polls while a batch waits for it (src/server.js /cli/poll): the outline its delivery carries is built in
|
|
620
|
+
* the anchor worker first, so the delivery finds it (#withOutline's memo) instead of reading the file on this thread.
|
|
621
|
+
*/
|
|
622
|
+
async outlineAhead(key, holder) {
|
|
623
|
+
const s = this.get(key);
|
|
624
|
+
if (!isAgentHolder(holder) || !s.pending || s.delivered || s.pending.instructions?.review || s.pending.instructions?.voice_request) return;
|
|
625
|
+
const r = await anchorsOffThread({ format: s.format, source: { path: s.file }, tasks: [], outline: true });
|
|
626
|
+
if (r.ok && r.r.read.ok && r.r.outline) this.#outline = { key: s.key, sha: r.sha, all: r.r.outline };
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/** An agent Onionskin started is waiting for this file: its delivery carries the outline, which the anchor worker then builds too. */
|
|
630
|
+
#wantsOutline(s) {
|
|
631
|
+
return s.pollers.some((p) => isAgentHolder(p.holder));
|
|
274
632
|
}
|
|
275
633
|
|
|
276
634
|
get(key) {
|
|
@@ -325,6 +683,9 @@ export class Sessions extends EventEmitter {
|
|
|
325
683
|
strandedCommand: state === "stranded" ? `${PRODUCT.bin} poll ${shellQuote(s.file)}` : null,
|
|
326
684
|
startCommand: state === "stranded" ? `${PRODUCT.bin} ${shellQuote(s.file)}` : null, // what a person types in a terminal to start an agent on this file
|
|
327
685
|
seq: s.seq,
|
|
686
|
+
// per tab, the newest Send (or quick fix's Use) of it that committed: {tab: {rev, noteRev}}. A page whose Send got no answer learns
|
|
687
|
+
// from it whether that Send went through (web/src/App.jsx checkSent)
|
|
688
|
+
sent: s.sent || {},
|
|
328
689
|
editor: this.editors.get(s.key) || null,
|
|
329
690
|
conflict: s.conflict || null,
|
|
330
691
|
canReveal: canReveal(),
|
|
@@ -343,31 +704,198 @@ export class Sessions extends EventEmitter {
|
|
|
343
704
|
}
|
|
344
705
|
|
|
345
706
|
// ---- drafts ----------------------------------------------------------------------------------------------
|
|
346
|
-
|
|
347
|
-
|
|
707
|
+
// key -> {tab, rev}: the newest save or Send a tab numbered (memory only: a reloaded page is a new tab and counts again). Saves and Sends
|
|
708
|
+
// run in turn (inTurn), but two requests of one tab can still arrive out of order: an autosave sent just before Send was pressed can land
|
|
709
|
+
// after the Send committed, and would put the note the Send took back in the box. One numbered at or below what the tab already sent
|
|
710
|
+
// changes nothing (test/send-race.test.js).
|
|
711
|
+
#revs = new Map();
|
|
712
|
+
// s.sent = {tab: {rev, noteRev}}: per tab (the last SENT_TABS), the newest numbered Send (or Use) of it that committed, and the revision
|
|
713
|
+
// of its note box the newest such Send took. Stored with the session, so a page still asking whether its Send went through gets the right
|
|
714
|
+
// answer after the server restarts. A save of that tab whose note is that revision or older (its save on the way out while the Send was in
|
|
715
|
+
// flight) leaves the note alone, and so does its handover: a note a Send delivered is never put back.
|
|
716
|
+
#sentRev(s, { rev, tab, noteRev }, took = false) {
|
|
717
|
+
if (!Number.isSafeInteger(rev) || typeof tab !== "string") return;
|
|
718
|
+
const all = s.sent && typeof s.sent === "object" ? s.sent : {};
|
|
719
|
+
const last = all[tab];
|
|
720
|
+
if (last && last.rev > rev) return;
|
|
721
|
+
const { [tab]: _, ...rest } = all;
|
|
722
|
+
const entry = { rev, noteRev: took && Number.isSafeInteger(noteRev) ? noteRev : last?.noteRev ?? null };
|
|
723
|
+
s.sent = Object.fromEntries([...Object.entries(rest), [tab, entry]].slice(-SENT_TABS));
|
|
724
|
+
}
|
|
725
|
+
/** Whether `noteRev` of `tab` is a note a committed Send of that tab already took (see s.sent). */
|
|
726
|
+
#noteTaken(s, noteRev, tab) {
|
|
727
|
+
const last = typeof tab === "string" ? s.sent?.[tab] : null;
|
|
728
|
+
return Number.isSafeInteger(noteRev) && Number.isSafeInteger(last?.noteRev) && noteRev <= last.noteRev;
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
/**
|
|
732
|
+
* Browser replaces its list of comments/pins and the overall note. Server-owned fields (status, results) are kept: a sent, queued or
|
|
733
|
+
* finished comment is as the server holds it, whatever words the page sends for it. `note` absent (not a string): the stored note is
|
|
734
|
+
* kept (the page's save while a Send is in flight, which took the note). `rev` (optional): see #revs. Every field is rebuilt from what
|
|
735
|
+
* Onionskin reads (src/retain.js). Refused (413) before anything changes: more than COMMENTS_MAX comments, a note past NOTE_MAX, a
|
|
736
|
+
* comment past COMMENT_MAX after its cuts, a session past SESSION_MAX after the writer's. Nothing is dropped or cut without a word.
|
|
737
|
+
*/
|
|
738
|
+
setDraft(key, { comments, note, rev, tab, noteRev, seen, noteSeen } = {}) {
|
|
348
739
|
const s = this.get(key);
|
|
349
740
|
if (!Array.isArray(comments)) throw new HttpError(400, "comments must be an array");
|
|
741
|
+
if (this.#overtaken(s, rev, tab, true)) return s;
|
|
742
|
+
this.#commitDraft(s, this.#draftPlan(s, { comments, note: this.#noteTaken(s, noteRev, tab) ? undefined : note, seen, noteSeen, tab }), rev, tab);
|
|
743
|
+
if (typeof note === "string" && tab) {
|
|
744
|
+
const m = this.#noteFrom.get(key) || new Map();
|
|
745
|
+
m.delete(tab);
|
|
746
|
+
m.set(tab, { sent: userText(note, NOTE_MAX + 1), seq: s.seq, used: !this.#noteTaken(s, noteRev, tab) }); // see handOver and #draftPlan
|
|
747
|
+
if (m.size > 8) m.delete(m.keys().next().value); // the tabs of one file: a handful
|
|
748
|
+
this.#noteFrom.set(key, m);
|
|
749
|
+
}
|
|
750
|
+
return s;
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
/** Whether this tab already numbered a newer save or Send than `rev` (`orSame`: one numbered the same too). See #revs. */
|
|
754
|
+
#overtaken(s, rev, tab, orSame = false) {
|
|
755
|
+
if (!Number.isSafeInteger(rev)) return false;
|
|
756
|
+
const last = this.#revs.get(s.key);
|
|
757
|
+
return !!last && last.tab === tab && (orSame ? rev <= last.rev : rev < last.rev);
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/**
|
|
761
|
+
* What a draft save of `comments` and `note` would store, {comments, note}, with every limit checked and nothing changed. `note` absent:
|
|
762
|
+
* the stored note. A note past NOTE_MAX is refused, except one not longer than the stored note (Take back can join notes past it: the
|
|
763
|
+
* page saves the box it was given, and may shorten it).
|
|
764
|
+
*/
|
|
765
|
+
#draftPlan(s, { comments, note, seen, noteSeen, tab }) {
|
|
766
|
+
if (!Array.isArray(comments)) throw new HttpError(400, "comments must be an array");
|
|
767
|
+
const list = comments.filter((c) => c && typeof c === "object");
|
|
768
|
+
if (list.length > COMMENTS_MAX) throw tooMany();
|
|
769
|
+
const long = typeof note === "string" && note.length > NOTE_MAX && note.length <= s.note.length;
|
|
770
|
+
let text = typeof note !== "string" ? s.note : long ? userText(note, s.note.length) : noteOf(note);
|
|
771
|
+
// the server gave a note back to the box (#giveNote) that the page had not seen when it sent this (`seen`: the newest session `seq` it
|
|
772
|
+
// took): the page's note is merged into it, never put over it
|
|
773
|
+
// (`noteSeen`: the newest state whose note the page's box took or is based on; a page can see a state and keep its own words)
|
|
774
|
+
const given = s.noteGiven;
|
|
775
|
+
const ns = Number.isSafeInteger(noteSeen) ? noteSeen : seen;
|
|
776
|
+
if (typeof note === "string" && given && Number.isSafeInteger(ns) && ns <= given.seq) {
|
|
777
|
+
// from the note the server held before the first give the page had not seen (`steps`: each give; a session stored before them has
|
|
778
|
+
// the first one's only); or, when a save of this tab was merged in since (its answer still on its way), from the note that save
|
|
779
|
+
// carried: the box is that note and what was typed since, and the stored note already has that save's words
|
|
780
|
+
// (ceiling: the last 50 gives; past that, a page that saw none of the kept ones merges from the oldest note, as before `steps`)
|
|
781
|
+
const hit = given.steps?.find((g) => g.seq >= ns);
|
|
782
|
+
const first = hit && !(given.cut && hit === given.steps[0]) ? hit : given;
|
|
783
|
+
const last = this.#noteFrom.get(s.key)?.get(tab);
|
|
784
|
+
text = mergeNote(last?.used && last.seq > first.seq ? last.sent : first.before, s.note, text);
|
|
785
|
+
}
|
|
350
786
|
const prev = new Map(s.comments.map((c) => [c.id, c]));
|
|
351
|
-
|
|
352
|
-
const
|
|
353
|
-
const
|
|
354
|
-
|
|
355
|
-
kind: ["text", "element", "pin"].includes(c.kind) ? c.kind : "element",
|
|
356
|
-
text: userText(c.text),
|
|
357
|
-
anchor: c.anchor && typeof c.anchor === "object" ? c.anchor : null,
|
|
358
|
-
pin: c.pin && typeof c.pin === "object" ? cleanPin(c.pin) : null,
|
|
359
|
-
pinNo: Number.isInteger(c.pinNo) ? c.pinNo : null,
|
|
360
|
-
};
|
|
361
|
-
if (old && old.status !== "draft") return { ...old, text: old.text, kind: old.kind }; // sent/done comments are frozen
|
|
362
|
-
// an answered question: the words, the question and the answer stay as the person left them (the page cannot change them)
|
|
363
|
-
const keep = old?.skipLocal ? { skipLocal: true } : {}; // a reverted local run: the next Send of this comment goes to the agent
|
|
364
|
-
if (old?.localHint && userText(c.text) === old.text) keep.localHint = old.localHint; // the words changed: the hint no longer applies
|
|
365
|
-
if (old?.answer) return { ...base, text: old.text, status: "draft", durable: old.durable || null, question: old.question, answer: old.answer, ...keep };
|
|
366
|
-
return { ...base, status: "draft", durable: old?.durable || null, ...keep };
|
|
787
|
+
const next = list.map((c) => {
|
|
788
|
+
const id = cutText(String(c.id), 64);
|
|
789
|
+
const old = prev.get(id);
|
|
790
|
+
return old && old.status !== "draft" ? old : this.#draft(c, id, old); // sent/queued/finished comments are frozen
|
|
367
791
|
});
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
792
|
+
// a comment on its way to the agent (in a waiting batch, or queued) stays whatever list a save carries: a save never changes what a
|
|
793
|
+
// Send delivers (a page that had not seen it yet, after a reload, does not take it out of the batch). Nor does a save delete a draft
|
|
794
|
+
// another tab wrote after the state its page took (`seen`): the page did not know of it (a window that handed over its words, the save
|
|
795
|
+
// of a page that was left landing after the reloaded page loaded)
|
|
796
|
+
const named = new Set(next.map((c) => c.id));
|
|
797
|
+
const unseen = (c) => c.status === "draft" && Number.isSafeInteger(seen) && c.wroteBy && c.wroteBy.tab !== tab && c.wroteBy.seq > seen;
|
|
798
|
+
for (const c of s.comments) if ((c.status === "sent" || c.status === "queued" || unseen(c)) && !named.has(c.id)) next.push(c);
|
|
799
|
+
if (next.length > COMMENTS_MAX) throw tooMany();
|
|
800
|
+
for (const c of next) if (c.status === "draft" && jsonSize(c) > COMMENT_MAX) throw tooLarge("This draft");
|
|
801
|
+
return { comments: next, note: text, seen: Number.isSafeInteger(noteSeen) ? noteSeen : seen };
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
/**
|
|
805
|
+
* The server puts `text` in the note box itself (Take back, a run that stopped with comments queued behind it, the Revert of a run on this
|
|
806
|
+
* Mac). Until a save shows the page has seen it, a save of the note from the page is merged into it (#draftPlan): the note it had then is kept.
|
|
807
|
+
*/
|
|
808
|
+
#giveNote(s, text) {
|
|
809
|
+
if (text === s.note) return;
|
|
810
|
+
const step = { seq: s.seq, before: s.note };
|
|
811
|
+
if (s.noteGiven) {
|
|
812
|
+
const steps = [...(s.noteGiven.steps || []), step];
|
|
813
|
+
s.noteGiven = { ...s.noteGiven, seq: s.seq, steps: steps.slice(-50), ...(steps.length > 50 || !s.noteGiven.steps ? { cut: true } : {}) };
|
|
814
|
+
} else s.noteGiven = { ...step, steps: [step] };
|
|
815
|
+
s.note = text;
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
/** Stores a plan of #draftPlan as a draft save: refused (413) past the session cap before anything changes. */
|
|
819
|
+
#commitDraft(s, plan, rev, tab, quiet = false) {
|
|
820
|
+
// each draft remembers which tab last wrote its words, and when (#draftPlan: a save does not delete one its page has not seen)
|
|
821
|
+
const prev = new Map(s.comments.map((c) => [c.id, c]));
|
|
822
|
+
for (const c of plan.comments) {
|
|
823
|
+
if (c.status !== "draft") continue;
|
|
824
|
+
const old = prev.get(c.id);
|
|
825
|
+
if (!old || old.status !== "draft" || old.text !== c.text) c.wroteBy = { tab: tab ?? null, seq: s.seq + 1 };
|
|
826
|
+
else if (old.wroteBy && c !== old) c.wroteBy = old.wroteBy;
|
|
827
|
+
}
|
|
828
|
+
// the words of drafts this save deletes, by id (the last DELETED_KEPT): a window handing over words afterwards does not bring them back
|
|
829
|
+
const kept = new Set(plan.comments.map((c) => c.id));
|
|
830
|
+
const gone = [...prev.values()].filter((c) => c.status === "draft" && !kept.has(c.id));
|
|
831
|
+
const deleted = gone.length ? Object.fromEntries([...Object.entries(s.deletedDrafts || {}), ...gone.map((c) => [c.id, c.text])].slice(-DELETED_KEPT)) : s.deletedDrafts;
|
|
832
|
+
const patch = { comments: plan.comments, note: plan.note, seq: s.seq + 1, ...(deleted ? { deletedDrafts: deleted } : {}) };
|
|
833
|
+
const { raw, cut } = this.#encode({ ...s, ...patch });
|
|
834
|
+
if (Buffer.byteLength(raw) > SESSION_MAX) throw tooLarge("This draft");
|
|
835
|
+
if (Number.isSafeInteger(rev) && !this.#overtaken(s, rev, tab)) this.#revs.set(s.key, { tab, rev }); // never lowered
|
|
836
|
+
Object.assign(s, patch, cut || {});
|
|
837
|
+
if (s.noteGiven && Number.isSafeInteger(plan.seen) && plan.seen > s.noteGiven.seq) delete s.noteGiven; // the page has seen it
|
|
838
|
+
writeStateFile(path.join(sub("sessions"), `${s.key}.json`), raw);
|
|
839
|
+
// `quiet`: a Send's (or a Use's) save, which its commit announces; an event for the moment between would show its note still in the box
|
|
840
|
+
if (!quiet) this.emit("change", s, "draft");
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
/** One draft as stored, from the page's comment `c` (`old`: the stored draft of the same id, if any). */
|
|
844
|
+
#draft(c, id, old) {
|
|
845
|
+
const base = {
|
|
846
|
+
id,
|
|
847
|
+
kind: ["text", "element", "pin"].includes(c.kind) ? c.kind : "element",
|
|
848
|
+
text: userText(c.text, COMMENT_TEXT_MAX),
|
|
849
|
+
anchor: cleanPageAnchor(c.anchor), // the fields the server and the page read, each cut (src/retain.js)
|
|
850
|
+
pin: c.pin && typeof c.pin === "object" ? cleanPin(c.pin) : null,
|
|
851
|
+
pinNo: Number.isInteger(c.pinNo) ? c.pinNo : null,
|
|
852
|
+
};
|
|
853
|
+
// an answered question: the words, the question and the answer stay as the person left them (the page cannot change them)
|
|
854
|
+
const keep = old?.skipLocal ? { skipLocal: true } : {}; // a reverted local run: the next Send of this comment goes to the agent
|
|
855
|
+
if (old?.localHint && base.text === old.text) keep.localHint = old.localHint; // the words changed: the hint no longer applies
|
|
856
|
+
if (old?.answer) return capComment({ ...base, text: old.text, status: "draft", durable: old.durable || null, question: old.question, answer: old.answer, ...keep });
|
|
857
|
+
return capComment({ ...base, status: "draft", durable: old?.durable || null, ...keep });
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
/**
|
|
861
|
+
* What a Send (or a quick fix's Use: `useId`, the stamp it delivers) stores before it commits, as #draftPlan: nothing changed.
|
|
862
|
+
* Invariant: a Send commits exactly its own payload, whatever draft save is numbered before or after it. Normally its list and note
|
|
863
|
+
* replace the stored ones, as a draft save does. When this tab already numbered a newer save (one sent after the Send was pressed whose
|
|
864
|
+
* body arrived first, as a page's visibility-change save can), that save is the stored list and the Send is authoritative only for what
|
|
865
|
+
* it delivers: the drafts it names (their words as the Send carried them) and its note. Everything else stays as the newer save left it:
|
|
866
|
+
* a comment it added is a draft, one it deleted stays deleted (the page cannot delete what a Send in flight carries), its note (none:
|
|
867
|
+
* it leaves the note to the Send). A Use delivers only its stamp, so then it changes nothing else, not the note either.
|
|
868
|
+
*/
|
|
869
|
+
#sentPlan(s, { comments, note, rev, tab, seen, noteSeen, sends }, useId = null) {
|
|
870
|
+
const list = Array.isArray(comments) ? comments : s.comments;
|
|
871
|
+
// `sends` (optional): the ids the page holds read only for this Send, the drafts it means to deliver; a comment the page still showed as
|
|
872
|
+
// queued or sent when Send was pressed (and could be typed into after) is not delivered by it
|
|
873
|
+
const named = Array.isArray(sends) ? new Set(sends.slice(0, COMMENTS_MAX).map((id) => cutText(String(id), 64))) : null;
|
|
874
|
+
if (!this.#overtaken(s, rev, tab)) {
|
|
875
|
+
const plan = this.#draftPlan(s, { comments: list, note: typeof note === "string" ? note : s.note, seen, noteSeen, tab });
|
|
876
|
+
return named && useId == null ? { ...plan, only: named } : plan;
|
|
877
|
+
}
|
|
878
|
+
const own = this.#draftPlan(s, { comments: list.filter((c) => c && typeof c === "object" && (useId == null || String(c.id) === useId)), note: useId == null && typeof note === "string" ? note : s.note, seen, noteSeen, tab });
|
|
879
|
+
const mine = new Map(own.comments.map((c) => [c.id, c]));
|
|
880
|
+
// `only`: the drafts this Send delivers (the others are the newer save's, and stay drafts)
|
|
881
|
+
return { comments: s.comments.map((c) => (c.status === "draft" && mine.has(c.id) ? mine.get(c.id) : c)), note: own.note, only: new Set([...mine.keys()].filter((id) => !named || useId != null || named.has(id))) };
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
/** The drafts of `comments` a Send with `plan` delivers: all of them, or those it names when a newer save was stored first. */
|
|
885
|
+
#delivers(comments, plan) {
|
|
886
|
+
return comments.filter((c) => c.status === "draft" && (!plan.only || plan.only.has(c.id)));
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
/**
|
|
890
|
+
* The 413 refusals of a Send, on the state its plan would make (nothing changed yet): the bytes past the cap, a note that joined with
|
|
891
|
+
* the one waiting (the batch an agent has not taken, or the queued note during a run) would be past NOTE_MAX, a session that would not
|
|
892
|
+
* fit its file. Run before the worker is asked and again at the commit, against the state then.
|
|
893
|
+
*/
|
|
894
|
+
#sendLimits(s, plan, payload) {
|
|
895
|
+
if (payload.bytes?.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
|
|
896
|
+
if (s.delivered) joinNotes(s.queuedNote, plan.note);
|
|
897
|
+
else joinNotes(s.pending && !readOnlyKind(s.pending) ? s.pending.instructions?.overall_note || "" : "", plan.note);
|
|
898
|
+
if (!this.#fits(s, { comments: plan.comments, edits: [...s.edits, ...this.#editsToTake(s, payload.edits)] })) throw tooLarge("This Send");
|
|
371
899
|
}
|
|
372
900
|
|
|
373
901
|
// ---- send ------------------------------------------------------------------------------------------------
|
|
@@ -376,58 +904,211 @@ export class Sessions extends EventEmitter {
|
|
|
376
904
|
* Compare-and-swap on the disk hash. Returns the session.
|
|
377
905
|
*/
|
|
378
906
|
send(key, payload) {
|
|
907
|
+
return scanBudget(SCAN_BUDGET, () => this.#send(key, payload));
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
/**
|
|
911
|
+
* send() as the server's route calls it: the reading of the bytes that become the file and every anchor of the batch are worked out
|
|
912
|
+
* first, in the warm anchor worker with a time limit (src/offload.js anchorsOffThread), then send() commits with what came back and
|
|
913
|
+
* reads nothing itself. Work that did not finish leaves its comments weaker (src/anchor-work.js weakDurable), never lost; bytes it
|
|
914
|
+
* could not read in time are refused with "try again" unless that version was read whole before.
|
|
915
|
+
* Invariant: a Send and the page's draft saves, Take back and answers of one file run in turn (inTurn), as when Send was synchronous.
|
|
916
|
+
* The Send saves its list and note when its turn comes, then commits exactly that; whatever arrives meanwhile waits and is applied after
|
|
917
|
+
* the commit, against what it committed. A page loaded meanwhile waits for it too (settled).
|
|
918
|
+
*/
|
|
919
|
+
async sendAsync(key, payload) {
|
|
920
|
+
const s = this.get(key);
|
|
921
|
+
const hasBytes = !!payload.bytes?.length;
|
|
922
|
+
if (s.closed || (s.delivered && hasBytes)) return this.send(key, payload); // refused there, before anything changes
|
|
923
|
+
return this.#serial(key, async () => {
|
|
924
|
+
// refused there, before anything changes: closed, or locked
|
|
925
|
+
if (s.closed || (s.delivered && hasBytes)) return this.send(key, payload);
|
|
926
|
+
// nothing is stored before the commit: the worker works on what the Send would store, every limit is checked on it first (a refusal
|
|
927
|
+
// here changes nothing) and again at the commit
|
|
928
|
+
let prepared = null;
|
|
929
|
+
let html = {};
|
|
930
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
931
|
+
const queued = !!s.delivered;
|
|
932
|
+
const plan = this.#sentPlan(s, payload);
|
|
933
|
+
this.#sendLimits(s, plan, payload);
|
|
934
|
+
if (s.format === "html") this.#htmlSendChecks(s, payload);
|
|
935
|
+
const job = this.#sendJob(s, payload, plan);
|
|
936
|
+
// an HTML file that is not queued: the person's text edits are written through the write check, and every anchor is made on the
|
|
937
|
+
// version it was made on, then moved onto the bytes this Send writes (#htmlSendWork)
|
|
938
|
+
if (s.format === "html" && !s.delivered) ({ prepared, html } = await this.#htmlSendWork(s, payload, job));
|
|
939
|
+
else prepared = job ? await this.#offThread(s.format, job) : null;
|
|
940
|
+
if (!!s.delivered === queued) break; // a run started or ended while it worked: plan again once, for the other case
|
|
941
|
+
}
|
|
942
|
+
return this.send(key, { ...payload, ...html, prepared });
|
|
943
|
+
});
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
/** An HTML Send's own refusals, before any work: no bytes from the browser ever (PLAN-html 3.1), no text edits during a run (D5). */
|
|
947
|
+
#htmlSendChecks(s, payload) {
|
|
948
|
+
if (payload.bytes?.length && !payload.htmlChecked) throw new HttpError(400, "an HTML file's bytes never come from the page; send its text edits", { code: "no_bytes_for_html" });
|
|
949
|
+
if (payload.htmlEdits?.length && s.delivered) throw new HttpError(423, s.delivered.local ? "the document is locked while a change is made on this Mac" : "the document is locked while the agent works", { code: "locked" });
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
/**
|
|
953
|
+
* The work of an HTML Send that is not queued, off this thread (the parser exists only in workers):
|
|
954
|
+
* 1. the person's text edits (R9), when there are any: on the version the page loaded, which must be the file on disk (else 409
|
|
955
|
+
* version_changed: the page reloads the view and places its edits again), applied by the write check in a worker (src/html-write.js
|
|
956
|
+
* checkedHtmlWrite "edits": one write, the proof, the inventory). Its bytes become the file at the commit (#takePage), which checks
|
|
957
|
+
* the disk hash again after these awaits (R12).
|
|
958
|
+
* 2. the anchors: a page anchor names a node of ONE version (`location.version`, a node id means nothing in another). An anchor made on
|
|
959
|
+
* the version this Send writes is built there; one made on another version (a draft that outlived a reload, the typed edits'
|
|
960
|
+
* places when this Send changes the bytes) is built on that version first (the file on disk or a stored snapshot), then moved onto
|
|
961
|
+
* the bytes this Send writes (reanchor), as a queued comment is moved onto the agent's file at done.
|
|
962
|
+
* Returns {prepared (what #offThread returns, keyed by this Send's own tasks), html: {bytes, htmlChecked} or {}}.
|
|
963
|
+
*/
|
|
964
|
+
async #htmlSendWork(s, payload, job) {
|
|
965
|
+
let disk;
|
|
966
|
+
try {
|
|
967
|
+
checkRegularOwned(s.file);
|
|
968
|
+
disk = readCapped(s.file);
|
|
969
|
+
} catch (err) {
|
|
970
|
+
throw new HttpError(409, `cannot read the file: ${err.message}`, { code: "gone" });
|
|
971
|
+
}
|
|
972
|
+
const diskSha = sha256(disk);
|
|
973
|
+
let target = { bytes: disk, sha: diskSha };
|
|
974
|
+
let html = {};
|
|
975
|
+
const edits = cleanHtmlEdits(payload.htmlEdits);
|
|
976
|
+
if (edits.length) {
|
|
977
|
+
if (payload.baseSha !== diskSha || edits.some((e) => e.versionSha !== diskSha)) throw new HttpError(409, VERSION_CHANGED, { code: "version_changed", diskSha });
|
|
978
|
+
const c = await htmlWriteOffThread({ before: disk, edits: edits.map((e) => ({ nodeId: e.nodeId, text: e.newText, baseTextHash: e.baseTextHash })) });
|
|
979
|
+
if (!c.ok) throw htmlEditRefusal(c);
|
|
980
|
+
const bytes = Buffer.from(c.bytes.buffer, c.bytes.byteOffset, c.bytes.byteLength);
|
|
981
|
+
target = { bytes, sha: sha256(bytes) };
|
|
982
|
+
html = { bytes, htmlChecked: { from: diskSha, to: target.sha } };
|
|
983
|
+
}
|
|
984
|
+
const prepared = await this.#htmlAnchorWork(s, job.tasks, target, { disk, diskSha, check: true, outline: job.outline });
|
|
985
|
+
return { prepared, html };
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
/**
|
|
989
|
+
* `tasks` (build, reanchor and view tasks of one request) on the version `target` ({bytes, sha}), in the anchor worker, with every build
|
|
990
|
+
* task whose page anchor was made on another version (`location.version`) built on that version first (`known.disk` when it is the file
|
|
991
|
+
* on disk, else a stored snapshot) and moved onto `target`. One made on a version no longer stored is moved as a weak anchor (its quote
|
|
992
|
+
* searched). Returns what #offThread returns, its map keyed by the tasks given.
|
|
993
|
+
*/
|
|
994
|
+
async #htmlAnchorWork(s, tasks, target, { disk = null, diskSha = null, check = false, outline = false } = {}) {
|
|
995
|
+
const elsewhere = new Map(); // version -> [task index]
|
|
996
|
+
tasks.forEach((t, i) => {
|
|
997
|
+
const v = t.op === "build" ? t.anchor?.location?.version : null;
|
|
998
|
+
if (v && v !== target.sha) (elsewhere.get(v) || elsewhere.set(v, []).get(v)).push(i);
|
|
999
|
+
});
|
|
1000
|
+
const moved = new Map(); // task index -> the durable made on its own version (null: not there; undefined: not worked out)
|
|
1001
|
+
for (const [v, list] of elsewhere) {
|
|
1002
|
+
const source = v === diskSha && disk ? { bytes: disk } : s.snapshots.has(v) ? { path: s.snapshots.blob(v), sha: v } : null;
|
|
1003
|
+
if (!source) continue;
|
|
1004
|
+
const r = await this.#offThread("html", { source, tasks: list.map((i) => tasks[i]) });
|
|
1005
|
+
if (r.ok && r.read?.ok) list.forEach((i) => moved.set(i, r.map.get(taskKey(tasks[i]))));
|
|
1006
|
+
}
|
|
1007
|
+
const asked = tasks.map((t, i) => {
|
|
1008
|
+
if (!elsewhere.size || !(t.op === "build" && t.anchor?.location?.version && t.anchor.location.version !== target.sha)) return t;
|
|
1009
|
+
const d = moved.get(i);
|
|
1010
|
+
return { op: "reanchor", c: { durable: usable(d) && d ? d : weakDurable("html", t.anchor), anchor: null } };
|
|
1011
|
+
});
|
|
1012
|
+
const r = await this.#offThread("html", { source: { bytes: target.bytes }, check, tasks: asked, outline });
|
|
1013
|
+
if (!r.ok) return r;
|
|
1014
|
+
const map = new Map();
|
|
1015
|
+
asked.forEach((t, i) => map.set(taskKey(tasks[i]), r.map.get(taskKey(t))));
|
|
1016
|
+
return { ...r, sha: target.sha, map };
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
/** What a Send asks of the anchor worker (the same tasks #send makes, for the list it would store), or null when it needs nothing read. */
|
|
1020
|
+
#sendJob(s, payload, plan) {
|
|
1021
|
+
if (s.delivered) {
|
|
1022
|
+
const need = this.#delivers(plan.comments, plan).filter((c) => !c.durable);
|
|
1023
|
+
if (!need.length || !s.snapshots.has(s.delivered.preSha)) return null;
|
|
1024
|
+
return { source: { path: s.snapshots.blob(s.delivered.preSha), sha: s.delivered.preSha }, tasks: need.map(buildTask) };
|
|
1025
|
+
}
|
|
1026
|
+
const { tasks } = this.#sendTasks(s, this.#editsToTake(s, payload.edits), payload.viewState, plan);
|
|
1027
|
+
const bytes = payload.bytes?.length ? payload.bytes : null;
|
|
1028
|
+
return { source: bytes ? { bytes } : { path: s.file }, check: true, tasks, outline: this.#wantsOutline(s) };
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
/** The tasks of a Send that is not queued: every comment that goes out (drafts, and those of a batch still waiting), the typed edits without an anchor yet, the view. */
|
|
1032
|
+
#sendTasks(s, newEdits, viewState, plan = { comments: s.comments }) {
|
|
1033
|
+
const go = new Set(this.#delivers(plan.comments, plan));
|
|
1034
|
+
const eligible = plan.comments.filter((c) => go.has(c) || c.status === "sent");
|
|
1035
|
+
const edits = [...s.edits, ...newEdits].filter((e) => !e.durable);
|
|
1036
|
+
const vs = cleanView(viewState);
|
|
1037
|
+
return { eligible, edits, vs, tasks: [...eligible.map(buildTask), ...edits.map((e) => ({ op: "build", anchor: editAnchor(e), pin: false })), ...(vs ? [{ op: "view", vs }] : [])] };
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
/**
|
|
1041
|
+
* The commit of a Send: the Send's own list and note are saved first (#sentPlan), then every draft goes out with the note. A batch still
|
|
1042
|
+
* waiting for an agent (not delivered) is replaced by one that carries its comments and its note, then this one's; during a run the
|
|
1043
|
+
* drafts and the note are queued after those already queued. Every 413 (#sendLimits) is refused before anything changes.
|
|
1044
|
+
*/
|
|
1045
|
+
#send(key, payload) {
|
|
379
1046
|
const s = this.get(key);
|
|
380
1047
|
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
1048
|
+
if (s.format === "html") this.#htmlSendChecks(s, payload);
|
|
381
1049
|
const hasBytes = !!payload.bytes?.length;
|
|
382
1050
|
if (s.delivered && hasBytes) throw new HttpError(423, s.delivered.local ? "the document is locked while a change is made on this Mac" : "the document is locked while the agent works", { code: "locked" }); // before touching any state
|
|
383
|
-
|
|
1051
|
+
// the Send's own list and note (#sentPlan), every limit checked on them before anything changes; then stored as a draft save is, so a
|
|
1052
|
+
// refusal after this (the file changed, could not be read in time) leaves them as editable drafts
|
|
1053
|
+
const plan = this.#sentPlan(s, payload);
|
|
1054
|
+
this.#sendLimits(s, plan, payload);
|
|
1055
|
+
this.#commitDraft(s, plan, payload.rev, payload.tab, true);
|
|
1056
|
+
try {
|
|
1057
|
+
return this.#sendCommit(s, payload, plan);
|
|
1058
|
+
} catch (err) {
|
|
1059
|
+
this.emit("change", s, "draft"); // refused after the save: the page's tabs see the drafts it stored (a Send that commits says so once, at the end)
|
|
1060
|
+
throw err;
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
/** The rest of #send, once its list and note are stored: the batch (or the queue) and the commit. */
|
|
1065
|
+
#sendCommit(s, payload, plan) {
|
|
384
1066
|
if (s.delivered) {
|
|
385
1067
|
// queue: comments and note only. A new comment gets its durable anchor from the bytes the page shows (the pre-agent snapshot),
|
|
386
1068
|
// never from the disk, which the agent may already have changed; done() then moves it onto the post-agent file.
|
|
387
|
-
const
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
}
|
|
395
|
-
if (units) {
|
|
396
|
-
for (const c of fresh) {
|
|
397
|
-
if (c.durable) continue;
|
|
398
|
-
c.durable = buildDurable(units, c.anchor);
|
|
399
|
-
if (units.format === "pptx" && c.kind === "pin" && c.durable?.shapeId != null) c.durable.pinFallback = buildDurable(units, { location: { slideIndex: c.durable.slideIndex, slideId: c.durable.slideId } });
|
|
400
|
-
}
|
|
401
|
-
}
|
|
1069
|
+
const queuedNote = joinNotes(s.queuedNote, s.note); // before anything changes
|
|
1070
|
+
const fresh = this.#delivers(s.comments, plan);
|
|
1071
|
+
const need = fresh.filter((c) => !c.durable);
|
|
1072
|
+
if (need.length) {
|
|
1073
|
+
// no snapshot, or work that did not finish: the anchor stays unset, and done() builds it from the post-agent file
|
|
1074
|
+
const work = this.#results(s.format, () => s.snapshots.read(s.delivered.preSha), s.delivered.preSha, need.map(buildTask), payload.prepared);
|
|
1075
|
+
if (work.read?.ok) need.forEach((c, i) => (c.durable = usable(work.at(i)) ? work.at(i) : null));
|
|
402
1076
|
}
|
|
403
1077
|
for (const c of fresh) c.status = "queued";
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
s.note = "";
|
|
408
|
-
}
|
|
1078
|
+
// the note belongs to the queued batch now; it leaves the box so the next Send does not repeat it
|
|
1079
|
+
s.queuedNote = queuedNote;
|
|
1080
|
+
s.note = "";
|
|
409
1081
|
if (fresh.length || s.queuedNote) s.queued = true; // a Send with nothing new queues nothing (no empty batch after this one)
|
|
1082
|
+
this.#sentRev(s, payload, true);
|
|
410
1083
|
this.#changed(s);
|
|
411
1084
|
this.#voice(s, { sent: fresh });
|
|
412
1085
|
return s;
|
|
413
1086
|
}
|
|
414
|
-
|
|
1087
|
+
// a batch still waiting for an agent is replaced: its comments go again (they are "sent"), and so does its note, first
|
|
1088
|
+
const waiting = s.pending && !readOnlyKind(s.pending) ? s.pending.instructions?.overall_note || "" : "";
|
|
1089
|
+
const note = joinNotes(waiting, s.note); // before anything is written
|
|
1090
|
+
// the bytes that become the file are read whole before anything is written (one past a reader's cap is refused: CapError, nothing
|
|
1091
|
+
// changes), and every anchor of the batch is built from them, by the anchor worker (sendAsync) or here
|
|
1092
|
+
const newEdits = this.#editsToTake(s, payload.edits);
|
|
1093
|
+
const { eligible, edits, vs, tasks } = this.#sendTasks(s, newEdits, payload.viewState, { ...plan, comments: s.comments });
|
|
1094
|
+
// what is stored must fit its file once cut (src/retain.js SESSION_MAX): checked before the file is written or anything changes
|
|
1095
|
+
if (!this.#fits(s, { comments: s.comments, edits: [...s.edits, ...newEdits] })) throw tooLarge("This Send");
|
|
1096
|
+
let work = null;
|
|
1097
|
+
const { diskSha, writtenSha, writeInfo, lastUpdate } = this.#takePage(s, payload, (b, sha) => {
|
|
1098
|
+
work = this.#results(s.format, b, sha, tasks, payload.prepared, { check: true });
|
|
1099
|
+
this.#readVerdict(work, s.format, sha, "refusing to write");
|
|
1100
|
+
});
|
|
415
1101
|
|
|
416
1102
|
// user edits since the last Send (document-derived text) accumulate until a batch is delivered
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
const
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
}
|
|
427
|
-
for (const e of s.edits) e.durable = e.durable || resolve({ location: e.location, quote: e.after });
|
|
428
|
-
s.view = this.#resolveView(s, units, payload.viewState);
|
|
429
|
-
|
|
430
|
-
const batch = buildBatch({ file: s.file, format: s.format, baseSha: writtenSha, note: s.note, comments: eligible, edits: s.edits, style: styleNote(), guide: voiceGuide() });
|
|
1103
|
+
s.edits.push(...newEdits);
|
|
1104
|
+
// a comment whose anchor work did not finish goes out weaker (src/anchor-work.js weakDurable); a PPTX pin on a shape also remembers its slide
|
|
1105
|
+
eligible.forEach((c, i) => (c.durable = usable(work.at(i)) ? work.at(i) : weakDurable(s.format, c.anchor)));
|
|
1106
|
+
edits.forEach((e, i) => (e.durable = usable(work.at(eligible.length + i)) ? work.at(eligible.length + i) : weakDurable(s.format, editAnchor(e))));
|
|
1107
|
+
const v = vs ? work.at(eligible.length + edits.length) : null;
|
|
1108
|
+
s.view = !vs ? null : usable(v) ? v : { format: s.format, raw: vs };
|
|
1109
|
+
if (work.outline) this.#outline = { key: s.key, sha: writtenSha, all: work.outline };
|
|
1110
|
+
|
|
1111
|
+
const batch = buildBatch({ file: s.file, format: s.format, baseSha: writtenSha, note, comments: eligible, edits: s.edits, style: styleNote(), guide: voiceGuide() });
|
|
431
1112
|
const sent = eligible.filter((c) => c.status === "draft");
|
|
432
1113
|
for (const c of eligible) {
|
|
433
1114
|
c.status = "sent";
|
|
@@ -444,6 +1125,7 @@ export class Sessions extends EventEmitter {
|
|
|
444
1125
|
this.#deliverIfPolled(s);
|
|
445
1126
|
delete s.quickConfirm;
|
|
446
1127
|
delete s.skipLocalNote; // it held for this Send only (the decision for its batch is made)
|
|
1128
|
+
this.#sentRev(s, payload, true);
|
|
447
1129
|
this.#changed(s);
|
|
448
1130
|
this.#voice(s, { last: lastUpdate, diskSha, writtenSha, sent });
|
|
449
1131
|
if (this.state(s) === "stranded" && !s.pollers.length) this.emit("stranded", s); // nobody took it: the server may start its background agent
|
|
@@ -453,8 +1135,8 @@ export class Sessions extends EventEmitter {
|
|
|
453
1135
|
/**
|
|
454
1136
|
* The page's bytes become the file, as at Send (and at a quick fix's Use): the disk read, the compare-and-swap against the version
|
|
455
1137
|
* the page loaded, the package check, the guarded replace, the "sent" snapshot, the new base and the person's own history update.
|
|
456
|
-
* `check(bytes)` (optional) sees the bytes that are about to be the file, after the package check and before
|
|
457
|
-
* may throw to refuse. Returns {disk, diskSha, written, writtenSha, writeInfo, lastUpdate}.
|
|
1138
|
+
* `check(bytes, sha)` (optional) sees the bytes that are about to be the file and their sha256, after the package check and before
|
|
1139
|
+
* anything is written, and may throw to refuse. Returns {disk, diskSha, written, writtenSha, writeInfo, lastUpdate}.
|
|
458
1140
|
*/
|
|
459
1141
|
#takePage(s, payload, check = null) {
|
|
460
1142
|
const hasBytes = !!payload.bytes?.length;
|
|
@@ -466,25 +1148,33 @@ export class Sessions extends EventEmitter {
|
|
|
466
1148
|
throw new HttpError(409, `cannot read the file: ${err.message}`, { code: "gone" });
|
|
467
1149
|
}
|
|
468
1150
|
const diskSha = sha256(disk);
|
|
469
|
-
|
|
470
|
-
|
|
1151
|
+
// an HTML file's text edits were applied by its write check (#htmlSendWork) to the version the page loaded: no bytes come from the page
|
|
1152
|
+
const htmlChecked = s.format === "html" ? payload.htmlChecked ?? null : null;
|
|
1153
|
+
if (diskSha !== payload.baseSha || (htmlChecked && htmlChecked.from !== diskSha)) {
|
|
1154
|
+
// an HTML page sent no bytes: nothing of its to keep as a copy (its edits stay in the page, which reloads and places them again)
|
|
1155
|
+
throw this.#outsideChange(s, { side: "user", bytes: s.format === "html" ? null : payload.bytes, diskSha, baseSha: payload.baseSha });
|
|
471
1156
|
}
|
|
472
1157
|
let written = disk;
|
|
473
1158
|
let writeInfo = null;
|
|
474
1159
|
if (hasBytes) {
|
|
475
|
-
if (payload.bytes.length > MAX_BYTES) throw new HttpError(413,
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
}
|
|
479
|
-
|
|
1160
|
+
if (payload.bytes.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
|
|
1161
|
+
if (s.format === "html") {
|
|
1162
|
+
if (htmlChecked?.to !== sha256(payload.bytes)) throw new HttpError(500, "an HTML file is written only after its write check", { code: "unchecked_write" });
|
|
1163
|
+
} else {
|
|
1164
|
+
try {
|
|
1165
|
+
validatePackage(payload.bytes, s.format);
|
|
1166
|
+
} catch (err) {
|
|
1167
|
+
throw new HttpError(422, `refusing to write: ${err.message}`, { code: "invalid_package" });
|
|
1168
|
+
}
|
|
480
1169
|
}
|
|
481
1170
|
}
|
|
482
|
-
|
|
1171
|
+
const writtenSha = hasBytes ? sha256(payload.bytes) : diskSha;
|
|
1172
|
+
if (htmlChecked && hasBytes) readInCaps("html", writtenSha); // the write check read these bytes whole within the limits
|
|
1173
|
+
if (check) check(hasBytes ? payload.bytes : disk, writtenSha);
|
|
483
1174
|
if (hasBytes) {
|
|
484
|
-
writeInfo = this.#replace(s, payload.bytes, { expectSha: diskSha, label: "before send", side: "user" });
|
|
1175
|
+
writeInfo = this.#replace(s, payload.bytes, { expectSha: diskSha, label: "before send", side: "user", htmlChecked });
|
|
485
1176
|
written = payload.bytes;
|
|
486
1177
|
}
|
|
487
|
-
const writtenSha = sha256(written);
|
|
488
1178
|
s.snapshots.add(written, "sent");
|
|
489
1179
|
s.baseSha = writtenSha;
|
|
490
1180
|
const lastUpdate = s.history?.at(-1); // learning pairs an agent run with the person's edits right after it
|
|
@@ -508,16 +1198,21 @@ export class Sessions extends EventEmitter {
|
|
|
508
1198
|
* (DOCX paragraphs, formatting and moves: src/merge.js; XLSX cells: src/merge-xlsx.js; PPTX shape text: src/merge-pptx.js) is carried onto it as ops (same engine, expect, protected and package checks
|
|
509
1199
|
* as an agent's apply) and written as the user's own update. Refused while a batch is delivered (423): the page waits and asks
|
|
510
1200
|
* again. Returns {merged, unplaced, sha, same, mineSha, copy_name}.
|
|
511
|
-
*
|
|
1201
|
+
* The reading of the three versions within the readers' caps, the plan and the engine's run happen in a worker thread with a time and
|
|
1202
|
+
* memory limit (src/offload.js mergeOffThread): the server's thread stays free whatever the document. A merge that fails or runs out of
|
|
1203
|
+
* time writes nothing: the person's version is saved as a copy next to the file and the page keeps their edits (503 merge_failed).
|
|
1204
|
+
* One merge per session at a time (another is 423 locked; the page has only one in flight).
|
|
512
1205
|
*/
|
|
513
1206
|
async mergeUserEdits(key, { baseSha, bytes, edits = [] }) {
|
|
514
1207
|
const s = this.get(key);
|
|
515
1208
|
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
516
1209
|
if (s.delivered) throw new HttpError(423, s.delivered.local ? "a change is being made on this Mac" : "the agent is working", { code: "locked" });
|
|
517
|
-
|
|
1210
|
+
if (s.merging) throw new HttpError(423, "your edits are being merged already", { code: "locked" });
|
|
1211
|
+
// an HTML view is read only during a run (PLAN-html D5): nothing is typed to merge, and its bytes never come from the page
|
|
1212
|
+
if (s.format === "html") throw new HttpError(400, "an HTML file's edits are never merged: its view is read only while the agent works", { code: "not_for_html" });
|
|
518
1213
|
if (!/^[0-9a-f]{64}$/.test(baseSha)) throw new HttpError(400, "baseSha must be a sha256"); // it names a snapshot file
|
|
519
1214
|
if (!bytes?.length) throw new HttpError(400, "no document bytes");
|
|
520
|
-
if (bytes.length > MAX_BYTES) throw new HttpError(413,
|
|
1215
|
+
if (bytes.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
|
|
521
1216
|
try {
|
|
522
1217
|
validatePackage(bytes, s.format);
|
|
523
1218
|
} catch (err) {
|
|
@@ -539,8 +1234,19 @@ export class Sessions extends EventEmitter {
|
|
|
539
1234
|
} catch {
|
|
540
1235
|
throw new HttpError(409, "the version this page started from is no longer stored", { code: "no_base" });
|
|
541
1236
|
}
|
|
542
|
-
|
|
543
|
-
|
|
1237
|
+
// the three versions are read whole within the readers' caps in the worker first (the page's version, a file swapped on disk and an
|
|
1238
|
+
// old snapshot alike): past one, the merge is refused with the reason and nothing is written
|
|
1239
|
+
let r;
|
|
1240
|
+
s.merging = true;
|
|
1241
|
+
try {
|
|
1242
|
+
r = await mergeOffThread({ format: s.format, base, mine: bytes, theirs: disk, name: path.basename(s.file), edits: editKinds(edits), agent: s.format === "xlsx" ? this.#writesBetween(s, baseSha, diskSha, disk) : null });
|
|
1243
|
+
} finally {
|
|
1244
|
+
s.merging = false;
|
|
1245
|
+
}
|
|
1246
|
+
if (r.cap) throw new CapError(r.cap);
|
|
1247
|
+
if (r.invalid) throw new HttpError(422, `refusing to merge: ${untrusted(r.invalid, 200)}`, { code: "invalid_package" });
|
|
1248
|
+
if (!r.ok) throw this.#mergeFailed(s, bytes, r);
|
|
1249
|
+
if (r.bytes) r.bytes = Buffer.from(r.bytes.buffer, r.bytes.byteOffset, r.bytes.byteLength); // a Uint8Array once it crossed threads
|
|
544
1250
|
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
545
1251
|
if (s.delivered) throw new HttpError(423, s.delivered.local ? "a change is being made on this Mac" : "the agent is working", { code: "locked" }); // a batch was delivered while the engine ran
|
|
546
1252
|
// The page shows the first HELD_SHOWN items and drops its own version when it reloads. When the list stands for more than it
|
|
@@ -570,6 +1276,24 @@ export class Sessions extends EventEmitter {
|
|
|
570
1276
|
return { same: false, merged: r.merged, unplaced: r.unplaced.slice(0, 50), sha, mineSha: sha256(bytes), copy_name: copy?.name ?? null };
|
|
571
1277
|
}
|
|
572
1278
|
|
|
1279
|
+
/**
|
|
1280
|
+
* A merge that did not finish (out of time or memory, busy, crashed) or whose result the proof refused: nothing was written. The person's version is saved next to the
|
|
1281
|
+
* file (except when Onionskin was only busy) and the answer says so in words the page shows inside its own sentence ("could not be
|
|
1282
|
+
* merged into its version (...). Your edits are still in this page: download them before you reload.").
|
|
1283
|
+
*/
|
|
1284
|
+
#mergeFailed(s, bytes, r) {
|
|
1285
|
+
if (r.failed === "busy") return new HttpError(503, "Onionskin is merging other files right now; nothing was written, so try again in a moment", { code: "merge_busy" });
|
|
1286
|
+
let copy = null;
|
|
1287
|
+
try {
|
|
1288
|
+
copy = writeSiblingCopy(s.file, bytes);
|
|
1289
|
+
this.createdCopies.add(copy.path);
|
|
1290
|
+
} catch {}
|
|
1291
|
+
// refused: the proof of the merge (src/merge-proof.js) could not account for every edit of the person's in the result; the reason
|
|
1292
|
+
// holds counts and kinds of edits, never their text
|
|
1293
|
+
const why = r.failed === "timeout" || r.failed === "memory" ? r.why : r.refused ? `the merged file could not be checked to hold every one of your edits (${r.refused})` : "the merge stopped with an error";
|
|
1294
|
+
return new HttpError(503, `${why}; nothing was written${copy ? `, and a copy of your version was saved as ${copy.name} next to the file` : ""}`, { code: "merge_failed", copy_name: copy?.name ?? null });
|
|
1295
|
+
}
|
|
1296
|
+
|
|
573
1297
|
#logWrite(s, entry) {
|
|
574
1298
|
const list = this.writes.get(s.key) || [];
|
|
575
1299
|
list.push(entry);
|
|
@@ -610,7 +1334,9 @@ export class Sessions extends EventEmitter {
|
|
|
610
1334
|
* replacement (see replaceGuarded). `expectSha` is the hash the caller compared against; a file that is different now, or becomes
|
|
611
1335
|
* different just before the rename, is not overwritten (the newer bytes are snapshotted) and the caller gets the 409 conflict.
|
|
612
1336
|
*/
|
|
613
|
-
#replace(s, bytes, { expectSha, label, side = null }) {
|
|
1337
|
+
#replace(s, bytes, { expectSha, label, side = null, htmlChecked = null }) {
|
|
1338
|
+
// R1, R12: an HTML file is written only with bytes its write check passed in a worker (agentWriteChecked); every other path refuses
|
|
1339
|
+
if (s.format === "html" && htmlChecked?.to !== sha256(bytes)) throw new HttpError(500, "an HTML file is written only after its write check", { code: "unchecked_write" });
|
|
614
1340
|
try {
|
|
615
1341
|
return replaceGuarded(s.file, bytes, {
|
|
616
1342
|
expectSha,
|
|
@@ -684,14 +1410,6 @@ export class Sessions extends EventEmitter {
|
|
|
684
1410
|
return { stale: !s.delivered && !!loadedSha && diskSha !== loadedSha, diskSha };
|
|
685
1411
|
}
|
|
686
1412
|
|
|
687
|
-
#resolveView(s, units, vs) {
|
|
688
|
-
if (!vs) return null;
|
|
689
|
-
if (units.format === "docx" && vs.format !== "docx") return { format: s.format, raw: vs };
|
|
690
|
-
if (units.format !== "docx") return viewDurable(units, vs) || { format: s.format, raw: vs };
|
|
691
|
-
const d = durableFromEngine(units.paragraphs, { blockPath: vs.blockPath }, { quote: vs.quote });
|
|
692
|
-
return { format: "docx", durable: d, offsetPx: Number(vs.offsetPx) || 0, zoom: vs.zoom ?? "fit", fraction: vs.fraction };
|
|
693
|
-
}
|
|
694
|
-
|
|
695
1413
|
// ---- poll / delivery -------------------------------------------------------------------------------------
|
|
696
1414
|
/**
|
|
697
1415
|
* Registers a poller. Returns a handle {promise, cancel}. Resolves with the batch object, or {status:'superseded'|'closed'}.
|
|
@@ -776,17 +1494,30 @@ export class Sessions extends EventEmitter {
|
|
|
776
1494
|
* written to <state>/agents/<key>/outline/outline-<batch id>.jsonl (0600, atomic, one unit per line), the only folder that agent may read,
|
|
777
1495
|
* and the batch names it. Older outline files of the session go at each such delivery, and with the folder on Close.
|
|
778
1496
|
* Any other holder, a review (it has `units`) and a voice request (it quotes only the sources the person ticked) get the batch as stored.
|
|
779
|
-
* A file that cannot be read gets no outline (the agent declines what it cannot find); an outline file that cannot be written leaves the
|
|
780
|
-
* inline part with `outline_more`.
|
|
1497
|
+
* A file that cannot be read gets no outline, and `outline_unavailable` says why (the agent declines what it cannot find); an outline file that cannot be written leaves the
|
|
1498
|
+
* inline part with `outline_more`. The outline is built in the anchor worker at Send, at done and before a poll's delivery (outlineAhead),
|
|
1499
|
+
* and found here by the file's hash. Ceiling: a delivery that finds none (a synchronous caller, or the file changed in between) reads and
|
|
1500
|
+
* parses the file on this thread, as a review does; upgrade path as in unitsOf.
|
|
781
1501
|
*/
|
|
782
1502
|
#withOutline(s, batch, holder) {
|
|
783
1503
|
if (!isAgentHolder(holder) || batch.instructions?.review || batch.instructions?.voice_request) return batch;
|
|
784
1504
|
let all;
|
|
785
1505
|
try {
|
|
786
1506
|
checkRegularOwned(s.file);
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
1507
|
+
const bytes = readCapped(s.file);
|
|
1508
|
+
const sha = sha256(bytes);
|
|
1509
|
+
// the same version's outline is built once (an agent's polls and a Send's delivery read the same file), on the memo's reading of it
|
|
1510
|
+
if (this.#outline?.key === s.key && this.#outline.sha === sha) all = this.#outline.all;
|
|
1511
|
+
// an HTML file is never parsed on this thread: its outline comes from the anchor worker (Send, done, outlineAhead) or not at all
|
|
1512
|
+
else if (s.format === "html") return { ...batch, untrusted_document_evidence: { ...batch.untrusted_document_evidence, outline_unavailable: "Onionskin did not finish reading this page's outline in time. Use the anchors and their quotes; poll again for an outline." } };
|
|
1513
|
+
else {
|
|
1514
|
+
all = outlineOf(s.format, bytes, OUTLINE_FILE_MAX, unitsFor(s.format, bytes, sha));
|
|
1515
|
+
this.#outline = { key: s.key, sha, all };
|
|
1516
|
+
}
|
|
1517
|
+
} catch (err) {
|
|
1518
|
+
// the agent is told why it has no outline (a file changed past a reader's cap behind Onionskin, or gone), in the reader's plain words
|
|
1519
|
+
const why = err instanceof CapError ? `Onionskin cannot read this file: ${err.message}.` : "Onionskin cannot read this file as it is now.";
|
|
1520
|
+
return { ...batch, untrusted_document_evidence: { ...batch.untrusted_document_evidence, outline_unavailable: why } };
|
|
790
1521
|
}
|
|
791
1522
|
const inline = withOutline(batch, all, VOICE_BATCH_MAX_CHARS, { inlineMax: UNITS_MAX });
|
|
792
1523
|
const name = `outline-${batch.batch_id}.jsonl`;
|
|
@@ -848,7 +1579,7 @@ export class Sessions extends EventEmitter {
|
|
|
848
1579
|
#queuedToDrafts(s) {
|
|
849
1580
|
if (s.queued) {
|
|
850
1581
|
for (const c of s.comments) if (c.status === "queued") c.status = "draft";
|
|
851
|
-
s
|
|
1582
|
+
this.#giveNote(s, [s.queuedNote, s.note].filter(Boolean).join("\n\n"));
|
|
852
1583
|
s.queuedNote = "";
|
|
853
1584
|
}
|
|
854
1585
|
s.queued = false;
|
|
@@ -943,7 +1674,16 @@ export class Sessions extends EventEmitter {
|
|
|
943
1674
|
|
|
944
1675
|
/** Called on a timer: a delivered batch whose lease expired goes back to pending (stranded) and the document unlocks. */
|
|
945
1676
|
tick() {
|
|
946
|
-
|
|
1677
|
+
const now = this.now();
|
|
1678
|
+
const checkMax = Math.max(LEASE_MS(), 5 * 60_000);
|
|
1679
|
+
for (const s of this.map.values()) {
|
|
1680
|
+
if (!s.delivered || s.delivered.lease.expiresAt > now) continue;
|
|
1681
|
+
if (s.delivered.stage === "checking") {
|
|
1682
|
+
const since = s.delivered.checkingAt ?? s.delivered.lease.expiresAt;
|
|
1683
|
+
if (now - since < checkMax) continue;
|
|
1684
|
+
}
|
|
1685
|
+
this.#expire(s);
|
|
1686
|
+
}
|
|
947
1687
|
}
|
|
948
1688
|
|
|
949
1689
|
/**
|
|
@@ -979,6 +1719,7 @@ export class Sessions extends EventEmitter {
|
|
|
979
1719
|
const s = this.get(key);
|
|
980
1720
|
if (!s.delivered || s.delivered.stage === stage) return;
|
|
981
1721
|
s.delivered.stage = stage;
|
|
1722
|
+
s.delivered.checkingAt = stage === "checking" ? this.now() : null;
|
|
982
1723
|
this.#changed(s);
|
|
983
1724
|
}
|
|
984
1725
|
|
|
@@ -999,11 +1740,22 @@ export class Sessions extends EventEmitter {
|
|
|
999
1740
|
if (token != null && token !== d.lease.token) throw stale();
|
|
1000
1741
|
}
|
|
1001
1742
|
|
|
1743
|
+
/**
|
|
1744
|
+
* The delivered batch, for its lease holder only (`GET /cli/batch` with a claim): holder, batch id and token are all required and checked
|
|
1745
|
+
* before anything of the batch is handed out, so `reply` and `apply` read no anchors, comments or file text for a delivery that is not theirs.
|
|
1746
|
+
*/
|
|
1747
|
+
deliveredBatch(key, claim) {
|
|
1748
|
+
const s = this.get(key);
|
|
1749
|
+
if (!s.delivered) throw new HttpError(409, "no batch is delivered for this file, so the ticket is void", { code: "no_batch" });
|
|
1750
|
+
this.#assertLease(s, { ...claim, strict: true });
|
|
1751
|
+
return s.delivered.batch;
|
|
1752
|
+
}
|
|
1753
|
+
|
|
1002
1754
|
/**
|
|
1003
1755
|
* `apply` through a live session: the single authority for the write. Accepted only while a batch is delivered (state working or
|
|
1004
1756
|
* queued: both hold a live lease) and only for the lease holder; compare-and-swap on the disk hash; renews the lease.
|
|
1005
1757
|
*/
|
|
1006
|
-
agentWrite(key, { baseSha, bytes, holder, token, batchId, label = "before agent write", route = "commit", ops = null, partial = false }) {
|
|
1758
|
+
agentWrite(key, { baseSha, bytes, holder, token, batchId, label = "before agent write", route = "commit", ops = null, partial = false, htmlChecked = null }) {
|
|
1007
1759
|
const s = this.get(key);
|
|
1008
1760
|
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
1009
1761
|
const state = this.state(s);
|
|
@@ -1014,11 +1766,20 @@ export class Sessions extends EventEmitter {
|
|
|
1014
1766
|
// `commit` does not know the hash its candidate was edited from: the base is the last hash this server wrote or served for the delivery
|
|
1015
1767
|
if (baseSha == null) baseSha = s.delivered.knownSha ?? s.delivered.preSha;
|
|
1016
1768
|
if (!bytes?.length) throw new HttpError(400, "no bytes");
|
|
1017
|
-
if (bytes.length > MAX_BYTES) throw new HttpError(413,
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
throw new HttpError(
|
|
1769
|
+
if (bytes.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
|
|
1770
|
+
const newSha = sha256(bytes);
|
|
1771
|
+
if (s.format === "html") {
|
|
1772
|
+
// checked in a worker (agentWriteChecked) for exactly these bytes; the disk hash it was checked against is compared below
|
|
1773
|
+
if (htmlChecked?.to !== newSha) throw new HttpError(500, "an HTML file is written only after its write check", { code: "unchecked_write" });
|
|
1774
|
+
} else {
|
|
1775
|
+
try {
|
|
1776
|
+
validatePackage(bytes, s.format);
|
|
1777
|
+
} catch (err) {
|
|
1778
|
+
throw new HttpError(422, `refusing to write: ${err.message}`, { code: "invalid_package" });
|
|
1779
|
+
}
|
|
1780
|
+
// read whole before anything is written, as a Send's bytes are: bytes past a reader's cap are refused with the reason (CapError, 422
|
|
1781
|
+
// too_large), never written to leave the next outline, review and done without a reading. The reading is kept for them (unitsFor).
|
|
1782
|
+
readWhole(s.format, bytes, newSha);
|
|
1022
1783
|
}
|
|
1023
1784
|
let disk;
|
|
1024
1785
|
try {
|
|
@@ -1029,9 +1790,10 @@ export class Sessions extends EventEmitter {
|
|
|
1029
1790
|
}
|
|
1030
1791
|
const diskSha = sha256(disk);
|
|
1031
1792
|
if (diskSha !== baseSha) throw this.#outsideChange(s, { side: "agent", bytes, diskSha, baseSha, message: "the file changed on disk since apply read it" });
|
|
1793
|
+
// R12: the compare-and-swap again after the check's await: the version it checked against must still be the one on disk
|
|
1794
|
+
if (s.format === "html" && htmlChecked.from !== diskSha) throw this.#outsideChange(s, { side: "agent", bytes, diskSha, baseSha: htmlChecked.from, message: "the file changed on disk while the write was being checked" });
|
|
1032
1795
|
this.setStage(key, "writing"); // only once the checks passed: a refused write does not claim to be writing
|
|
1033
|
-
const info = this.#replace(s, bytes, { expectSha: diskSha, label, side: "agent" });
|
|
1034
|
-
const newSha = sha256(bytes);
|
|
1796
|
+
const info = this.#replace(s, bytes, { expectSha: diskSha, label, side: "agent", htmlChecked });
|
|
1035
1797
|
// what the merge may rely on later: the ops `apply` says it ran (checked against the bytes when a merge needs them), or that it was a commit
|
|
1036
1798
|
const checked = route === "apply" && s.format === "xlsx" && Array.isArray(ops) && validateOps({ schema: "onionskin.ops/1", ops }).ok; // only the XLSX merge reads them
|
|
1037
1799
|
this.#logWrite(s, checked ? { from: diskSha, to: newSha, route, ops, partial: partial === true, anchors: s.delivered.batch.untrusted_document_evidence?.anchors || [] } : { from: diskSha, to: newSha, route: route === "apply" ? "apply" : "commit", ops: null });
|
|
@@ -1043,6 +1805,41 @@ export class Sessions extends EventEmitter {
|
|
|
1043
1805
|
return { ok: true, sha256: newSha, lease_expires_at: s.delivered.lease.expiresAt };
|
|
1044
1806
|
}
|
|
1045
1807
|
|
|
1808
|
+
/**
|
|
1809
|
+
* The agent's write (`/cli/agent-write`, `/cli/commit`) for every format. An HTML file's bytes first go through the write check
|
|
1810
|
+
* (src/html-write.js checkedHtmlWrite: decoding, limits, a parse, the charset rule, the inventory of active content) in a worker; the
|
|
1811
|
+
* bytes it returns (the charset declaration may be added) are then written by agentWrite, which checks the lease, the state and the
|
|
1812
|
+
* disk hash again after that await (R12). Other formats: agentWrite as it is.
|
|
1813
|
+
*/
|
|
1814
|
+
async agentWriteChecked(key, args, check = htmlWriteOffThread) {
|
|
1815
|
+
const s = this.get(key);
|
|
1816
|
+
if (s.format !== "html") return this.agentWrite(key, args);
|
|
1817
|
+
// the cheap refusals first, so a write that cannot land costs no worker; agentWrite makes every one of them again after the await
|
|
1818
|
+
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
1819
|
+
if (!s.delivered) throw new HttpError(409, `no batch is delivered to an agent (state: ${this.state(s)}); run poll and wait for the user's Send`, { code: "not_working", state: this.state(s) });
|
|
1820
|
+
this.#assertLease(s, { holder: args.holder, token: args.token, batchId: args.batchId, strict: true });
|
|
1821
|
+
if (!args.bytes?.length) throw new HttpError(400, "no bytes");
|
|
1822
|
+
let disk;
|
|
1823
|
+
try {
|
|
1824
|
+
checkRegularOwned(s.file);
|
|
1825
|
+
disk = readCapped(s.file);
|
|
1826
|
+
} catch (err) {
|
|
1827
|
+
throw new HttpError(409, `cannot read the file: ${err.message}`, { code: "gone" });
|
|
1828
|
+
}
|
|
1829
|
+
// the bytes checked are the ones whose hash agentWrite compares with the base and the disk after this await (from === disk === base)
|
|
1830
|
+
const c = await check({ before: disk, after: args.bytes });
|
|
1831
|
+
if (!c.ok) {
|
|
1832
|
+
// a refused write is part of the run's result: the page lists it with the run (the R1 sentence for active content), nothing was written
|
|
1833
|
+
if (c.code !== "check_failed" && s.delivered && !s.closed) {
|
|
1834
|
+
const p = cleanProblem({ key: `write:${c.code}`, op: args.route === "apply" ? "apply" : "commit", status: "refused", code: c.code, reason: c.reason });
|
|
1835
|
+
s.delivered.applyProblems = [...(s.delivered.applyProblems || []).filter((x) => x.key !== p.key), p].slice(0, 40);
|
|
1836
|
+
this.#changed(s);
|
|
1837
|
+
}
|
|
1838
|
+
throw new HttpError(c.code === "check_failed" ? 503 : 422, c.reason, { code: c.code, ...(c.reasons ? { reasons: c.reasons } : {}) });
|
|
1839
|
+
}
|
|
1840
|
+
return this.agentWrite(key, { ...args, bytes: c.bytes, htmlChecked: { from: sha256(disk), to: sha256(c.bytes) } });
|
|
1841
|
+
}
|
|
1842
|
+
|
|
1046
1843
|
/**
|
|
1047
1844
|
* What `apply` could not do (refused by the engine, protected, anchor or expectation mismatch), told by the lease holder after each run
|
|
1048
1845
|
* (also when nothing was written). Shown to the user with the activity entry at `done`.
|
|
@@ -1067,6 +1864,7 @@ export class Sessions extends EventEmitter {
|
|
|
1067
1864
|
if (!s.delivered) throw new HttpError(409, "no batch is in progress for this file (taken back or already finished)", { code: "no_batch" });
|
|
1068
1865
|
this.#assertLease(s, claim, { staleOnHolder: true });
|
|
1069
1866
|
this.setStage(key, "checking");
|
|
1867
|
+
this.#renew(s);
|
|
1070
1868
|
let post;
|
|
1071
1869
|
let pre;
|
|
1072
1870
|
try {
|
|
@@ -1082,7 +1880,25 @@ export class Sessions extends EventEmitter {
|
|
|
1082
1880
|
return { pre, post, format: s.format, postSha: sha256(post), batchId: s.delivered.batch.batch_id, deferSoffice: env("SOFFICE_SYNC", "") !== "1" };
|
|
1083
1881
|
}
|
|
1084
1882
|
|
|
1085
|
-
|
|
1883
|
+
/**
|
|
1884
|
+
* done() as the server calls it (src/server.js finishRun): the anchors of the comments queued during the run (moved onto the file the
|
|
1885
|
+
* agent leaves), a review's cards and the outline for a waiting agent are worked out first in the anchor worker, then done() commits
|
|
1886
|
+
* with them and reads none of it on this thread. Work that did not finish leaves those comments weaker, with a flag; never lost.
|
|
1887
|
+
*/
|
|
1888
|
+
async doneAsync(key, results, claim = {}, analysis = null, opts = {}) {
|
|
1889
|
+
return this.#serial(key, async () => {
|
|
1890
|
+
const s = this.get(key);
|
|
1891
|
+
const run = s.delivered;
|
|
1892
|
+
const tasks = [];
|
|
1893
|
+
if (run && s.queued) tasks.push(...s.comments.filter((c) => c.status === "queued").map(reanchorTask));
|
|
1894
|
+
if (run?.batch.instructions?.review && s.review?.batchId === run.batch.batch_id && results && typeof results === "object") tasks.push(cardsTask(results, s.review, run));
|
|
1895
|
+
const prepared = run && (tasks.length || this.#wantsOutline(s)) ? await this.#offThread(s.format, { source: { path: s.file }, tasks, outline: this.#wantsOutline(s) }) : null;
|
|
1896
|
+
return this.done(key, results, claim, analysis, { ...opts, prepared });
|
|
1897
|
+
});
|
|
1898
|
+
}
|
|
1899
|
+
|
|
1900
|
+
/** `inline: false` (the server): never analyze on this thread; an analysis for other bytes (the file changed while it ran) is a stated failure. */
|
|
1901
|
+
done(key, results, claim = {}, analysis = null, { inline = true, prepared = null } = {}) {
|
|
1086
1902
|
const s = this.get(key);
|
|
1087
1903
|
if (!s.delivered) throw new HttpError(409, "no batch is in progress for this file (taken back or already finished)", { code: "no_batch" });
|
|
1088
1904
|
this.#assertLease(s, claim, { staleOnHolder: true });
|
|
@@ -1096,10 +1912,10 @@ export class Sessions extends EventEmitter {
|
|
|
1096
1912
|
throw new HttpError(409, `cannot read the file: ${err.message}`);
|
|
1097
1913
|
}
|
|
1098
1914
|
const postSha = sha256(post);
|
|
1099
|
-
if (readOnlyKind(run.batch) === "voice") return this.#doneVoice(s, results, run, post, postSha);
|
|
1915
|
+
if (readOnlyKind(run.batch) === "voice") return this.#doneVoice(s, results, run, post, postSha, prepared);
|
|
1100
1916
|
// a review that wrote nothing ends with cards. One whose file changed anyway (a direct save past review_only) is finished as a run below:
|
|
1101
1917
|
// its write is flagged and can be reverted, and no card is kept
|
|
1102
|
-
if (run.batch.instructions?.review && postSha === run.preSha) return this.#doneReview(s, results, run, post);
|
|
1918
|
+
if (run.batch.instructions?.review && postSha === run.preSha) return this.#doneReview(s, results, run, post, prepared);
|
|
1103
1919
|
s.snapshots.add(post, "after agent");
|
|
1104
1920
|
|
|
1105
1921
|
// ---- plan: everything that can throw happens here, on locals; the session is not touched until the commit below ------
|
|
@@ -1113,7 +1929,14 @@ export class Sessions extends EventEmitter {
|
|
|
1113
1929
|
let found;
|
|
1114
1930
|
if (!pre) found = { flags: [{ code: "no_pre_snapshot", severity: "warn" }], checks: { node: "skipped", deep: { status: "skipped", reason: "no pre-agent snapshot" }, notice: "Checks skipped: the pre-agent version is no longer stored." }, changes: null };
|
|
1115
1931
|
else if (analysis && analysis.postSha === postSha && analysis.batchId === run.batch.batch_id) found = analysis;
|
|
1116
|
-
else
|
|
1932
|
+
else if (!inline) {
|
|
1933
|
+
const why = "the file changed while it was being checked";
|
|
1934
|
+
found = { flags: [{ code: "checks_failed", severity: "warn", detail: why }], checks: { node: "failed", deep: { status: "skipped", reason: why }, notice: `The package checks did not finish: ${why}. The file was not verified.` }, changes: null };
|
|
1935
|
+
} else if (s.format === "html") {
|
|
1936
|
+
// an HTML file is read in workers only: a caller without the worker's analysis gets it stated as not run, never as clean
|
|
1937
|
+
const why = "the checks of an HTML file run in a worker";
|
|
1938
|
+
found = { flags: [{ code: "checks_failed", severity: "warn", detail: why }], checks: { node: "failed", deep: { status: "skipped", reason: why }, notice: `The checks did not run: ${why}. The file was not verified.` }, changes: null };
|
|
1939
|
+
} else found = analyze(pre, post, s.format);
|
|
1117
1940
|
const flags = [...found.flags];
|
|
1118
1941
|
const { checks, changes } = found;
|
|
1119
1942
|
// the file must be what this server last wrote (apply/commit) or delivered; anything else changed it behind the lease (a direct save by
|
|
@@ -1154,7 +1977,8 @@ export class Sessions extends EventEmitter {
|
|
|
1154
1977
|
// style.json changed by something else while this batch was out (an agent with a shell, say): already put back by src/style.js
|
|
1155
1978
|
this.#guardFlags(run, flags);
|
|
1156
1979
|
if (results.voice_draft !== undefined) flags.push({ code: "voice_draft_not_allowed", severity: "warn" }); // only a voice batch takes a draft; nothing is stored
|
|
1157
|
-
const queuedPlan = this.#planQueued(s, post, postSha, flags);
|
|
1980
|
+
const queuedPlan = this.#planQueued(s, post, postSha, flags, prepared);
|
|
1981
|
+
if (prepared?.ok && prepared.sha === postSha && prepared.outline) this.#outline = { key: s.key, sha: postSha, all: prepared.outline };
|
|
1158
1982
|
const cleanFlags = flags.map(cleanFlag);
|
|
1159
1983
|
const { list: ignored, more: ignoredMore } = ignoredInstructions(results.document_instructions_ignored);
|
|
1160
1984
|
const entry = {
|
|
@@ -1280,27 +2104,21 @@ export class Sessions extends EventEmitter {
|
|
|
1280
2104
|
}
|
|
1281
2105
|
|
|
1282
2106
|
/** Comments queued while the agent worked become the next pending batch, anchored on the file the agent leaves. Pure planning: touches no session state; re-anchoring problems are pushed onto `flags`. */
|
|
1283
|
-
|
|
2107
|
+
// `prepared`: the anchor worker's answer for the same tasks (doneAsync); without it the anchors are moved on this thread.
|
|
2108
|
+
#planQueued(s, post, postSha, flags, prepared = null) {
|
|
1284
2109
|
let plan = null;
|
|
1285
2110
|
if (s.queued) {
|
|
1286
2111
|
const queued = s.comments.filter((c) => c.status === "queued");
|
|
1287
2112
|
if (queued.length || s.queuedNote) {
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
} catch (err) {
|
|
1292
|
-
flags.push({ code: "queued_reanchor_failed", severity: "warn", detail: err.message });
|
|
1293
|
-
}
|
|
2113
|
+
const work = this.#results(s.format, post, postSha, queued.map(reanchorTask), prepared);
|
|
2114
|
+
const read = work.read;
|
|
2115
|
+
if (!read?.ok) flags.push({ code: "queued_reanchor_failed", severity: "warn", detail: read?.invalid || read?.cap || "the anchors could not be worked out in time" });
|
|
1294
2116
|
const durables = new Map();
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
flags.push({ code: "queued_reanchor_failed", severity: "warn", comment_id: c.id, detail: err.message });
|
|
1301
|
-
}
|
|
1302
|
-
durables.set(c.id, d);
|
|
1303
|
-
}
|
|
2117
|
+
queued.forEach((c, i) => {
|
|
2118
|
+
const r = work.at(i);
|
|
2119
|
+
if (read?.ok && !usable(r)) flags.push({ code: "queued_reanchor_failed", severity: "warn", comment_id: c.id, detail: r?.error ?? "the anchor could not be worked out in time" });
|
|
2120
|
+
durables.set(c.id, read?.ok && usable(r) ? r : weakDurable(s.format, c.anchor));
|
|
2121
|
+
});
|
|
1304
2122
|
const batch = buildBatch({ file: s.file, format: s.format, baseSha: postSha, note: s.queuedNote, comments: queued.map((c) => ({ ...c, durable: durables.get(c.id) })), edits: [], style: styleNote(), guide: voiceGuide() });
|
|
1305
2123
|
plan = { queued, durables, batch };
|
|
1306
2124
|
}
|
|
@@ -1318,6 +2136,7 @@ export class Sessions extends EventEmitter {
|
|
|
1318
2136
|
const s = this.get(key);
|
|
1319
2137
|
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
1320
2138
|
if (!REVIEW_KINDS.includes(kind)) throw new HttpError(400, `kind must be one of ${REVIEW_KINDS.join(", ")}`, { code: "bad_kind" });
|
|
2139
|
+
if (s.format === "html") throw new HttpError(409, "Review first is not available for HTML files in this version. Comment on the page and press Send.", { code: "not_for_html" }); // PLAN-html 2a: the page hides the button
|
|
1321
2140
|
if (s.delivered || s.pending) throw new HttpError(409, s.delivered?.local ? "Wait for the change on this Mac to finish, then try again." : "Wait for the agent to finish, then try again.", { code: "busy" });
|
|
1322
2141
|
let disk;
|
|
1323
2142
|
try {
|
|
@@ -1334,7 +2153,7 @@ export class Sessions extends EventEmitter {
|
|
|
1334
2153
|
}
|
|
1335
2154
|
|
|
1336
2155
|
#reviewBatch(s, kind, bytes, baseSha) {
|
|
1337
|
-
const { units, more, index } = unitsOf(s.format, bytes);
|
|
2156
|
+
const { units, more, index } = unitsOf(s.format, bytes, unitsFor(s.format, bytes, baseSha));
|
|
1338
2157
|
const batch = buildBatch({ file: s.file, format: s.format, baseSha, note: "", comments: [], edits: [], review: { kind, ask: REVIEW_ASKS[kind] }, units, unitsMore: more });
|
|
1339
2158
|
s.review = { batchId: batch.batch_id, kind, units: units.map((u) => ({ id: u.id, protected: u.protected })), index };
|
|
1340
2159
|
s.decidedBatch = batch.batch_id; // never the local runner's: only an agent reads the file
|
|
@@ -1342,13 +2161,19 @@ export class Sessions extends EventEmitter {
|
|
|
1342
2161
|
}
|
|
1343
2162
|
|
|
1344
2163
|
/** `done` of a review that wrote nothing: the suggestions become cards (checked against the file), an activity entry of kind "review", no history update. */
|
|
1345
|
-
#doneReview(s, results, run, post) {
|
|
2164
|
+
#doneReview(s, results, run, post, prepared = null) {
|
|
1346
2165
|
const flags = [];
|
|
1347
2166
|
const ix = s.review?.batchId === run.batch.batch_id ? s.review : null;
|
|
1348
|
-
const
|
|
2167
|
+
const gone = (why) => ({ cards: [], dropped: (Array.isArray(results.suggestions) ? results.suggestions : []).map((_, i) => ({ index: i, reason: why })) });
|
|
2168
|
+
let checked = gone("the review's outline is gone");
|
|
2169
|
+
if (ix) {
|
|
2170
|
+
const r = this.#results(s.format, post, run.preSha, [cardsTask(results, ix, run)], prepared).at(0);
|
|
2171
|
+
checked = usable(r) && r ? r : gone("the suggestions could not be checked against the file in time");
|
|
2172
|
+
}
|
|
2173
|
+
const { cards, dropped } = checked;
|
|
1349
2174
|
for (const d of dropped) flags.push({ code: "bad_suggestion", severity: "warn", ...(d.index != null ? { index: d.index } : {}), reason: d.reason });
|
|
1350
2175
|
if (results.voice_draft !== undefined) flags.push({ code: "voice_draft_not_allowed", severity: "warn" });
|
|
1351
|
-
const queuedPlan = this.#planQueued(s, post, run.preSha, flags);
|
|
2176
|
+
const queuedPlan = this.#planQueued(s, post, run.preSha, flags, prepared);
|
|
1352
2177
|
const cleanFlags = flags.map(cleanFlag);
|
|
1353
2178
|
const { list: ignored, more: ignoredMore } = ignoredInstructions(results.document_instructions_ignored); // a review reads the whole file: its report counts most here
|
|
1354
2179
|
const entry = {
|
|
@@ -1428,8 +2253,25 @@ export class Sessions extends EventEmitter {
|
|
|
1428
2253
|
* and Revert, as for any local run. The text written is the stored suggestion, never text from the page. Every refusal happens before
|
|
1429
2254
|
* anything is written. Other comments and the overall note stay for the next Send; typed edits ride with it too.
|
|
1430
2255
|
*/
|
|
1431
|
-
|
|
2256
|
+
/**
|
|
2257
|
+
* useSuggestion() as the server's route calls it (src/suggest.js): where the suggestion goes in the bytes about to be the file, and
|
|
2258
|
+
* the one reply that writes it, are worked out in the anchor worker first. Work that did not finish writes nothing: the comment stays a
|
|
2259
|
+
* draft and says it can go to the agent, as when the sentence is not found.
|
|
2260
|
+
*/
|
|
2261
|
+
async useSuggestionAsync(key, args) {
|
|
1432
2262
|
const s = this.get(key);
|
|
2263
|
+
this.#useChecks(s, args); // refused before it waits its turn
|
|
2264
|
+
return this.#serial(key, async () => {
|
|
2265
|
+
const { id } = this.#useChecks(s, args);
|
|
2266
|
+
const { c } = this.#usePlan(s, args, id); // nothing is stored before the commit, as a Send (sendAsync)
|
|
2267
|
+
const bytes = args.bytes?.length && args.bytes.length <= MAX_BYTES ? args.bytes : null;
|
|
2268
|
+
const prepared = await this.#offThread(s.format, { source: bytes ? { bytes } : { path: s.file }, tasks: [useTask(c.anchor, args.suggestion, id)] });
|
|
2269
|
+
return this.useSuggestion(key, { ...args, prepared });
|
|
2270
|
+
});
|
|
2271
|
+
}
|
|
2272
|
+
|
|
2273
|
+
/** The refusals of a quick fix's Use that need nothing read (useSuggestion runs them first). Returns {id}. */
|
|
2274
|
+
#useChecks(s, { suggestion: rec, commentId, comments }) {
|
|
1433
2275
|
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
1434
2276
|
if (s.delivered || s.pending) throw new HttpError(409, "The agent started working on this file, so nothing was written. Ask the agent instead, or try again when it is done.", { code: "busy" });
|
|
1435
2277
|
const words = rec && Object.hasOwn(STAMP_WORDS, rec.task) && rec.task !== "translate" ? STAMP_WORDS[rec.task] : null;
|
|
@@ -1439,22 +2281,37 @@ export class Sessions extends EventEmitter {
|
|
|
1439
2281
|
const old = s.comments.find((c) => c.id === id);
|
|
1440
2282
|
const sameAnchor = (a) => a && typeof a === "object" && a.quote === rec.quote && JSON.stringify(a.location ?? null) === JSON.stringify(rec.location ?? null);
|
|
1441
2283
|
if (!mine || userText(mine.text) !== words || !sameAnchor(mine.anchor) || (old && old.status !== "draft")) throw new HttpError(409, "This comment is not the suggestion's stamp.", { code: "not_this_comment" });
|
|
2284
|
+
return { id };
|
|
2285
|
+
}
|
|
1442
2286
|
|
|
1443
|
-
|
|
2287
|
+
/** What a Use stores before it commits (#sentPlan with its stamp), the stamp as planned, and every limit checked: nothing changed. */
|
|
2288
|
+
#usePlan(s, { comments, note, edits = [], rev, tab, seen, noteSeen, bytes = null }, id) {
|
|
2289
|
+
if (bytes?.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" }); // before anything is stored, as a Send
|
|
2290
|
+
const saved = this.#sentPlan(s, { comments, note, rev, tab, seen, noteSeen }, id);
|
|
2291
|
+
const c = saved.comments.find((x) => x.id === id);
|
|
2292
|
+
if (c?.status !== "draft") throw new HttpError(409, "This comment is not the suggestion's stamp.", { code: "not_this_comment" });
|
|
2293
|
+
if (!this.#fits(s, { comments: saved.comments, edits: [...s.edits, ...this.#editsToTake(s, edits)] })) throw tooLarge("This request");
|
|
2294
|
+
return { saved, c };
|
|
2295
|
+
}
|
|
2296
|
+
|
|
2297
|
+
useSuggestion(key, { suggestion: rec, commentId, comments, note, baseSha, bytes = null, edits = [], prepared = null, rev, tab, seen, noteSeen }) {
|
|
2298
|
+
const s = this.get(key);
|
|
2299
|
+
const { id } = this.#useChecks(s, { suggestion: rec, commentId, comments });
|
|
2300
|
+
// the page's list and note are saved first (only the stamp goes out), after every limit is checked on them
|
|
2301
|
+
this.#commitDraft(s, this.#usePlan(s, { comments, note, edits, rev, tab, seen, noteSeen, bytes }, id).saved, rev, tab, true);
|
|
1444
2302
|
const c = s.comments.find((x) => x.id === id);
|
|
1445
|
-
const anchor = { location: c.anchor.location, quote: c.anchor.quote, prefix: c.anchor.prefix, suffix: c.anchor.suffix };
|
|
1446
2303
|
let plan = null;
|
|
1447
|
-
// the place and the reply are worked out on the very bytes that are about to be the file
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
const
|
|
1451
|
-
const place =
|
|
2304
|
+
// the place and the reply are worked out on the very bytes that are about to be the file (by the anchor worker, or here); any
|
|
2305
|
+
// refusal here writes nothing, and work that did not finish is "not here"
|
|
2306
|
+
const check = (b, sha) => {
|
|
2307
|
+
const r = this.#results(s.format, b, sha, [useTask(c.anchor, rec, id)], prepared).at(0);
|
|
2308
|
+
const { durable, place, reply } = usable(r) ? r : { durable: null, place: { ok: false, code: "not_here" }, reply: null };
|
|
1452
2309
|
if (!place.ok) throw new HttpError(409, place.code === "protected" ? "This part of the file is protected, so it cannot be changed here." : LOCAL_HINT, { code: place.code });
|
|
1453
|
-
const
|
|
1454
|
-
const batch = buildBatch({ file: s.file, format: s.format, baseSha: sha256(b), note: "", comments: [{ ...c, durable }], edits: [] });
|
|
2310
|
+
const batch = buildBatch({ file: s.file, format: s.format, baseSha: sha, note: "", comments: [{ ...c, durable }], edits: [] });
|
|
1455
2311
|
if (!reply || batch.instructions.comments[0]?.reply_with !== "find_replace_or_text") throw new HttpError(409, LOCAL_HINT, { code: "not_here" });
|
|
1456
2312
|
plan = { durable, reply, batch };
|
|
1457
2313
|
};
|
|
2314
|
+
if (!this.#fits(s, { comments: s.comments, edits: [...s.edits, ...this.#editsToTake(s, edits)] })) throw tooLarge("This request"); // before anything is written
|
|
1458
2315
|
let taken;
|
|
1459
2316
|
try {
|
|
1460
2317
|
taken = this.#takePage(s, { baseSha, bytes }, check);
|
|
@@ -1463,15 +2320,16 @@ export class Sessions extends EventEmitter {
|
|
|
1463
2320
|
if (["not_here", "not_eligible", "protected"].includes(err?.extra?.code)) {
|
|
1464
2321
|
c.localHint = LOCAL_HINT;
|
|
1465
2322
|
this.#changed(s);
|
|
1466
|
-
}
|
|
2323
|
+
} else this.emit("change", s, "draft"); // the drafts it stored
|
|
1467
2324
|
throw err;
|
|
1468
2325
|
}
|
|
1469
|
-
|
|
2326
|
+
s.edits.push(...this.#editsToTake(s, edits));
|
|
1470
2327
|
c.durable = plan.durable;
|
|
1471
2328
|
c.status = "sent";
|
|
1472
2329
|
c.batchId = plan.batch.batch_id;
|
|
1473
2330
|
delete c.localHint;
|
|
1474
2331
|
s.lastWrite = taken.writeInfo;
|
|
2332
|
+
this.#sentRev(s, { rev, tab });
|
|
1475
2333
|
try {
|
|
1476
2334
|
this.startLocal(key, plan.batch, { reply: plan.reply, summary: USE_SUMMARY, model: true }); // saves, tells the page, starts the runner
|
|
1477
2335
|
} catch (err) {
|
|
@@ -1530,13 +2388,13 @@ export class Sessions extends EventEmitter {
|
|
|
1530
2388
|
* "voice" with a fixed summary, no history update and no analysis. A file that changed anyway (a direct save past voice_only) is
|
|
1531
2389
|
* recorded as a withdrawn run would be: flagged, kept Revert-able.
|
|
1532
2390
|
*/
|
|
1533
|
-
#doneVoice(s, results, run, post, postSha) {
|
|
2391
|
+
#doneVoice(s, results, run, post, postSha, prepared = null) {
|
|
1534
2392
|
const flags = [];
|
|
1535
2393
|
for (const r of (Array.isArray(results.results) ? results.results : []).slice(0, 1000)) flags.push({ code: "unknown_result", severity: "warn", comment_id: String(r?.comment_id ?? "").slice(0, 64) });
|
|
1536
2394
|
const check = cleanVoiceDraft(results.voice_draft); // size first, then hidden characters, links and markup, the layout, the caps
|
|
1537
2395
|
if (check.code) flags.push({ code: "bad_voice_draft", severity: "warn", reason: check.code });
|
|
1538
2396
|
this.#guardFlags(run, flags);
|
|
1539
|
-
const queuedPlan = this.#planQueued(s, post, postSha, flags);
|
|
2397
|
+
const queuedPlan = this.#planQueued(s, post, postSha, flags, prepared);
|
|
1540
2398
|
const { list: ignored, more: ignoredMore } = ignoredInstructions(results.document_instructions_ignored);
|
|
1541
2399
|
// ---- commit: the draft file first (a failed write keeps nothing and says so), then the session
|
|
1542
2400
|
let kept = false;
|
|
@@ -1589,7 +2447,7 @@ export class Sessions extends EventEmitter {
|
|
|
1589
2447
|
if (!batch) throw new HttpError(409, "nothing to take back");
|
|
1590
2448
|
// the note travels back with the comments: the withdrawn batch's note, then a queued one, then whatever was typed since
|
|
1591
2449
|
const notes = [batch.instructions?.overall_note, s.queuedNote, s.note].filter(Boolean);
|
|
1592
|
-
s
|
|
2450
|
+
this.#giveNote(s, [...new Set(notes)].join("\n\n"));
|
|
1593
2451
|
s.queuedNote = "";
|
|
1594
2452
|
let event = "state";
|
|
1595
2453
|
if (s.delivered) {
|
|
@@ -1640,7 +2498,27 @@ export class Sessions extends EventEmitter {
|
|
|
1640
2498
|
return true;
|
|
1641
2499
|
}
|
|
1642
2500
|
|
|
1643
|
-
revert(
|
|
2501
|
+
/** revert() as the server's route calls it: a waiting batch's anchors are moved onto the restored version in the anchor worker first. */
|
|
2502
|
+
async revertAsync(key, activityId) {
|
|
2503
|
+
return this.#serial(key, async () => {
|
|
2504
|
+
const s = this.get(key);
|
|
2505
|
+
const entry = s.activity.find((a) => a.id === activityId && a.kind === "agent");
|
|
2506
|
+
let prepared = null;
|
|
2507
|
+
if (entry && !entry.reverted && !s.delivered && s.pending && !readOnlyKind(s.pending) && !s.pending.instructions?.review && s.snapshots.has(entry.preSha)) {
|
|
2508
|
+
const tasks = this.#revertComments(s, entry).map(reanchorTask);
|
|
2509
|
+
if (tasks.length) prepared = await this.#offThread(s.format, { source: { path: s.snapshots.blob(entry.preSha), sha: entry.preSha }, tasks });
|
|
2510
|
+
}
|
|
2511
|
+
return this.revert(key, activityId, { prepared });
|
|
2512
|
+
});
|
|
2513
|
+
}
|
|
2514
|
+
|
|
2515
|
+
/** The comments of the waiting batch, as revert() moves them: the run's comments become drafts first, so only the others are left. */
|
|
2516
|
+
#revertComments(s, entry) {
|
|
2517
|
+
const ids = new Set(s.pending.instructions.comments.map((c) => c.id));
|
|
2518
|
+
return s.comments.filter((c) => ids.has(c.id) && !(c.batchId === entry.batchId && ["done", "declined", "asked"].includes(c.status)));
|
|
2519
|
+
}
|
|
2520
|
+
|
|
2521
|
+
revert(key, activityId, { prepared = null } = {}) {
|
|
1644
2522
|
const s = this.get(key);
|
|
1645
2523
|
if (s.delivered) throw new HttpError(423, s.delivered.local ? "a change is being made on this Mac" : "the agent is working", { code: "locked" });
|
|
1646
2524
|
const entry = s.activity.find((a) => a.id === activityId && a.kind === "agent");
|
|
@@ -1652,7 +2530,15 @@ export class Sessions extends EventEmitter {
|
|
|
1652
2530
|
if (sha256(disk) !== entry.postSha) throw new HttpError(409, mac ? "the file changed after the change was made; revert refused" : "the file changed after the agent finished; revert refused", { code: "changed_since" });
|
|
1653
2531
|
if (!s.snapshots.has(entry.preSha)) throw new HttpError(410, mac ? "the version before this change is no longer stored" : "the pre-agent version is no longer stored");
|
|
1654
2532
|
const pre = s.snapshots.read(entry.preSha);
|
|
1655
|
-
|
|
2533
|
+
// an HTML file: a revert restores the version before the run, by identity (the stored bytes must hash to the run's preSha), never
|
|
2534
|
+
// through the write check: that version was the file before (opened by the person or written after its own check), and checking it
|
|
2535
|
+
// against the run's result would refuse undoing a run that removed active content (the revert would "add" it back)
|
|
2536
|
+
let htmlChecked = null;
|
|
2537
|
+
if (s.format === "html") {
|
|
2538
|
+
if (sha256(pre) !== entry.preSha) throw new HttpError(410, mac ? "the version before this change is no longer stored" : "the pre-agent version is no longer stored");
|
|
2539
|
+
htmlChecked = { from: entry.postSha, to: entry.preSha };
|
|
2540
|
+
}
|
|
2541
|
+
this.#replace(s, pre, { expectSha: entry.postSha, label: "before revert", htmlChecked }); // a revert is a new version
|
|
1656
2542
|
s.snapshots.add(pre, "reverted");
|
|
1657
2543
|
s.baseSha = entry.preSha;
|
|
1658
2544
|
addUpdate(s, { kind: "revert", before: entry.postSha, after: entry.preSha, local: mac }, this.now());
|
|
@@ -1670,7 +2556,7 @@ export class Sessions extends EventEmitter {
|
|
|
1670
2556
|
}
|
|
1671
2557
|
for (const g of s.suggestions || []) if (g.status === "accepted" && g.acceptedBatch === entry.batchId) g.status = "open"; // the text is back: it can be accepted again
|
|
1672
2558
|
if (entry.by === "local" && entry.localNote && !s.note) {
|
|
1673
|
-
s
|
|
2559
|
+
this.#giveNote(s, entry.localNote); // the request comes back to the box, and the same words go to the agent next time
|
|
1674
2560
|
s.skipLocalNote = entry.localNote;
|
|
1675
2561
|
}
|
|
1676
2562
|
// a batch built against the reverted version is rebuilt against the restored bytes (a review: its outline is read again; a voice
|
|
@@ -1680,19 +2566,9 @@ export class Sessions extends EventEmitter {
|
|
|
1680
2566
|
else if (s.pending) {
|
|
1681
2567
|
const ids = new Set(s.pending.instructions.comments.map((c) => c.id));
|
|
1682
2568
|
const cs = s.comments.filter((c) => ids.has(c.id));
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
} catch {
|
|
1687
|
-
units = null;
|
|
1688
|
-
}
|
|
1689
|
-
for (const c of cs) {
|
|
1690
|
-
try {
|
|
1691
|
-
if (units) c.durable = reanchor(units, c);
|
|
1692
|
-
} catch {
|
|
1693
|
-
c.durable = null;
|
|
1694
|
-
}
|
|
1695
|
-
}
|
|
2569
|
+
// moved onto the restored version by the anchor worker (revertAsync) or here; one whose work did not finish goes out weaker
|
|
2570
|
+
const work = this.#results(s.format, pre, entry.preSha, cs.map(reanchorTask), prepared);
|
|
2571
|
+
if (work.read?.ok) cs.forEach((c, i) => (c.durable = usable(work.at(i)) ? work.at(i) : work.at(i)?.error ? null : weakDurable(s.format, c.anchor)));
|
|
1696
2572
|
s.pending = buildBatch({ file: s.file, format: s.format, baseSha: entry.preSha, note: s.pending.instructions.overall_note, comments: cs, edits: [], style: styleNote(), guide: voiceGuide() });
|
|
1697
2573
|
for (const c of cs) c.batchId = s.pending.batch_id;
|
|
1698
2574
|
}
|
|
@@ -1704,6 +2580,7 @@ export class Sessions extends EventEmitter {
|
|
|
1704
2580
|
close(key) {
|
|
1705
2581
|
const s = this.get(key);
|
|
1706
2582
|
s.closed = true;
|
|
2583
|
+
this.#views.delete(key); // the render of an HTML file (htmlView)
|
|
1707
2584
|
this.editors.delete(key); // the closed tab may stay on screen with its event stream open: it must not hold the file against the tab that reopens it
|
|
1708
2585
|
for (const p of s.pollers.splice(0)) p.resolve({ status: "closed" });
|
|
1709
2586
|
fs.rmSync(path.join(stateDir(), "agents", s.key, "outline"), { recursive: true, force: true }); // the outline files of Onionskin's own agent
|
|
@@ -1733,6 +2610,37 @@ export class Sessions extends EventEmitter {
|
|
|
1733
2610
|
return out;
|
|
1734
2611
|
}
|
|
1735
2612
|
|
|
2613
|
+
// ---- the view of an HTML file (GET /api/html-view) ------------------------------------------------------------------------------
|
|
2614
|
+
// key -> {sha, full, plain}: the render of one version (promises, so two requests for it share one worker run); one version per file,
|
|
2615
|
+
// dropped when the file changes or the session closes. Ceiling: one entry per open HTML file, each up to ~43 MB (the render copy at the
|
|
2616
|
+
// limits): a person with many large pages open holds them all; an LRU over files would be the upgrade.
|
|
2617
|
+
#views = new Map();
|
|
2618
|
+
/**
|
|
2619
|
+
* The page's view of an HTML file (PLAN-html 3.2, R12): the disk bytes are taken as /api/file takes them (loadForPage: an outside change
|
|
2620
|
+
* is recorded, the watcher told), then the render copy of exactly those bytes is built in the render worker (src/offload.js
|
|
2621
|
+
* renderOffThread), or the plain copy with `plain`. Resolves to {sha, plain, reason, stats, texts, editable, ident, bytes}; throws HttpError
|
|
2622
|
+
* 422 with the encoding's sentence and code (and `copyable`) or a limit's, 503 when the worker was busy or did not finish.
|
|
2623
|
+
*/
|
|
2624
|
+
async htmlView(key, { plain = false } = {}) {
|
|
2625
|
+
const s = this.get(key);
|
|
2626
|
+
if (s.format !== "html") throw new HttpError(400, "this file is not an HTML file", { code: "not_html" });
|
|
2627
|
+
const { bytes, diskSha } = this.loadForPage(key);
|
|
2628
|
+
let c = this.#views.get(key);
|
|
2629
|
+
if (!c || c.sha !== diskSha) this.#views.set(key, (c = { sha: diskSha, full: null, plain: null }));
|
|
2630
|
+
const k = plain ? "plain" : "full";
|
|
2631
|
+
if (!c[k]) {
|
|
2632
|
+
const p = renderOffThread({ bytes, fileDir: path.dirname(s.file), plain }).then((r) => {
|
|
2633
|
+
if (r.ok) return { sha: diskSha, plain: r.plain, reason: r.reason, stats: r.stats, texts: r.texts, editable: r.editable, ident: r.ident, bytes: r.bytes };
|
|
2634
|
+
if (c[k] === p) c[k] = null; // a failure is not kept: the next request tries again
|
|
2635
|
+
if (r.failed) throw new HttpError(503, `Onionskin could not prepare this page: ${r.why}. Try again in a moment.`, { code: r.failed === "busy" ? "busy" : "render_failed" });
|
|
2636
|
+
if (r.code === "not_utf8" || ["utf16", "invalid_utf8", "roundtrip"].includes(r.code)) throw encodingRefusal(s.file, { error: r.error, code: r.code, copyable: r.copyable });
|
|
2637
|
+
throw new HttpError(422, `Onionskin cannot read this file: ${r.error}.`, { code: "too_large" });
|
|
2638
|
+
});
|
|
2639
|
+
c[k] = p;
|
|
2640
|
+
}
|
|
2641
|
+
return c[k];
|
|
2642
|
+
}
|
|
2643
|
+
|
|
1736
2644
|
/** The history list and one update (browser routes, read-only; see src/history.js). */
|
|
1737
2645
|
history(key) {
|
|
1738
2646
|
return historyList(this.get(key));
|
|
@@ -1847,25 +2755,58 @@ export class Sessions extends EventEmitter {
|
|
|
1847
2755
|
*/
|
|
1848
2756
|
resolveAll(key) {
|
|
1849
2757
|
const s = this.get(key);
|
|
2758
|
+
// an HTML file is read in workers only: on this thread every anchor is "moved or removed" (the server's route resolves in the worker)
|
|
2759
|
+
if (s.format === "html") return { baseSha: sha256(readCapped(s.file)), format: s.format, anchors: Object.fromEntries(this.#resolveTask(s).comments.map((c) => [c.id, { method: "none" }])), view: null };
|
|
1850
2760
|
const bytes = readCapped(s.file);
|
|
1851
|
-
const
|
|
1852
|
-
const
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
2761
|
+
const baseSha = sha256(bytes);
|
|
2762
|
+
const r = anchorWork({ format: s.format, bytes, sha: baseSha, tasks: [this.#resolveTask(s)], budget: SCAN_BUDGET });
|
|
2763
|
+
if (!r.read.ok) throw r.read.cap ? new CapError(r.read.cap) : new Error(r.read.invalid);
|
|
2764
|
+
if (r.results[0]?.error) throw new Error(r.results[0].error);
|
|
2765
|
+
return { baseSha, format: s.format, ...r.results[0] };
|
|
2766
|
+
}
|
|
2767
|
+
|
|
2768
|
+
/**
|
|
2769
|
+
* resolveAll() as the server's route calls it: resolved in the anchor worker against the file on disk. Work that did not finish
|
|
2770
|
+
* answers as a spent budget does: every anchor "moved or removed" (the page keeps the comments in its list), the view unknown.
|
|
2771
|
+
*/
|
|
2772
|
+
async resolveAllAsync(key) {
|
|
2773
|
+
const s = this.get(key);
|
|
2774
|
+
const task = this.#resolveTask(s);
|
|
2775
|
+
if (s.format === "html") await this.#htmlDrafts(s, task);
|
|
2776
|
+
const r = await anchorsOffThread({ format: s.format, source: { path: s.file }, tasks: [task] });
|
|
2777
|
+
if (r.ok && r.r.read.cap) throw new CapError(r.r.read.cap);
|
|
2778
|
+
if (r.ok && r.r.read.ok && usable(r.r.results[0])) return { baseSha: r.sha, format: s.format, ...r.r.results[0] };
|
|
2779
|
+
return { baseSha: r.ok ? r.sha : sha256(readCapped(s.file)), format: s.format, anchors: Object.fromEntries(task.comments.map((c) => [c.id, { method: "none" }])), view: null };
|
|
2780
|
+
}
|
|
2781
|
+
|
|
2782
|
+
/**
|
|
2783
|
+
* HTML: a draft (no durable yet) whose page anchor was made on another version than the file on disk (an outside change, a revert, a
|
|
2784
|
+
* save) would be drawn at a node id of that other version. It is made durable on its own version first (a stored snapshot) and resolved
|
|
2785
|
+
* with the others; one whose version is no longer stored resolves by its quote. Drafts made on the file's version need nothing: the page
|
|
2786
|
+
* draws them at their own place.
|
|
2787
|
+
*/
|
|
2788
|
+
async #htmlDrafts(s, task) {
|
|
2789
|
+
let cur;
|
|
2790
|
+
try {
|
|
2791
|
+
cur = sha256(readCapped(s.file));
|
|
2792
|
+
} catch {
|
|
2793
|
+
return;
|
|
2794
|
+
}
|
|
2795
|
+
const other = s.comments.filter((c) => !c.durable && c.anchor?.location?.version && c.anchor.location.version !== cur);
|
|
2796
|
+
const byVersion = new Map();
|
|
2797
|
+
for (const c of other) (byVersion.get(c.anchor.location.version) || byVersion.set(c.anchor.location.version, []).get(c.anchor.location.version)).push(c);
|
|
2798
|
+
for (const [v, list] of byVersion) {
|
|
2799
|
+
const r = s.snapshots.has(v) ? await this.#offThread("html", { source: { path: s.snapshots.blob(v), sha: v }, tasks: list.map(buildTask) }) : null;
|
|
2800
|
+
for (const c of list) {
|
|
2801
|
+
const d = r?.ok && r.read?.ok ? r.map.get(taskKey(buildTask(c))) : undefined;
|
|
2802
|
+
const durable = usable(d) && d ? d : weakDurable("html", c.anchor);
|
|
2803
|
+
if (durable) task.comments.push({ id: c.id, durable });
|
|
1859
2804
|
}
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
if (s.view?.durable) out.view = { ...resolveDurable(units.paragraphs, s.view.durable), offsetPx: s.view.offsetPx, zoom: s.view.zoom, fraction: s.view.fraction };
|
|
1866
|
-
} else out.view = resolveView(units, s.view);
|
|
1867
|
-
out.sheets = s.format === "xlsx" ? units.wb.sheets.map((x) => ({ index: x.index, name: x.name, sheetId: x.sheetId })) : undefined;
|
|
1868
|
-
return out;
|
|
2805
|
+
}
|
|
2806
|
+
}
|
|
2807
|
+
|
|
2808
|
+
#resolveTask(s) {
|
|
2809
|
+
return { op: "resolveAll", comments: s.comments.filter((c) => c.durable).map((c) => ({ id: c.id, durable: c.durable })), view: s.view };
|
|
1869
2810
|
}
|
|
1870
2811
|
}
|
|
1871
2812
|
|
|
@@ -1906,7 +2847,7 @@ function exportInWorker(job) {
|
|
|
1906
2847
|
};
|
|
1907
2848
|
const timer = setTimeout(() => finish({ timeout: true }), EXPORT_TIMEOUT_MS());
|
|
1908
2849
|
try {
|
|
1909
|
-
worker =
|
|
2850
|
+
worker = startWorker("redline-worker.js", { workerData: job });
|
|
1910
2851
|
} catch (err) {
|
|
1911
2852
|
return finish({ check_failed: String(err?.message || err) });
|
|
1912
2853
|
}
|