onionskin-cli 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +100 -0
- package/INSTALL.md +5 -1
- package/NOTICE +34 -0
- package/README.md +101 -33
- package/docs/contracts/batch.schema.json +27 -4
- package/docs/contracts/ops.schema.json +117 -5
- package/package.json +4 -2
- package/skills/onionskin/SKILL.md +9 -6
- package/skills/onionskin/reference/batch.md +5 -3
- package/skills/onionskin/reference/formats.md +5 -1
- package/skills/onionskin/reference/ops.md +47 -3
- package/skills/onionskin/reference/troubleshooting.md +31 -0
- package/src/agent-detect.js +28 -2
- package/src/agent-prompt.js +12 -1
- package/src/agent-runner.js +16 -5
- package/src/anchor-work.js +154 -0
- package/src/anchor-worker.js +38 -0
- package/src/anchors.js +61 -11
- package/src/apply.js +119 -47
- package/src/batch.js +32 -8
- package/src/check-worker.js +8 -5
- package/src/cli.js +91 -25
- package/src/constants.js +34 -1
- package/src/diff.js +3 -1
- package/src/docx.js +368 -58
- package/src/edit-hash.js +19 -0
- package/src/fsutil.js +1 -1
- package/src/history-worker.js +8 -8
- package/src/history.js +135 -31
- package/src/html-active.js +384 -0
- package/src/html-envelope.js +51 -0
- package/src/html-parser.js +8 -0
- package/src/html-render-worker.js +25 -0
- package/src/html-render.js +880 -0
- package/src/html-utf8.js +65 -0
- package/src/html-write-worker.js +14 -0
- package/src/html-write.js +380 -0
- package/src/html.js +1257 -0
- package/src/local.js +68 -4
- package/src/merge-package.js +1110 -0
- package/src/merge-pptx.js +98 -38
- package/src/merge-proof.js +1019 -0
- package/src/merge-worker.js +60 -0
- package/src/merge-xlsx.js +84 -29
- package/src/merge.js +286 -83
- package/src/nfc.js +46 -0
- package/src/notes.js +122 -0
- package/src/offload.js +400 -1
- package/src/open-worker.js +27 -0
- package/src/ops/common.js +2 -1
- package/src/ops/docx.js +58 -29
- package/src/ops/html.js +394 -0
- package/src/ops/pptx.js +74 -21
- package/src/ops/run.js +23 -3
- package/src/ops/runs.js +215 -13
- package/src/ops/schema.js +8 -1
- package/src/ops/xlsx.js +12 -7
- package/src/ops-worker.js +17 -0
- package/src/package.js +26 -2
- package/src/person.js +2 -1
- package/src/pptx.js +85 -21
- package/src/quick-plan.js +13 -6
- package/src/redline-worker.js +10 -14
- package/src/reply.js +46 -19
- package/src/retain.js +223 -0
- package/src/review.js +32 -18
- package/src/rewrite-check.js +2 -2
- package/src/sanitize.js +16 -3
- package/src/server.js +80 -36
- package/src/session.js +1138 -208
- package/src/stop-gate.js +34 -0
- package/src/suggest-reply.js +42 -7
- package/src/suggest.js +14 -29
- package/src/text.js +161 -11
- package/src/update.js +288 -0
- package/src/voice-guide.js +4 -3
- package/src/voice-learn.js +4 -3
- package/src/voice-sample-worker.js +7 -6
- package/src/voice-worker.js +4 -6
- package/src/worker-host.js +23 -0
- package/src/xlsx.js +355 -94
- package/src/xml-tree.js +6 -3
- package/src/xml.js +82 -34
- package/src/zip-write.js +2 -1
- package/src/zip.js +9 -5
- package/vendor/parse5/LICENSE.txt +32 -0
- package/vendor/parse5/parse5.mjs +7822 -0
- package/vendor/parse5.lock.json +23 -0
- package/web/dist/assets/index-DkNx0LSH.css +1 -0
- package/web/dist/assets/{index-DS-WST1E.js → index-DytKkMdi.js} +112 -108
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/session.js
CHANGED
|
@@ -2,9 +2,11 @@ import crypto from "node:crypto";
|
|
|
2
2
|
import { EventEmitter } from "node:events";
|
|
3
3
|
import fs from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
|
-
import { REFUSED_EXTENSIONS,
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
5
|
+
import { EDITS_OVERFLOW, ONLY_FORMATS, REFUSED_EXTENSIONS, REFUSED_PAGES, MAX_DOC_BYTES, PRODUCT, env, formatOfExt } from "./constants.js";
|
|
6
|
+
import { readAll, unitsFor } from "./anchors.js";
|
|
7
|
+
import { anchorWork, weakDurable } from "./anchor-work.js";
|
|
8
|
+
import { scanBudget } from "./text.js";
|
|
9
|
+
import { CapError } from "./xml.js";
|
|
8
10
|
import { checkRegularOwned, readDoc, replaceGuarded, sha256, Snapshots, WriteConflict, writeSiblingCopy } from "./fsutil.js";
|
|
9
11
|
import { buildBatch, compactBatch, withOutline } from "./batch.js";
|
|
10
12
|
import { styleChangedOutside, styleNote } from "./style.js";
|
|
@@ -13,7 +15,10 @@ import { cleanVoiceDraft } from "./voice-guide.js";
|
|
|
13
15
|
import { voiceJob } from "./voice-learn.js";
|
|
14
16
|
import { clearTickets, saveTicket, ticketOf } from "./lease.js";
|
|
15
17
|
import { analyze } from "./analyze.js";
|
|
18
|
+
import { anchorTimeoutMs, anchorsOffThread, htmlWriteOffThread, mergeOffThread, readOffThread, renderOffThread, startWorker } from "./offload.js";
|
|
16
19
|
import { deepNotice, validatePackage } from "./package.js";
|
|
20
|
+
import { HtmlEncodingError } from "./html.js";
|
|
21
|
+
import { ACTIVE_REFUSED } from "./html-active.js";
|
|
17
22
|
import { execFile } from "node:child_process";
|
|
18
23
|
import { reopenCommand } from "./person.js";
|
|
19
24
|
import { sessionKey, stateDir, sub, writeStateFile } from "./paths.js";
|
|
@@ -21,16 +26,17 @@ import { cleanFlag, hasControlChars, untrusted, userText } from "./sanitize.js";
|
|
|
21
26
|
import { addUpdate, historyDetail, historyList, visibleUpdates } from "./history.js";
|
|
22
27
|
import { BAD_QUESTION_NOTE, QUESTIONS_PER_BATCH, cleanQuestion } from "./question.js";
|
|
23
28
|
import { validateOps } from "./ops/schema.js";
|
|
24
|
-
import { Worker } from "node:worker_threads";
|
|
25
29
|
import { AUTHORS, wordDate } from "./redline.js";
|
|
26
30
|
import { deepRedline } from "./checker.js";
|
|
27
31
|
import { TRACKED, formattingText, savedText, unsupportedText } from "./tracked-words.js";
|
|
28
32
|
import { LOCAL_HOLDER } from "./local.js";
|
|
29
33
|
import { bgHolder, isAgentHolder, isBgHolder, termHolder, TERM_PREFIX } from "./agent-detect.js";
|
|
30
|
-
import { OUTLINE_FILE_MAX, REVIEW_ASKS, REVIEW_KINDS, SUGGESTIONS_MAX, UNITS_MAX,
|
|
34
|
+
import { OUTLINE_FILE_MAX, REVIEW_ASKS, REVIEW_KINDS, SUGGESTIONS_MAX, UNITS_MAX, outlineOf, reviewSummary, unitsOf } from "./review.js";
|
|
31
35
|
import { VOICE_BATCH_MAX_CHARS } from "./voice-samples.js";
|
|
32
36
|
import { STAMP_WORDS } from "./stamp-words.js";
|
|
33
|
-
import { USE_SUMMARY
|
|
37
|
+
import { USE_SUMMARY } from "./suggest-reply.js";
|
|
38
|
+
import { mergeNote } from "./notes.js";
|
|
39
|
+
import { COMMENT_MAX, COMMENT_TEXT_MAX, COMMENTS_MAX, EDITS_MAX, NOTE_MAX, TOO_MANY_COMMENTS, REQUEST_MAX, SESSION_MAX, capComment, cleanEdit, cleanPageAnchor, cleanView, cutText, jsonSize } from "./retain.js";
|
|
34
40
|
|
|
35
41
|
/** The line under a comment whose quick fixes could not be written on this Mac (docs/contracts/routes.md). */
|
|
36
42
|
export const LOCAL_HINT = "This could not be changed on this Mac, so nothing was written. Press Send to ask the agent.";
|
|
@@ -59,8 +65,33 @@ export class HttpError extends Error {
|
|
|
59
65
|
}
|
|
60
66
|
}
|
|
61
67
|
|
|
68
|
+
/** One comment more than a file keeps (src/retain.js COMMENTS_MAX): 413 too_many_comments, in words the page shows as they are. */
|
|
69
|
+
const tooMany = () => new HttpError(413, TOO_MANY_COMMENTS, { code: "too_many_comments" });
|
|
70
|
+
/** A note longer than Onionskin keeps (src/retain.js NOTE_MAX): 413 note_too_long. `joined`: a waiting note and this one together. */
|
|
71
|
+
const noteTooLong = (joined) =>
|
|
72
|
+
new HttpError(413, joined ? `The note waiting for the agent and this one together are longer than Onionskin takes (${NOTE_MAX.toLocaleString("en-US")} characters), so nothing was sent. Shorten this note, then Send again.` : `This note is longer than Onionskin takes (${NOTE_MAX.toLocaleString("en-US")} characters), so it was not saved. Shorten it, then try again.`, { code: "note_too_long" });
|
|
73
|
+
/** The page's note as stored: refused (never cut) past NOTE_MAX. */
|
|
74
|
+
function noteOf(v) {
|
|
75
|
+
const t = userText(v, NOTE_MAX + 1);
|
|
76
|
+
if ([...t].length > NOTE_MAX) throw noteTooLong(false);
|
|
77
|
+
return t;
|
|
78
|
+
}
|
|
79
|
+
/** Two notes, the earlier first, as one (a waiting batch's or the queued ones, then the new one): refused past NOTE_MAX. */
|
|
80
|
+
function joinNotes(a, b) {
|
|
81
|
+
const t = [a, b].filter(Boolean).join("\n\n");
|
|
82
|
+
if ([...t].length > NOTE_MAX) throw noteTooLong(true);
|
|
83
|
+
return t;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** A request whose own data is more than Onionskin keeps (src/retain.js): 413 too_large_request, in words the page shows as they are. */
|
|
87
|
+
export const tooLarge = (what) => new HttpError(413, `${what} is larger than Onionskin takes (${Math.round(REQUEST_MAX / 1024 / 1024)} MB), so nothing was saved. Remove some comments or shorten the longest ones, then try again.`, { code: "too_large_request" });
|
|
88
|
+
|
|
62
89
|
const LEASE_MS = () => Number(env("LEASE_MS", 15 * 60 * 1000));
|
|
90
|
+
const SENT_TABS = 16; // tabs of one file whose newest committed Send a session remembers (s.sent)
|
|
91
|
+
const DELETED_KEPT = 200; // ids and words of deleted drafts a session remembers for a handover (Sessions.handOver)
|
|
92
|
+
const HANDOVER_MS = 60_000; // how long a tab that lost editing may hand over its unsaved words (Sessions.handOver)
|
|
63
93
|
const MAX_BYTES = MAX_DOC_BYTES;
|
|
94
|
+
const TOO_BIG = `Onionskin cannot read this file: it is larger than ${Math.round(MAX_BYTES / 1024 / 1024)} MB, more than Onionskin can read. Make it smaller (pictures and videos are often most of a file) or split it, and open that.`;
|
|
64
95
|
/** "Show in folder" exists on macOS (`open -R`); elsewhere the page shows the path. ONIONSKIN_NO_OPEN=1 turns it off, ONIONSKIN_OPEN_STUB=<file> records instead of launching. */
|
|
65
96
|
const canReveal = () => !!env("OPEN_STUB") || (process.platform === "darwin" && !env("NO_OPEN"));
|
|
66
97
|
/** readDoc with the size refusal as an HTTP error */
|
|
@@ -68,31 +99,81 @@ function readCapped(file) {
|
|
|
68
99
|
try {
|
|
69
100
|
return readDoc(file);
|
|
70
101
|
} catch (err) {
|
|
71
|
-
if (err.code === "TOO_LARGE") throw new HttpError(413, `
|
|
102
|
+
if (err.code === "TOO_LARGE") throw new HttpError(413, `Onionskin cannot read this file: ${err.message}.`, { code: "too_large" });
|
|
72
103
|
throw err;
|
|
73
104
|
}
|
|
74
105
|
}
|
|
75
106
|
|
|
76
|
-
|
|
107
|
+
/**
|
|
108
|
+
* Every part the server reads later, of bytes about to become the file (a Send's, an agent's write), so they are refused before anything is
|
|
109
|
+
* written: past a reader's cap with the reason (CapError: 422 too_large), damaged where the readers look as an invalid package (422), never
|
|
110
|
+
* as an internal error. The reading is kept for what reads the same version next (unitsFor).
|
|
111
|
+
*/
|
|
112
|
+
function readWhole(format, bytes, sha, refusal = "refusing to write") {
|
|
113
|
+
try {
|
|
114
|
+
const units = readAll(unitsFor(format, bytes, sha));
|
|
115
|
+
readInCaps(format, sha);
|
|
116
|
+
return units;
|
|
117
|
+
} catch (err) {
|
|
118
|
+
if (err instanceof CapError || err instanceof HttpError) throw err;
|
|
119
|
+
throw new HttpError(422, `${refusal}: ${untrusted(err?.message || err, 200)}`, { code: "invalid_package" });
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// "<format>:<sha256>" of the versions read whole within the readers' caps, on this thread (readWhole) or by the open worker, so a version
|
|
124
|
+
// read again is not read whole again. Ceiling: VERIFIED_KEPT entries, the oldest dropped first (one dropped is read whole again when needed).
|
|
125
|
+
const VERIFIED_KEPT = 256;
|
|
126
|
+
const verified = new Set();
|
|
127
|
+
function readInCaps(format, sha) {
|
|
128
|
+
const id = `${format}:${sha}`;
|
|
129
|
+
verified.delete(id);
|
|
130
|
+
verified.add(id);
|
|
131
|
+
if (verified.size > VERIFIED_KEPT) verified.delete(verified.values().next().value);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Bytes about to be handed to the engine on this thread (a quick preview, a merge, a change made on this Mac), which reads without the
|
|
136
|
+
* readers' caps: they are read whole within the caps first (readWhole) unless this version already was, so a file that changed on disk
|
|
137
|
+
* past a cap is refused with the reason (CapError: 422 too_large) before the engine parses any of it. Returns `bytes`.
|
|
138
|
+
*/
|
|
139
|
+
export function readChecked(format, bytes, { sha = sha256(bytes), refusal = "cannot read the file" } = {}) {
|
|
140
|
+
if (!verified.has(`${format}:${sha}`)) readWhole(format, bytes, sha, refusal);
|
|
141
|
+
return bytes;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// bounded before it is rounded, so the result is always finite (1e308 * 1000 is Infinity, stored as null); anything that is not a finite
|
|
145
|
+
// number is 0, as src/retain.js drops what is not of its type
|
|
146
|
+
const num = (v, max) => {
|
|
147
|
+
const n = Number(v);
|
|
148
|
+
return Number.isFinite(n) ? Math.round(Math.min(max, Math.max(0, n)) * 1000) / 1000 : 0;
|
|
149
|
+
};
|
|
150
|
+
const frac = (v) => num(v, 1);
|
|
151
|
+
const PX_MAX = 1_000_000; // a paragraph taller than this is not drawn by any page
|
|
77
152
|
/** A pin: x/y as fractions of the element it sits on (`yPx` is kept for DOCX paragraphs); `on` says what that element is. */
|
|
78
153
|
function cleanPin(p) {
|
|
79
|
-
const out = { xFrac:
|
|
80
|
-
if (p.yFrac != null) out.yFrac =
|
|
154
|
+
const out = { xFrac: frac(p.xFrac), yPx: num(p.yPx, PX_MAX) };
|
|
155
|
+
if (p.yFrac != null) out.yFrac = frac(p.yFrac);
|
|
81
156
|
if (["paragraph", "cell", "shape", "slide"].includes(p.on)) out.on = p.on;
|
|
82
157
|
if (p.sxFrac != null) {
|
|
83
|
-
out.sxFrac =
|
|
84
|
-
out.syFrac =
|
|
158
|
+
out.sxFrac = frac(p.sxFrac);
|
|
159
|
+
out.syFrac = frac(p.syFrac);
|
|
85
160
|
}
|
|
86
161
|
return out;
|
|
87
162
|
}
|
|
88
163
|
|
|
89
164
|
const APPLY_STATUS = new Set(["refused", "skipped_protected", "anchor_not_found", "expect_mismatch", "not_applied"]);
|
|
90
|
-
// the three-way merge of each format (loaded on first use: they pull in the engine)
|
|
91
|
-
const MERGERS = { docx: { module: "./merge.js", fn: "mergeDocx" }, xlsx: { module: "./merge-xlsx.js", fn: "mergeXlsx" }, pptx: { module: "./merge-pptx.js", fn: "mergePptx" } };
|
|
92
165
|
const HELD_SHOWN = 5; // items the page's "not in the document" message lists (web/src/Banners.jsx)
|
|
93
166
|
const WRITES_KEPT = 20; // writes per file the merge can trace (src/merge-xlsx.js agentChanges); an older base finds no record and is handed back
|
|
94
|
-
/**
|
|
95
|
-
|
|
167
|
+
/**
|
|
168
|
+
* What the page says the user did during the run: kinds and places only, used to hold edits back or to name what is not carried, never to
|
|
169
|
+
* write. The first EDITS_KEPT; past them one more entry of kind EDITS_OVERFLOW says the list was cut, so a planner that needs the whole list
|
|
170
|
+
* refuses the merge (src/merge-xlsx.js) instead of reading the cut list as complete. The proof of the merge never reads it.
|
|
171
|
+
*/
|
|
172
|
+
const EDITS_KEPT = 2000;
|
|
173
|
+
export const editKinds = (edits) => {
|
|
174
|
+
const list = (Array.isArray(edits) ? edits.slice(0, EDITS_KEPT) : []).map((e) => ({ kind: String(e?.kind ?? "").slice(0, 40), sheet: typeof e?.sheet === "string" ? e.sheet.slice(0, 100) : null, slide: Number.isInteger(e?.slide) ? e.slide : null }));
|
|
175
|
+
return Array.isArray(edits) && edits.length > EDITS_KEPT ? [...list, { kind: EDITS_OVERFLOW, sheet: null, slide: null }] : list;
|
|
176
|
+
};
|
|
96
177
|
/** One op `apply` did not apply, as the page shows it: codes and short reasons only (never document text). */
|
|
97
178
|
function cleanProblem(p, run = "") {
|
|
98
179
|
return {
|
|
@@ -101,7 +182,8 @@ function cleanProblem(p, run = "") {
|
|
|
101
182
|
op: untrusted(p?.op, 40),
|
|
102
183
|
status: APPLY_STATUS.has(p?.status) ? p.status : "refused",
|
|
103
184
|
code: p?.code ? untrusted(p.code, 60) : null,
|
|
104
|
-
|
|
185
|
+
// the R1 refusal in the server's own words, never the reason text an agent's report carries
|
|
186
|
+
reason: p?.code === "active_content" ? ACTIVE_REFUSED : untrusted(p?.reason, 200),
|
|
105
187
|
};
|
|
106
188
|
}
|
|
107
189
|
|
|
@@ -109,17 +191,47 @@ function shellQuote(s) {
|
|
|
109
191
|
return `'${String(s).replace(/'/g, `'\\''`)}'`;
|
|
110
192
|
}
|
|
111
193
|
|
|
112
|
-
/**
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
if (units.format === "docx") return { ...d, ...r, format: "docx", quote: d.quote, prefix: d.prefix, suffix: d.suffix, paraText: r.text, engineDelta: r.engineDelta };
|
|
119
|
-
if (units.format === "xlsx") return buildDurable(units, { location: { sheetIndex: r.sheetIndex, sheetName: r.sheetName, range: r.range } });
|
|
120
|
-
return buildDurable(units, { location: { slideIndex: r.slideIndex, slideId: r.slideId, shapeId: r.shapeId ?? undefined, shapeName: r.shapeName, cell: d.cell || undefined }, quote: d.quote, prefix: d.prefix, suffix: d.suffix, hitKind: d.quote ? undefined : "shape" });
|
|
194
|
+
/**
|
|
195
|
+
* An HTML file that is not saved as UTF-8 (PLAN-html R10, R18): 400 with the fixed sentence as it is and `code` (not_utf8 or the
|
|
196
|
+
* reason). The page shows the sentence; the UTF-8 copy it offers is a CLI action (`onionskin open` prints its command: src/html-utf8.js).
|
|
197
|
+
*/
|
|
198
|
+
function encodingRefusal(real, r) {
|
|
199
|
+
return new HttpError(400, r.error, { code: r.code === "not_utf8" ? "not_utf8" : "encoding" });
|
|
121
200
|
}
|
|
122
201
|
|
|
202
|
+
/** The person's pending HTML text edits a Send or "Save my edits" carries (web/src/html-view.js createEdits): checked fields only. */
|
|
203
|
+
const HTML_EDITS_MAX = 2000;
|
|
204
|
+
const HTML_EDIT_TEXT_MAX = 1_000_000; // one text node's new text; the whole request is held to REQUEST_MAX (src/server.js)
|
|
205
|
+
export function cleanHtmlEdits(list) {
|
|
206
|
+
if (list == null) return [];
|
|
207
|
+
if (!Array.isArray(list) || list.length > HTML_EDITS_MAX) throw new HttpError(400, `htmlEdits must be a list of at most ${HTML_EDITS_MAX} edits`, { code: "bad_edits" });
|
|
208
|
+
return list.map((e) => {
|
|
209
|
+
const ok = e && typeof e === "object" && Number.isSafeInteger(e.nodeId) && e.nodeId >= 0 && typeof e.versionSha === "string" && /^[0-9a-f]{64}$/.test(e.versionSha) && typeof e.baseTextHash === "string" && /^[0-9a-f]{14}$/.test(e.baseTextHash) && typeof e.newText === "string" && e.newText.length <= HTML_EDIT_TEXT_MAX;
|
|
210
|
+
if (!ok) throw new HttpError(400, "an HTML text edit must be {nodeId, versionSha, baseTextHash, newText}", { code: "bad_edits" });
|
|
211
|
+
return { nodeId: e.nodeId, versionSha: e.versionSha, baseTextHash: e.baseTextHash, newText: e.newText };
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
/** 409 version_changed: the page's edits were made on a version that is not the file on disk now (it reloads the view and places them again). */
|
|
215
|
+
const VERSION_CHANGED = "The file changed since this page loaded it, so your text edits were not written. The page shows the new version and puts your edits back where it can find their text; check them, then Send again.";
|
|
216
|
+
/** The write check refused the person's text edits: 409 for an edit made on other text (the page reloads), 422 for the others; nothing was written. */
|
|
217
|
+
function htmlEditRefusal(c) {
|
|
218
|
+
if (c.code === "edit_moved") return new HttpError(409, VERSION_CHANGED, { code: "version_changed" });
|
|
219
|
+
if (c.code === "check_failed" || c.failed) return new HttpError(503, `Your text edits could not be checked: ${c.reason || c.why || "the check did not finish"}. Nothing was written; your edits are still in this page. Try again in a moment.`, { code: "check_failed" });
|
|
220
|
+
return new HttpError(422, `Your text edits were not written: ${c.reason}.`, { code: c.code, ...(c.reasons ? { reasons: c.reasons } : {}) });
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// The tasks the anchor work does for a request (src/anchor-work.js), and how a result is matched to its task: by the task's own content,
|
|
224
|
+
// so a comment changed while the worker ran finds no answer and goes out weaker instead of with another one's anchor.
|
|
225
|
+
const buildTask = (c) => ({ op: "build", anchor: c.anchor, pin: c.kind === "pin" });
|
|
226
|
+
const reanchorTask = (c) => ({ op: "reanchor", c: { durable: c.durable ?? null, anchor: c.durable ? null : c.anchor } });
|
|
227
|
+
const editAnchor = (e) => ({ location: e.location, quote: e.after, ...(e.cut ? { cut: true } : {}) });
|
|
228
|
+
const cardsTask = (results, ix, run) => ({ op: "cards", list: results.suggestions, units: ix.units, index: ix.index, reviewId: run.batch.batch_id });
|
|
229
|
+
// a quick fix's Use: the page anchor's location and text only (its kind of selection does not change where the sentence is)
|
|
230
|
+
const useTask = (a, rec, id) => ({ op: "use", anchor: { location: a.location, quote: a.quote, prefix: a.prefix, suffix: a.suffix }, rec: { quote: rec.quote, before: rec.before, after: rec.after }, id });
|
|
231
|
+
const taskKey = (t) => JSON.stringify(t);
|
|
232
|
+
/** A task's result that can be used: not missing, not an error of its own. A null anchor (the unit is not in the file) is a result. */
|
|
233
|
+
const usable = (r) => r !== undefined && !(r && typeof r === "object" && "error" in r && Object.keys(r).length === 1);
|
|
234
|
+
|
|
123
235
|
/**
|
|
124
236
|
* All session state and transitions, with no HTTP in it.
|
|
125
237
|
* States (derived): idle, listening, working, queued, stranded, closed. poll results add timeout and superseded.
|
|
@@ -128,17 +240,31 @@ function reanchor(units, c) {
|
|
|
128
240
|
* - Send while working: comments only, kept as `queued`; released to the next poller after `done`.
|
|
129
241
|
* - Lease expiry or Take back: `delivered` is withdrawn, the document unlocks.
|
|
130
242
|
*/
|
|
243
|
+
// What one request may spend scanning a whole reading for moved anchors (src/text.js scanBudget): about 0.1 to 0.3 s here. Fifty moved
|
|
244
|
+
// comments on a Word document of 26,000 paragraphs took 10 s before. Past it, the rest resolve as moved or removed, and an anchor being
|
|
245
|
+
// made is made without the text it cannot pay for (src/xlsx.js xlsxDurable). The server's routes make and resolve anchors in the anchor
|
|
246
|
+
// worker (src/anchor-worker.js, ten times this budget, within a time limit); the synchronous methods (Send, done, revert, resolve, Use of
|
|
247
|
+
// a quick fix) do it on the caller's thread under this budget.
|
|
248
|
+
export const SCAN_BUDGET = 40_000_000;
|
|
249
|
+
const REFUSED_KEPT = 64;
|
|
250
|
+
const REFUSED_MS = 2 * 60_000;
|
|
251
|
+
|
|
131
252
|
export class Sessions extends EventEmitter {
|
|
132
253
|
// delivered run -> styleChangedOutside() when it went out; memory only, so a run reloaded after a restart has none and is not flagged
|
|
133
254
|
#styleSeen = new WeakMap();
|
|
134
255
|
// delivered run -> guideChangedOutside() when it went out (the same idea for voice-guide.json); taking it also arms the guide's guard
|
|
135
256
|
#guideSeen = new WeakMap();
|
|
136
|
-
|
|
257
|
+
// the last outline built for an agent Onionskin started: {key, sha, all} (one entry; ceiling: two such agents in turn build twice)
|
|
258
|
+
#outline = null;
|
|
259
|
+
// `anchors`: the anchor worker's call (src/offload.js anchorsOffThread); a test passes its own to control when the work answers
|
|
260
|
+
constructor({ now = Date.now, anchors = anchorsOffThread } = {}) {
|
|
137
261
|
super();
|
|
138
262
|
this.now = now;
|
|
263
|
+
this.anchors = anchors;
|
|
139
264
|
this.map = new Map(); // key -> session
|
|
140
265
|
this.createdCopies = new Set(); // absolute paths of conflict copies THIS server wrote: the only paths `showCopy` will reveal
|
|
141
266
|
this.editors = new Map(); // key -> id of the one browser tab that may edit (in memory only: a tab's id lives as long as its page)
|
|
267
|
+
this.handovers = new Map(); // key -> {tab, until}: the tab another tab just took editing from (memory only), see handOver()
|
|
142
268
|
this.briefed = new Map(); // key -> Set of holders that already received a complete batch in this session (memory only: a restarted server briefs again)
|
|
143
269
|
// key -> how the file got from one version to the next, oldest first: {from, to, route: "apply", ops, partial, anchors} for an agent's
|
|
144
270
|
// apply, {from, to, route: "commit"} for a commit, {from, to, route: "merge", stable} for a pass of the user's merge. The merge reads it
|
|
@@ -157,12 +283,69 @@ export class Sessions extends EventEmitter {
|
|
|
157
283
|
if (s.closed) return false; // a closed session has no editor; the tab that reopens it claims it
|
|
158
284
|
if (cur && cur !== tab && !takeover) return false;
|
|
159
285
|
if (cur !== tab) {
|
|
286
|
+
if (cur) this.handovers.set(key, { tab: cur, until: this.now() + HANDOVER_MS }); // the tab that lost editing may hand over its unsaved words once
|
|
160
287
|
this.editors.set(key, tab);
|
|
161
288
|
this.emit("change", s, "state");
|
|
162
289
|
}
|
|
163
290
|
return true;
|
|
164
291
|
}
|
|
165
292
|
|
|
293
|
+
/**
|
|
294
|
+
* The tab that just lost editing to another ("Edit here instead") hands over the words it had not saved yet: once, within HANDOVER_MS, and
|
|
295
|
+
* only that tab. Nothing another tab saved is replaced: a comment it made is added as a draft; words it typed into a draft nobody changed since
|
|
296
|
+
* it last saved it (`base`, per comment) replace it; words typed into one the store holds otherwise become a new draft next to it; its note is merged into the stored one from the note it started from (`base`, src/notes.js mergeNote),
|
|
297
|
+
* and the tab that edits now gets it as a note the server gave back (its stale save cannot overwrite it). Refused (409 `no_handover`) for
|
|
298
|
+
* any other tab or later, and like a draft save past a cap (413, nothing changes). The page that cannot hand over keeps the words visible
|
|
299
|
+
* with Copy my comments (web/src/App.jsx).
|
|
300
|
+
*/
|
|
301
|
+
// key -> (tab -> {sent, seq}): the note the newest draft save of each tab carried, and the session `seq` it stored at (memory only, the last 8 tabs). A tab that hands over a note the server already took
|
|
302
|
+
// from it (its save's answer was lost, so it did not know) hands over nothing new: whatever happened to that note since is not its loss
|
|
303
|
+
#noteFrom = new Map();
|
|
304
|
+
handOver(key, tab, { comments, note, base, baseSeq, noteRev } = {}) {
|
|
305
|
+
const s = this.get(key);
|
|
306
|
+
const h = this.handovers.get(key);
|
|
307
|
+
if (!h || h.tab !== tab || h.until < this.now()) throw new HttpError(409, "This tab no longer edits the file.", { code: "no_handover" });
|
|
308
|
+
if (!Array.isArray(comments)) throw new HttpError(400, "comments must be an array");
|
|
309
|
+
const byId = new Map(s.comments.map((c) => [c.id, c]));
|
|
310
|
+
const next = [...s.comments];
|
|
311
|
+
for (const c of comments.filter((x) => x && typeof x === "object")) {
|
|
312
|
+
const id = cutText(String(c.id), 64);
|
|
313
|
+
const old = byId.get(id);
|
|
314
|
+
const text = userText(c.text, COMMENT_TEXT_MAX);
|
|
315
|
+
const del = s.deletedDrafts?.[id];
|
|
316
|
+
if (!old && del === text) continue; // deleted in the tab that edits now, with these very words: it stays deleted
|
|
317
|
+
if (!old) next.push(this.#draft(c, del === undefined ? id : cutText(`${id}~h${crypto.randomBytes(3).toString("hex")}`, 64), null)); // deleted there, with other words: these come back as a new draft
|
|
318
|
+
else if (text === old.text) continue;
|
|
319
|
+
else if (old.status === "draft" && !old.answer && typeof c.base === "string" && userText(c.base, COMMENT_TEXT_MAX) === old.text) next[next.indexOf(old)] = this.#draft(c, id, old); // nobody changed it since: the newer words replace it
|
|
320
|
+
else next.push(this.#draft(c, cutText(`${id}~h${crypto.randomBytes(3).toString("hex")}`, 64), null));
|
|
321
|
+
}
|
|
322
|
+
if (next.length > COMMENTS_MAX) throw tooMany();
|
|
323
|
+
for (const c of next) if (c.status === "draft" && jsonSize(c) > COMMENT_MAX) throw tooLarge("This draft");
|
|
324
|
+
let text = s.note;
|
|
325
|
+
const from = this.#noteFrom.get(key);
|
|
326
|
+
const last = from?.get(tab);
|
|
327
|
+
const already = typeof note === "string" && (last?.sent === userText(note, NOTE_MAX + 1) || this.#noteTaken(s, noteRev, tab)); // or a Send of that tab took it
|
|
328
|
+
if (typeof note === "string" && note !== s.note && !already) {
|
|
329
|
+
// what it started from: the note the page's box is based on (`base`) when that came from a state at or after that tab's last save
|
|
330
|
+
// (`baseSeq`: the page took that save's answer, or something newer, as a Take back's); else the note that save carried (its answer
|
|
331
|
+
// was lost, so the box is that note and what was typed since); with no save of that tab, what the page says
|
|
332
|
+
const fresh = typeof base === "string" && Number.isSafeInteger(baseSeq) && (!last || baseSeq >= last.seq);
|
|
333
|
+
const from = fresh ? base : last ? last.sent : typeof base === "string" ? base : "";
|
|
334
|
+
const mine = noteOf(note);
|
|
335
|
+
// the box was emptied here since (a Send took the note that tab started from): only what that tab added after it is handed over
|
|
336
|
+
text = !s.note && from && mine.startsWith(from) ? mine.slice(from.length).replace(/^\s+/, "") : mergeNote(from, s.note, mine);
|
|
337
|
+
}
|
|
338
|
+
if ([...text].length > NOTE_MAX && [...text].length > [...s.note].length) throw noteTooLong(true);
|
|
339
|
+
const before = s.note;
|
|
340
|
+
this.#commitDraft(s, { comments: next, note: before }, null, tab, true);
|
|
341
|
+
this.handovers.delete(key);
|
|
342
|
+
if (text !== before) {
|
|
343
|
+
this.#giveNote(s, text);
|
|
344
|
+
this.#changed(s, "draft");
|
|
345
|
+
} else this.emit("change", s, "draft");
|
|
346
|
+
return s;
|
|
347
|
+
}
|
|
348
|
+
|
|
166
349
|
/** The tab's event stream ended (closed, reloaded): the file has no editor until a tab claims it. */
|
|
167
350
|
releaseEditor(key, tab) {
|
|
168
351
|
if (this.editors.get(key) !== tab) return;
|
|
@@ -178,8 +361,50 @@ export class Sessions extends EventEmitter {
|
|
|
178
361
|
}
|
|
179
362
|
|
|
180
363
|
// ---- open ------------------------------------------------------------------------------------------------
|
|
181
|
-
/**
|
|
182
|
-
|
|
364
|
+
/**
|
|
365
|
+
* open() as the server's routes call it: the file is first read in a worker thread with a time and memory limit (src/offload.js
|
|
366
|
+
* readOffThread: the package check and every part the server reads later), so a slow or hostile file never stalls the server, and a
|
|
367
|
+
* file no reader accepts is refused here, with the reason, instead of failing later at Send.
|
|
368
|
+
*/
|
|
369
|
+
async openAsync(input, opts = {}) {
|
|
370
|
+
const { real, format } = this.#checkInput(input);
|
|
371
|
+
// A file that changes between the worker's reading and open()'s own (a swap, a save in progress) is read in the worker again, once:
|
|
372
|
+
// open() never takes bytes the worker has not read, so nothing on this thread reads a version past the readers' caps.
|
|
373
|
+
for (let attempt = 0; ; attempt++) {
|
|
374
|
+
const bytes = readCapped(real);
|
|
375
|
+
const sha = sha256(bytes);
|
|
376
|
+
const known = this.#refused.get(`${format}:${sha}`);
|
|
377
|
+
const r = known && this.now() - known.at < REFUSED_MS ? known : await readOffThread(bytes, format);
|
|
378
|
+
if (r.ok) readInCaps(format, r.sha);
|
|
379
|
+
if (!r.ok) {
|
|
380
|
+
// the same bytes are refused again at once (an agent's polls fall back to opening a file that is not open, again and again)
|
|
381
|
+
if (r.stage !== "busy") this.#refuse(`${format}:${sha}`, r);
|
|
382
|
+
if (r.stage === "encoding") throw encodingRefusal(real, r);
|
|
383
|
+
if (r.stage === "invalid") throw new HttpError(400, `Not a valid .${format}: ${r.error}`);
|
|
384
|
+
if (r.stage === "busy") throw new HttpError(503, `Onionskin is busy opening other files: ${r.error}. Try again in a minute.`, { code: "busy" });
|
|
385
|
+
const hint = r.stage === "timeout" || r.stage === "memory" ? " It may be damaged, or made to stall programs that read it." : "";
|
|
386
|
+
throw new HttpError(400, `This file could not be opened: ${r.error}.${hint}`, { code: r.stage === "timeout" ? "open_timeout" : "unreadable" });
|
|
387
|
+
}
|
|
388
|
+
try {
|
|
389
|
+
return this.open(input, { ...opts, checkedSha: r.sha, onlyChecked: true });
|
|
390
|
+
} catch (err) {
|
|
391
|
+
if (err?.extra?.code !== "changed_while_opening" || attempt >= 1) throw err;
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
// "<format>:<sha256>" -> what the open worker said about those bytes when it refused them, for REFUSED_MS (a refusal for time may not hold once the
|
|
397
|
+
// Mac is less busy); at most REFUSED_KEPT, oldest dropped first
|
|
398
|
+
#refused = new Map();
|
|
399
|
+
#refuse(id, r) {
|
|
400
|
+
if (r.at) return; // already a remembered answer
|
|
401
|
+
this.#refused.delete(id);
|
|
402
|
+
this.#refused.set(id, { ok: false, stage: r.stage, error: r.error, code: r.code, copyable: r.copyable, at: this.now() });
|
|
403
|
+
if (this.#refused.size > REFUSED_KEPT) this.#refused.delete(this.#refused.keys().next().value);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** The path checks of open(): the real path, its format and its lstat. Throws HttpError. */
|
|
407
|
+
#checkInput(input) {
|
|
183
408
|
let real;
|
|
184
409
|
try {
|
|
185
410
|
real = fs.realpathSync(path.resolve(input));
|
|
@@ -190,23 +415,36 @@ export class Sessions extends EventEmitter {
|
|
|
190
415
|
if (hasControlChars(real)) throw new HttpError(400, "This file's path contains control or bidirectional-text characters. Rename or move the file and open it again.", { code: "unsafe_path" });
|
|
191
416
|
const ext = path.extname(real).toLowerCase();
|
|
192
417
|
if (REFUSED_EXTENSIONS.has(ext)) throw new HttpError(400, `${ext} (macro-enabled) files are refused`);
|
|
193
|
-
|
|
194
|
-
|
|
418
|
+
if (Object.hasOwn(REFUSED_PAGES, ext)) throw new HttpError(400, REFUSED_PAGES[ext], { code: "refused_kind" });
|
|
419
|
+
const format = formatOfExt(ext);
|
|
420
|
+
if (!format) throw new HttpError(400, ONLY_FORMATS);
|
|
195
421
|
let st;
|
|
196
422
|
try {
|
|
197
423
|
st = checkRegularOwned(real);
|
|
198
424
|
} catch (err) {
|
|
199
425
|
throw new HttpError(400, err.message);
|
|
200
426
|
}
|
|
427
|
+
return { real, format, st };
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/** `opts.agent === "background"`: the person opened it from a terminal, so the server's background agent works on it (stored on the session). */
|
|
431
|
+
open(input, opts = {}) {
|
|
432
|
+
const { real, format, st } = this.#checkInput(input);
|
|
201
433
|
const bytes = readCapped(real);
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
434
|
+
const sha = sha256(bytes);
|
|
435
|
+
// openAsync already read these exact bytes in a worker; for it, other bytes (the file changed since) go back to the worker. Any other
|
|
436
|
+
// caller (tests, tools) gets the package check here.
|
|
437
|
+
if (opts.onlyChecked && opts.checkedSha !== sha) throw new HttpError(409, "The file kept changing while Onionskin opened it. Open it again once it is saved.", { code: "changed_while_opening" });
|
|
438
|
+
if (opts.checkedSha !== sha) {
|
|
439
|
+
try {
|
|
440
|
+
validatePackage(bytes, format);
|
|
441
|
+
} catch (err) {
|
|
442
|
+
if (err instanceof HtmlEncodingError) throw encodingRefusal(real, { error: err.message, code: err.code, copyable: err.copyable });
|
|
443
|
+
throw new HttpError(400, `Not a valid .${format}: ${err.message}`);
|
|
444
|
+
}
|
|
206
445
|
}
|
|
207
446
|
const key = sessionKey(real);
|
|
208
447
|
let s = this.map.get(key);
|
|
209
|
-
const sha = sha256(bytes);
|
|
210
448
|
if (!s) {
|
|
211
449
|
s = this.#load(key) || this.#fresh(key, real, format, sha);
|
|
212
450
|
s.file = real;
|
|
@@ -269,8 +507,128 @@ export class Sessions extends EventEmitter {
|
|
|
269
507
|
}
|
|
270
508
|
|
|
271
509
|
#save(s) {
|
|
510
|
+
const { raw, cut } = this.#encode(s);
|
|
511
|
+
if (cut) Object.assign(s, cut); // in memory too
|
|
512
|
+
writeStateFile(path.join(sub("sessions"), `${s.key}.json`), raw);
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
/**
|
|
516
|
+
* The session file's text for `s`: {raw, cut}. The one cap on a session file (src/retain.js SESSION_MAX, in bytes): past it every
|
|
517
|
+
* comment's page anchor and every typed edit are cut further (`cut`: those lists, for the caller to keep); nothing is dropped. Each part
|
|
518
|
+
* is cut where it is stored, so this is a last line. `raw` can still be past the cap; a change the page asked for is then refused (#fits).
|
|
519
|
+
*/
|
|
520
|
+
#encode(s) {
|
|
272
521
|
const { pollers, snapshots, ...rest } = s;
|
|
273
|
-
|
|
522
|
+
const raw = JSON.stringify(rest);
|
|
523
|
+
if (Buffer.byteLength(raw) <= SESSION_MAX) return { raw, cut: null };
|
|
524
|
+
const cut = {
|
|
525
|
+
comments: rest.comments.map((c) => capComment({ ...c, anchor: c.anchor && { ...c.anchor, cut: true, quote: cutText(c.anchor.quote, 2000), paraText: cutText(c.anchor.paraText, 2000) } })),
|
|
526
|
+
edits: rest.edits.map((e) => ({ ...e, before: cutText(e.before, 2000), after: cutText(e.after, 2000), cut: true })),
|
|
527
|
+
};
|
|
528
|
+
return { raw: JSON.stringify({ ...rest, ...cut }), cut };
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
/** Whether the session with `patch` fits its file once cut (#encode): checked before a change the page asked for touches anything. */
|
|
532
|
+
#fits(s, patch) {
|
|
533
|
+
return Buffer.byteLength(this.#encode({ ...s, ...patch }).raw) <= SESSION_MAX;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
/** The page's typed edits since the last Send, as they are kept for the next batch: each field cut (src/retain.js), at most EDITS_MAX in all. */
|
|
537
|
+
#editsToTake(s, edits) {
|
|
538
|
+
return (Array.isArray(edits) ? edits.slice(0, Math.max(0, EDITS_MAX - s.edits.length)) : []).map(cleanEdit);
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// ---- anchor work: the anchor worker, or this thread (src/anchor-work.js) ------------------------------------------------
|
|
542
|
+
/**
|
|
543
|
+
* The results of `tasks` on the version `sha` (`bytes`: a Buffer, or a function that returns it). With `prepared` (what the anchor
|
|
544
|
+
* worker answered for the same tasks, src/session.js #offThread) they are taken from it, and nothing is read here: a task it has no
|
|
545
|
+
* answer for (its work did not finish, or it was for another version) reads as not done. Without, they are worked out on this thread
|
|
546
|
+
* under the request's budget, as the synchronous methods always did. Returns {read, outline, at(i)}.
|
|
547
|
+
*/
|
|
548
|
+
#results(format, bytes, sha, tasks, prepared, { check = false } = {}) {
|
|
549
|
+
if (prepared) {
|
|
550
|
+
const ok = prepared.ok && prepared.sha === sha;
|
|
551
|
+
return { read: ok ? prepared.read : null, outline: ok ? prepared.outline : null, at: (i) => (ok ? prepared.map.get(taskKey(tasks[i])) : undefined) };
|
|
552
|
+
}
|
|
553
|
+
// an HTML file is read in workers only (the parser is not on this thread): without the worker's answer the work is not done, and each
|
|
554
|
+
// caller makes its anchors weaker, as when the worker did not finish
|
|
555
|
+
if (format === "html") return { read: null, outline: null, at: () => undefined };
|
|
556
|
+
const r = anchorWork({ format, bytes, sha, check, tasks, budget: SCAN_BUDGET });
|
|
557
|
+
return { read: r.read, outline: null, at: (i) => r.results[i] };
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/** Bytes about to become the file: what their reading found. Past a cap 422 (CapError); not read in time and never read whole before: 503. */
|
|
561
|
+
#readVerdict(work, format, sha, refusal) {
|
|
562
|
+
const read = work.read;
|
|
563
|
+
if (read?.ok) return readInCaps(format, sha);
|
|
564
|
+
if (read?.cap) throw new CapError(read.cap);
|
|
565
|
+
if (read?.invalid) throw new HttpError(422, `${refusal}: ${untrusted(read.invalid, 200)}`, { code: "invalid_package" });
|
|
566
|
+
if (verified.has(`${format}:${sha}`)) return; // the work did not finish, but this version was read whole within the caps before
|
|
567
|
+
throw new HttpError(503, "Onionskin could not finish reading the file in time, so nothing was sent. Your comments are kept. Try again in a moment.", { code: "read_timeout" });
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
/** `job` ({source, check, tasks, outline}) in the anchor worker: {ok: true, sha, read, outline, map: task key -> result} or {ok: false, why}. */
|
|
571
|
+
async #offThread(format, job) {
|
|
572
|
+
const r = await this.anchors({ format, ...job });
|
|
573
|
+
if (!r.ok) return { ok: false, why: r.failed };
|
|
574
|
+
const map = new Map();
|
|
575
|
+
job.tasks.forEach((t, i) => map.set(taskKey(t), r.r.results[i]));
|
|
576
|
+
return { ok: true, sha: r.sha, read: r.r.read, outline: r.r.outline ?? null, map };
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
// the async methods of one file run in turn (another file's never wait for them); synchronous ones keep their own order
|
|
580
|
+
#chains = new Map();
|
|
581
|
+
#serial(key, fn) {
|
|
582
|
+
const prev = this.#chains.get(key) || Promise.resolve();
|
|
583
|
+
const run = prev.then(fn, fn);
|
|
584
|
+
const tail = run.then(() => {}, () => {});
|
|
585
|
+
this.#chains.set(key, tail);
|
|
586
|
+
tail.then(() => this.#chains.get(key) === tail && this.#chains.delete(key));
|
|
587
|
+
return run;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* `fn` (a synchronous change the page asks for: a draft save, Take back, an answer) in this file's turn, as the server's routes call it.
|
|
592
|
+
* One that arrives while a Send waits for the anchor worker is applied after that Send commits, against what it committed, as when Send
|
|
593
|
+
* was synchronous. Never called from inside a turn (it would wait for itself).
|
|
594
|
+
*/
|
|
595
|
+
inTurn(key, fn) {
|
|
596
|
+
this.get(key);
|
|
597
|
+
return this.#serial(key, () => fn());
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
/** setDraft() in this file's turn (the route POST /api/draft). */
|
|
601
|
+
setDraftAsync(key, args) {
|
|
602
|
+
return this.inTurn(key, () => this.setDraft(key, args));
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
/**
|
|
606
|
+
* Resolves once every request of this file that arrived before now is answered, so a page that loads (or reloads) during a Send starts
|
|
607
|
+
* from what it committed, never from the middle of it. Bounded: a Send plans its worker job at most twice, each within the worker's time
|
|
608
|
+
* limit; past that the page gets the state as it is then.
|
|
609
|
+
*/
|
|
610
|
+
async settled(key, ms = 2 * anchorTimeoutMs() + 5_000) {
|
|
611
|
+
const tail = this.#chains.get(key);
|
|
612
|
+
if (!tail) return;
|
|
613
|
+
let timer;
|
|
614
|
+
await Promise.race([tail, new Promise((r) => (timer = setTimeout(r, ms)))]);
|
|
615
|
+
clearTimeout(timer);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
/**
|
|
619
|
+
* An agent Onionskin started polls while a batch waits for it (src/server.js /cli/poll): the outline its delivery carries is built in
|
|
620
|
+
* the anchor worker first, so the delivery finds it (#withOutline's memo) instead of reading the file on this thread.
|
|
621
|
+
*/
|
|
622
|
+
async outlineAhead(key, holder) {
|
|
623
|
+
const s = this.get(key);
|
|
624
|
+
if (!isAgentHolder(holder) || !s.pending || s.delivered || s.pending.instructions?.review || s.pending.instructions?.voice_request) return;
|
|
625
|
+
const r = await anchorsOffThread({ format: s.format, source: { path: s.file }, tasks: [], outline: true });
|
|
626
|
+
if (r.ok && r.r.read.ok && r.r.outline) this.#outline = { key: s.key, sha: r.sha, all: r.r.outline };
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/** An agent Onionskin started is waiting for this file: its delivery carries the outline, which the anchor worker then builds too. */
|
|
630
|
+
#wantsOutline(s) {
|
|
631
|
+
return s.pollers.some((p) => isAgentHolder(p.holder));
|
|
274
632
|
}
|
|
275
633
|
|
|
276
634
|
get(key) {
|
|
@@ -325,6 +683,9 @@ export class Sessions extends EventEmitter {
|
|
|
325
683
|
strandedCommand: state === "stranded" ? `${PRODUCT.bin} poll ${shellQuote(s.file)}` : null,
|
|
326
684
|
startCommand: state === "stranded" ? `${PRODUCT.bin} ${shellQuote(s.file)}` : null, // what a person types in a terminal to start an agent on this file
|
|
327
685
|
seq: s.seq,
|
|
686
|
+
// per tab, the newest Send (or quick fix's Use) of it that committed: {tab: {rev, noteRev}}. A page whose Send got no answer learns
|
|
687
|
+
// from it whether that Send went through (web/src/App.jsx checkSent)
|
|
688
|
+
sent: s.sent || {},
|
|
328
689
|
editor: this.editors.get(s.key) || null,
|
|
329
690
|
conflict: s.conflict || null,
|
|
330
691
|
canReveal: canReveal(),
|
|
@@ -343,31 +704,198 @@ export class Sessions extends EventEmitter {
|
|
|
343
704
|
}
|
|
344
705
|
|
|
345
706
|
// ---- drafts ----------------------------------------------------------------------------------------------
|
|
346
|
-
|
|
347
|
-
|
|
707
|
+
// key -> {tab, rev}: the newest save or Send a tab numbered (memory only: a reloaded page is a new tab and counts again). Saves and Sends
|
|
708
|
+
// run in turn (inTurn), but two requests of one tab can still arrive out of order: an autosave sent just before Send was pressed can land
|
|
709
|
+
// after the Send committed, and would put the note the Send took back in the box. One numbered at or below what the tab already sent
|
|
710
|
+
// changes nothing (test/send-race.test.js).
|
|
711
|
+
#revs = new Map();
|
|
712
|
+
// s.sent = {tab: {rev, noteRev}}: per tab (the last SENT_TABS), the newest numbered Send (or Use) of it that committed, and the revision
|
|
713
|
+
// of its note box the newest such Send took. Stored with the session, so a page still asking whether its Send went through gets the right
|
|
714
|
+
// answer after the server restarts. A save of that tab whose note is that revision or older (its save on the way out while the Send was in
|
|
715
|
+
// flight) leaves the note alone, and so does its handover: a note a Send delivered is never put back.
|
|
716
|
+
#sentRev(s, { rev, tab, noteRev }, took = false) {
|
|
717
|
+
if (!Number.isSafeInteger(rev) || typeof tab !== "string") return;
|
|
718
|
+
const all = s.sent && typeof s.sent === "object" ? s.sent : {};
|
|
719
|
+
const last = all[tab];
|
|
720
|
+
if (last && last.rev > rev) return;
|
|
721
|
+
const { [tab]: _, ...rest } = all;
|
|
722
|
+
const entry = { rev, noteRev: took && Number.isSafeInteger(noteRev) ? noteRev : last?.noteRev ?? null };
|
|
723
|
+
s.sent = Object.fromEntries([...Object.entries(rest), [tab, entry]].slice(-SENT_TABS));
|
|
724
|
+
}
|
|
725
|
+
/** Whether `noteRev` of `tab` is a note a committed Send of that tab already took (see s.sent). */
|
|
726
|
+
#noteTaken(s, noteRev, tab) {
|
|
727
|
+
const last = typeof tab === "string" ? s.sent?.[tab] : null;
|
|
728
|
+
return Number.isSafeInteger(noteRev) && Number.isSafeInteger(last?.noteRev) && noteRev <= last.noteRev;
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
/**
|
|
732
|
+
* Browser replaces its list of comments/pins and the overall note. Server-owned fields (status, results) are kept: a sent, queued or
|
|
733
|
+
* finished comment is as the server holds it, whatever words the page sends for it. `note` absent (not a string): the stored note is
|
|
734
|
+
* kept (the page's save while a Send is in flight, which took the note). `rev` (optional): see #revs. Every field is rebuilt from what
|
|
735
|
+
* Onionskin reads (src/retain.js). Refused (413) before anything changes: more than COMMENTS_MAX comments, a note past NOTE_MAX, a
|
|
736
|
+
* comment past COMMENT_MAX after its cuts, a session past SESSION_MAX after the writer's. Nothing is dropped or cut without a word.
|
|
737
|
+
*/
|
|
738
|
+
setDraft(key, { comments, note, rev, tab, noteRev, seen, noteSeen } = {}) {
|
|
348
739
|
const s = this.get(key);
|
|
349
740
|
if (!Array.isArray(comments)) throw new HttpError(400, "comments must be an array");
|
|
741
|
+
if (this.#overtaken(s, rev, tab, true)) return s;
|
|
742
|
+
this.#commitDraft(s, this.#draftPlan(s, { comments, note: this.#noteTaken(s, noteRev, tab) ? undefined : note, seen, noteSeen, tab }), rev, tab);
|
|
743
|
+
if (typeof note === "string" && tab) {
|
|
744
|
+
const m = this.#noteFrom.get(key) || new Map();
|
|
745
|
+
m.delete(tab);
|
|
746
|
+
m.set(tab, { sent: userText(note, NOTE_MAX + 1), seq: s.seq, used: !this.#noteTaken(s, noteRev, tab) }); // see handOver and #draftPlan
|
|
747
|
+
if (m.size > 8) m.delete(m.keys().next().value); // the tabs of one file: a handful
|
|
748
|
+
this.#noteFrom.set(key, m);
|
|
749
|
+
}
|
|
750
|
+
return s;
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
/** Whether this tab already numbered a newer save or Send than `rev` (`orSame`: one numbered the same too). See #revs. */
|
|
754
|
+
#overtaken(s, rev, tab, orSame = false) {
|
|
755
|
+
if (!Number.isSafeInteger(rev)) return false;
|
|
756
|
+
const last = this.#revs.get(s.key);
|
|
757
|
+
return !!last && last.tab === tab && (orSame ? rev <= last.rev : rev < last.rev);
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/**
|
|
761
|
+
* What a draft save of `comments` and `note` would store, {comments, note}, with every limit checked and nothing changed. `note` absent:
|
|
762
|
+
* the stored note. A note past NOTE_MAX is refused, except one not longer than the stored note (Take back can join notes past it: the
|
|
763
|
+
* page saves the box it was given, and may shorten it).
|
|
764
|
+
*/
|
|
765
|
+
#draftPlan(s, { comments, note, seen, noteSeen, tab }) {
|
|
766
|
+
if (!Array.isArray(comments)) throw new HttpError(400, "comments must be an array");
|
|
767
|
+
const list = comments.filter((c) => c && typeof c === "object");
|
|
768
|
+
if (list.length > COMMENTS_MAX) throw tooMany();
|
|
769
|
+
const long = typeof note === "string" && note.length > NOTE_MAX && note.length <= s.note.length;
|
|
770
|
+
let text = typeof note !== "string" ? s.note : long ? userText(note, s.note.length) : noteOf(note);
|
|
771
|
+
// the server gave a note back to the box (#giveNote) that the page had not seen when it sent this (`seen`: the newest session `seq` it
|
|
772
|
+
// took): the page's note is merged into it, never put over it
|
|
773
|
+
// (`noteSeen`: the newest state whose note the page's box took or is based on; a page can see a state and keep its own words)
|
|
774
|
+
const given = s.noteGiven;
|
|
775
|
+
const ns = Number.isSafeInteger(noteSeen) ? noteSeen : seen;
|
|
776
|
+
if (typeof note === "string" && given && Number.isSafeInteger(ns) && ns <= given.seq) {
|
|
777
|
+
// from the note the server held before the first give the page had not seen (`steps`: each give; a session stored before them has
|
|
778
|
+
// the first one's only); or, when a save of this tab was merged in since (its answer still on its way), from the note that save
|
|
779
|
+
// carried: the box is that note and what was typed since, and the stored note already has that save's words
|
|
780
|
+
// (ceiling: the last 50 gives; past that, a page that saw none of the kept ones merges from the oldest note, as before `steps`)
|
|
781
|
+
const hit = given.steps?.find((g) => g.seq >= ns);
|
|
782
|
+
const first = hit && !(given.cut && hit === given.steps[0]) ? hit : given;
|
|
783
|
+
const last = this.#noteFrom.get(s.key)?.get(tab);
|
|
784
|
+
text = mergeNote(last?.used && last.seq > first.seq ? last.sent : first.before, s.note, text);
|
|
785
|
+
}
|
|
350
786
|
const prev = new Map(s.comments.map((c) => [c.id, c]));
|
|
351
|
-
|
|
352
|
-
const
|
|
353
|
-
const
|
|
354
|
-
|
|
355
|
-
kind: ["text", "element", "pin"].includes(c.kind) ? c.kind : "element",
|
|
356
|
-
text: userText(c.text),
|
|
357
|
-
anchor: c.anchor && typeof c.anchor === "object" ? c.anchor : null,
|
|
358
|
-
pin: c.pin && typeof c.pin === "object" ? cleanPin(c.pin) : null,
|
|
359
|
-
pinNo: Number.isInteger(c.pinNo) ? c.pinNo : null,
|
|
360
|
-
};
|
|
361
|
-
if (old && old.status !== "draft") return { ...old, text: old.text, kind: old.kind }; // sent/done comments are frozen
|
|
362
|
-
// an answered question: the words, the question and the answer stay as the person left them (the page cannot change them)
|
|
363
|
-
const keep = old?.skipLocal ? { skipLocal: true } : {}; // a reverted local run: the next Send of this comment goes to the agent
|
|
364
|
-
if (old?.localHint && userText(c.text) === old.text) keep.localHint = old.localHint; // the words changed: the hint no longer applies
|
|
365
|
-
if (old?.answer) return { ...base, text: old.text, status: "draft", durable: old.durable || null, question: old.question, answer: old.answer, ...keep };
|
|
366
|
-
return { ...base, status: "draft", durable: old?.durable || null, ...keep };
|
|
787
|
+
const next = list.map((c) => {
|
|
788
|
+
const id = cutText(String(c.id), 64);
|
|
789
|
+
const old = prev.get(id);
|
|
790
|
+
return old && old.status !== "draft" ? old : this.#draft(c, id, old); // sent/queued/finished comments are frozen
|
|
367
791
|
});
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
792
|
+
// a comment on its way to the agent (in a waiting batch, or queued) stays whatever list a save carries: a save never changes what a
|
|
793
|
+
// Send delivers (a page that had not seen it yet, after a reload, does not take it out of the batch). Nor does a save delete a draft
|
|
794
|
+
// another tab wrote after the state its page took (`seen`): the page did not know of it (a window that handed over its words, the save
|
|
795
|
+
// of a page that was left landing after the reloaded page loaded)
|
|
796
|
+
const named = new Set(next.map((c) => c.id));
|
|
797
|
+
const unseen = (c) => c.status === "draft" && Number.isSafeInteger(seen) && c.wroteBy && c.wroteBy.tab !== tab && c.wroteBy.seq > seen;
|
|
798
|
+
for (const c of s.comments) if ((c.status === "sent" || c.status === "queued" || unseen(c)) && !named.has(c.id)) next.push(c);
|
|
799
|
+
if (next.length > COMMENTS_MAX) throw tooMany();
|
|
800
|
+
for (const c of next) if (c.status === "draft" && jsonSize(c) > COMMENT_MAX) throw tooLarge("This draft");
|
|
801
|
+
return { comments: next, note: text, seen: Number.isSafeInteger(noteSeen) ? noteSeen : seen };
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
/**
|
|
805
|
+
* The server puts `text` in the note box itself (Take back, a run that stopped with comments queued behind it, the Revert of a run on this
|
|
806
|
+
* Mac). Until a save shows the page has seen it, a save of the note from the page is merged into it (#draftPlan): the note it had then is kept.
|
|
807
|
+
*/
|
|
808
|
+
#giveNote(s, text) {
|
|
809
|
+
if (text === s.note) return;
|
|
810
|
+
const step = { seq: s.seq, before: s.note };
|
|
811
|
+
if (s.noteGiven) {
|
|
812
|
+
const steps = [...(s.noteGiven.steps || []), step];
|
|
813
|
+
s.noteGiven = { ...s.noteGiven, seq: s.seq, steps: steps.slice(-50), ...(steps.length > 50 || !s.noteGiven.steps ? { cut: true } : {}) };
|
|
814
|
+
} else s.noteGiven = { ...step, steps: [step] };
|
|
815
|
+
s.note = text;
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
/** Stores a plan of #draftPlan as a draft save: refused (413) past the session cap before anything changes. */
|
|
819
|
+
#commitDraft(s, plan, rev, tab, quiet = false) {
|
|
820
|
+
// each draft remembers which tab last wrote its words, and when (#draftPlan: a save does not delete one its page has not seen)
|
|
821
|
+
const prev = new Map(s.comments.map((c) => [c.id, c]));
|
|
822
|
+
for (const c of plan.comments) {
|
|
823
|
+
if (c.status !== "draft") continue;
|
|
824
|
+
const old = prev.get(c.id);
|
|
825
|
+
if (!old || old.status !== "draft" || old.text !== c.text) c.wroteBy = { tab: tab ?? null, seq: s.seq + 1 };
|
|
826
|
+
else if (old.wroteBy && c !== old) c.wroteBy = old.wroteBy;
|
|
827
|
+
}
|
|
828
|
+
// the words of drafts this save deletes, by id (the last DELETED_KEPT): a window handing over words afterwards does not bring them back
|
|
829
|
+
const kept = new Set(plan.comments.map((c) => c.id));
|
|
830
|
+
const gone = [...prev.values()].filter((c) => c.status === "draft" && !kept.has(c.id));
|
|
831
|
+
const deleted = gone.length ? Object.fromEntries([...Object.entries(s.deletedDrafts || {}), ...gone.map((c) => [c.id, c.text])].slice(-DELETED_KEPT)) : s.deletedDrafts;
|
|
832
|
+
const patch = { comments: plan.comments, note: plan.note, seq: s.seq + 1, ...(deleted ? { deletedDrafts: deleted } : {}) };
|
|
833
|
+
const { raw, cut } = this.#encode({ ...s, ...patch });
|
|
834
|
+
if (Buffer.byteLength(raw) > SESSION_MAX) throw tooLarge("This draft");
|
|
835
|
+
if (Number.isSafeInteger(rev) && !this.#overtaken(s, rev, tab)) this.#revs.set(s.key, { tab, rev }); // never lowered
|
|
836
|
+
Object.assign(s, patch, cut || {});
|
|
837
|
+
if (s.noteGiven && Number.isSafeInteger(plan.seen) && plan.seen > s.noteGiven.seq) delete s.noteGiven; // the page has seen it
|
|
838
|
+
writeStateFile(path.join(sub("sessions"), `${s.key}.json`), raw);
|
|
839
|
+
// `quiet`: a Send's (or a Use's) save, which its commit announces; an event for the moment between would show its note still in the box
|
|
840
|
+
if (!quiet) this.emit("change", s, "draft");
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
/** One draft as stored, from the page's comment `c` (`old`: the stored draft of the same id, if any). */
|
|
844
|
+
#draft(c, id, old) {
|
|
845
|
+
const base = {
|
|
846
|
+
id,
|
|
847
|
+
kind: ["text", "element", "pin"].includes(c.kind) ? c.kind : "element",
|
|
848
|
+
text: userText(c.text, COMMENT_TEXT_MAX),
|
|
849
|
+
anchor: cleanPageAnchor(c.anchor), // the fields the server and the page read, each cut (src/retain.js)
|
|
850
|
+
pin: c.pin && typeof c.pin === "object" ? cleanPin(c.pin) : null,
|
|
851
|
+
pinNo: Number.isInteger(c.pinNo) ? c.pinNo : null,
|
|
852
|
+
};
|
|
853
|
+
// an answered question: the words, the question and the answer stay as the person left them (the page cannot change them)
|
|
854
|
+
const keep = old?.skipLocal ? { skipLocal: true } : {}; // a reverted local run: the next Send of this comment goes to the agent
|
|
855
|
+
if (old?.localHint && base.text === old.text) keep.localHint = old.localHint; // the words changed: the hint no longer applies
|
|
856
|
+
if (old?.answer) return capComment({ ...base, text: old.text, status: "draft", durable: old.durable || null, question: old.question, answer: old.answer, ...keep });
|
|
857
|
+
return capComment({ ...base, status: "draft", durable: old?.durable || null, ...keep });
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
/**
|
|
861
|
+
* What a Send (or a quick fix's Use: `useId`, the stamp it delivers) stores before it commits, as #draftPlan: nothing changed.
|
|
862
|
+
* Invariant: a Send commits exactly its own payload, whatever draft save is numbered before or after it. Normally its list and note
|
|
863
|
+
* replace the stored ones, as a draft save does. When this tab already numbered a newer save (one sent after the Send was pressed whose
|
|
864
|
+
* body arrived first, as a page's visibility-change save can), that save is the stored list and the Send is authoritative only for what
|
|
865
|
+
* it delivers: the drafts it names (their words as the Send carried them) and its note. Everything else stays as the newer save left it:
|
|
866
|
+
* a comment it added is a draft, one it deleted stays deleted (the page cannot delete what a Send in flight carries), its note (none:
|
|
867
|
+
* it leaves the note to the Send). A Use delivers only its stamp, so then it changes nothing else, not the note either.
|
|
868
|
+
*/
|
|
869
|
+
#sentPlan(s, { comments, note, rev, tab, seen, noteSeen, sends }, useId = null) {
|
|
870
|
+
const list = Array.isArray(comments) ? comments : s.comments;
|
|
871
|
+
// `sends` (optional): the ids the page holds read only for this Send, the drafts it means to deliver; a comment the page still showed as
|
|
872
|
+
// queued or sent when Send was pressed (and could be typed into after) is not delivered by it
|
|
873
|
+
const named = Array.isArray(sends) ? new Set(sends.slice(0, COMMENTS_MAX).map((id) => cutText(String(id), 64))) : null;
|
|
874
|
+
if (!this.#overtaken(s, rev, tab)) {
|
|
875
|
+
const plan = this.#draftPlan(s, { comments: list, note: typeof note === "string" ? note : s.note, seen, noteSeen, tab });
|
|
876
|
+
return named && useId == null ? { ...plan, only: named } : plan;
|
|
877
|
+
}
|
|
878
|
+
const own = this.#draftPlan(s, { comments: list.filter((c) => c && typeof c === "object" && (useId == null || String(c.id) === useId)), note: useId == null && typeof note === "string" ? note : s.note, seen, noteSeen, tab });
|
|
879
|
+
const mine = new Map(own.comments.map((c) => [c.id, c]));
|
|
880
|
+
// `only`: the drafts this Send delivers (the others are the newer save's, and stay drafts)
|
|
881
|
+
return { comments: s.comments.map((c) => (c.status === "draft" && mine.has(c.id) ? mine.get(c.id) : c)), note: own.note, only: new Set([...mine.keys()].filter((id) => !named || useId != null || named.has(id))) };
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
/** The drafts of `comments` a Send with `plan` delivers: all of them, or those it names when a newer save was stored first. */
|
|
885
|
+
#delivers(comments, plan) {
|
|
886
|
+
return comments.filter((c) => c.status === "draft" && (!plan.only || plan.only.has(c.id)));
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
/**
|
|
890
|
+
* The 413 refusals of a Send, on the state its plan would make (nothing changed yet): the bytes past the cap, a note that joined with
|
|
891
|
+
* the one waiting (the batch an agent has not taken, or the queued note during a run) would be past NOTE_MAX, a session that would not
|
|
892
|
+
* fit its file. Run before the worker is asked and again at the commit, against the state then.
|
|
893
|
+
*/
|
|
894
|
+
#sendLimits(s, plan, payload) {
|
|
895
|
+
if (payload.bytes?.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
|
|
896
|
+
if (s.delivered) joinNotes(s.queuedNote, plan.note);
|
|
897
|
+
else joinNotes(s.pending && !readOnlyKind(s.pending) ? s.pending.instructions?.overall_note || "" : "", plan.note);
|
|
898
|
+
if (!this.#fits(s, { comments: plan.comments, edits: [...s.edits, ...this.#editsToTake(s, payload.edits)] })) throw tooLarge("This Send");
|
|
371
899
|
}
|
|
372
900
|
|
|
373
901
|
// ---- send ------------------------------------------------------------------------------------------------
|
|
@@ -376,58 +904,211 @@ export class Sessions extends EventEmitter {
|
|
|
376
904
|
* Compare-and-swap on the disk hash. Returns the session.
|
|
377
905
|
*/
|
|
378
906
|
send(key, payload) {
|
|
907
|
+
return scanBudget(SCAN_BUDGET, () => this.#send(key, payload));
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
/**
|
|
911
|
+
* send() as the server's route calls it: the reading of the bytes that become the file and every anchor of the batch are worked out
|
|
912
|
+
* first, in the warm anchor worker with a time limit (src/offload.js anchorsOffThread), then send() commits with what came back and
|
|
913
|
+
* reads nothing itself. Work that did not finish leaves its comments weaker (src/anchor-work.js weakDurable), never lost; bytes it
|
|
914
|
+
* could not read in time are refused with "try again" unless that version was read whole before.
|
|
915
|
+
* Invariant: a Send and the page's draft saves, Take back and answers of one file run in turn (inTurn), as when Send was synchronous.
|
|
916
|
+
* The Send saves its list and note when its turn comes, then commits exactly that; whatever arrives meanwhile waits and is applied after
|
|
917
|
+
* the commit, against what it committed. A page loaded meanwhile waits for it too (settled).
|
|
918
|
+
*/
|
|
919
|
+
async sendAsync(key, payload) {
|
|
920
|
+
const s = this.get(key);
|
|
921
|
+
const hasBytes = !!payload.bytes?.length;
|
|
922
|
+
if (s.closed || (s.delivered && hasBytes)) return this.send(key, payload); // refused there, before anything changes
|
|
923
|
+
return this.#serial(key, async () => {
|
|
924
|
+
// refused there, before anything changes: closed, or locked
|
|
925
|
+
if (s.closed || (s.delivered && hasBytes)) return this.send(key, payload);
|
|
926
|
+
// nothing is stored before the commit: the worker works on what the Send would store, every limit is checked on it first (a refusal
|
|
927
|
+
// here changes nothing) and again at the commit
|
|
928
|
+
let prepared = null;
|
|
929
|
+
let html = {};
|
|
930
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
931
|
+
const queued = !!s.delivered;
|
|
932
|
+
const plan = this.#sentPlan(s, payload);
|
|
933
|
+
this.#sendLimits(s, plan, payload);
|
|
934
|
+
if (s.format === "html") this.#htmlSendChecks(s, payload);
|
|
935
|
+
const job = this.#sendJob(s, payload, plan);
|
|
936
|
+
// an HTML file that is not queued: the person's text edits are written through the write check, and every anchor is made on the
|
|
937
|
+
// version it was made on, then moved onto the bytes this Send writes (#htmlSendWork)
|
|
938
|
+
if (s.format === "html" && !s.delivered) ({ prepared, html } = await this.#htmlSendWork(s, payload, job));
|
|
939
|
+
else prepared = job ? await this.#offThread(s.format, job) : null;
|
|
940
|
+
if (!!s.delivered === queued) break; // a run started or ended while it worked: plan again once, for the other case
|
|
941
|
+
}
|
|
942
|
+
return this.send(key, { ...payload, ...html, prepared });
|
|
943
|
+
});
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
/** An HTML Send's own refusals, before any work: no bytes from the browser ever (PLAN-html 3.1), no text edits during a run (D5). */
|
|
947
|
+
#htmlSendChecks(s, payload) {
|
|
948
|
+
if (payload.bytes?.length && !payload.htmlChecked) throw new HttpError(400, "an HTML file's bytes never come from the page; send its text edits", { code: "no_bytes_for_html" });
|
|
949
|
+
if (payload.htmlEdits?.length && s.delivered) throw new HttpError(423, s.delivered.local ? "the document is locked while a change is made on this Mac" : "the document is locked while the agent works", { code: "locked" });
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
/**
|
|
953
|
+
* The work of an HTML Send that is not queued, off this thread (the parser exists only in workers):
|
|
954
|
+
* 1. the person's text edits (R9), when there are any: on the version the page loaded, which must be the file on disk (else 409
|
|
955
|
+
* version_changed: the page reloads the view and places its edits again), applied by the write check in a worker (src/html-write.js
|
|
956
|
+
* checkedHtmlWrite "edits": one write, the proof, the inventory). Its bytes become the file at the commit (#takePage), which checks
|
|
957
|
+
* the disk hash again after these awaits (R12).
|
|
958
|
+
* 2. the anchors: a page anchor names a node of ONE version (`location.version`, a node id means nothing in another). An anchor made on
|
|
959
|
+
* the version this Send writes is built there; one made on another version (a draft that outlived a reload, the typed edits'
|
|
960
|
+
* places when this Send changes the bytes) is built on that version first (the file on disk or a stored snapshot), then moved onto
|
|
961
|
+
* the bytes this Send writes (reanchor), as a queued comment is moved onto the agent's file at done.
|
|
962
|
+
* Returns {prepared (what #offThread returns, keyed by this Send's own tasks), html: {bytes, htmlChecked} or {}}.
|
|
963
|
+
*/
|
|
964
|
+
async #htmlSendWork(s, payload, job) {
|
|
965
|
+
let disk;
|
|
966
|
+
try {
|
|
967
|
+
checkRegularOwned(s.file);
|
|
968
|
+
disk = readCapped(s.file);
|
|
969
|
+
} catch (err) {
|
|
970
|
+
throw new HttpError(409, `cannot read the file: ${err.message}`, { code: "gone" });
|
|
971
|
+
}
|
|
972
|
+
const diskSha = sha256(disk);
|
|
973
|
+
let target = { bytes: disk, sha: diskSha };
|
|
974
|
+
let html = {};
|
|
975
|
+
const edits = cleanHtmlEdits(payload.htmlEdits);
|
|
976
|
+
if (edits.length) {
|
|
977
|
+
if (payload.baseSha !== diskSha || edits.some((e) => e.versionSha !== diskSha)) throw new HttpError(409, VERSION_CHANGED, { code: "version_changed", diskSha });
|
|
978
|
+
const c = await htmlWriteOffThread({ before: disk, edits: edits.map((e) => ({ nodeId: e.nodeId, text: e.newText, baseTextHash: e.baseTextHash })) });
|
|
979
|
+
if (!c.ok) throw htmlEditRefusal(c);
|
|
980
|
+
const bytes = Buffer.from(c.bytes.buffer, c.bytes.byteOffset, c.bytes.byteLength);
|
|
981
|
+
target = { bytes, sha: sha256(bytes) };
|
|
982
|
+
html = { bytes, htmlChecked: { from: diskSha, to: target.sha } };
|
|
983
|
+
}
|
|
984
|
+
const prepared = await this.#htmlAnchorWork(s, job.tasks, target, { disk, diskSha, check: true, outline: job.outline });
|
|
985
|
+
return { prepared, html };
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
/**
|
|
989
|
+
* `tasks` (build, reanchor and view tasks of one request) on the version `target` ({bytes, sha}), in the anchor worker, with every build
|
|
990
|
+
* task whose page anchor was made on another version (`location.version`) built on that version first (`known.disk` when it is the file
|
|
991
|
+
* on disk, else a stored snapshot) and moved onto `target`. One made on a version no longer stored is moved as a weak anchor (its quote
|
|
992
|
+
* searched). Returns what #offThread returns, its map keyed by the tasks given.
|
|
993
|
+
*/
|
|
994
|
+
async #htmlAnchorWork(s, tasks, target, { disk = null, diskSha = null, check = false, outline = false } = {}) {
|
|
995
|
+
const elsewhere = new Map(); // version -> [task index]
|
|
996
|
+
tasks.forEach((t, i) => {
|
|
997
|
+
const v = t.op === "build" ? t.anchor?.location?.version : null;
|
|
998
|
+
if (v && v !== target.sha) (elsewhere.get(v) || elsewhere.set(v, []).get(v)).push(i);
|
|
999
|
+
});
|
|
1000
|
+
const moved = new Map(); // task index -> the durable made on its own version (null: not there; undefined: not worked out)
|
|
1001
|
+
for (const [v, list] of elsewhere) {
|
|
1002
|
+
const source = v === diskSha && disk ? { bytes: disk } : s.snapshots.has(v) ? { path: s.snapshots.blob(v), sha: v } : null;
|
|
1003
|
+
if (!source) continue;
|
|
1004
|
+
const r = await this.#offThread("html", { source, tasks: list.map((i) => tasks[i]) });
|
|
1005
|
+
if (r.ok && r.read?.ok) list.forEach((i) => moved.set(i, r.map.get(taskKey(tasks[i]))));
|
|
1006
|
+
}
|
|
1007
|
+
const asked = tasks.map((t, i) => {
|
|
1008
|
+
if (!elsewhere.size || !(t.op === "build" && t.anchor?.location?.version && t.anchor.location.version !== target.sha)) return t;
|
|
1009
|
+
const d = moved.get(i);
|
|
1010
|
+
return { op: "reanchor", c: { durable: usable(d) && d ? d : weakDurable("html", t.anchor), anchor: null } };
|
|
1011
|
+
});
|
|
1012
|
+
const r = await this.#offThread("html", { source: { bytes: target.bytes }, check, tasks: asked, outline });
|
|
1013
|
+
if (!r.ok) return r;
|
|
1014
|
+
const map = new Map();
|
|
1015
|
+
asked.forEach((t, i) => map.set(taskKey(tasks[i]), r.map.get(taskKey(t))));
|
|
1016
|
+
return { ...r, sha: target.sha, map };
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
/** What a Send asks of the anchor worker (the same tasks #send makes, for the list it would store), or null when it needs nothing read. */
|
|
1020
|
+
#sendJob(s, payload, plan) {
|
|
1021
|
+
if (s.delivered) {
|
|
1022
|
+
const need = this.#delivers(plan.comments, plan).filter((c) => !c.durable);
|
|
1023
|
+
if (!need.length || !s.snapshots.has(s.delivered.preSha)) return null;
|
|
1024
|
+
return { source: { path: s.snapshots.blob(s.delivered.preSha), sha: s.delivered.preSha }, tasks: need.map(buildTask) };
|
|
1025
|
+
}
|
|
1026
|
+
const { tasks } = this.#sendTasks(s, this.#editsToTake(s, payload.edits), payload.viewState, plan);
|
|
1027
|
+
const bytes = payload.bytes?.length ? payload.bytes : null;
|
|
1028
|
+
return { source: bytes ? { bytes } : { path: s.file }, check: true, tasks, outline: this.#wantsOutline(s) };
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
/** The tasks of a Send that is not queued: every comment that goes out (drafts, and those of a batch still waiting), the typed edits without an anchor yet, the view. */
|
|
1032
|
+
#sendTasks(s, newEdits, viewState, plan = { comments: s.comments }) {
|
|
1033
|
+
const go = new Set(this.#delivers(plan.comments, plan));
|
|
1034
|
+
const eligible = plan.comments.filter((c) => go.has(c) || c.status === "sent");
|
|
1035
|
+
const edits = [...s.edits, ...newEdits].filter((e) => !e.durable);
|
|
1036
|
+
const vs = cleanView(viewState);
|
|
1037
|
+
return { eligible, edits, vs, tasks: [...eligible.map(buildTask), ...edits.map((e) => ({ op: "build", anchor: editAnchor(e), pin: false })), ...(vs ? [{ op: "view", vs }] : [])] };
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
/**
|
|
1041
|
+
* The commit of a Send: the Send's own list and note are saved first (#sentPlan), then every draft goes out with the note. A batch still
|
|
1042
|
+
* waiting for an agent (not delivered) is replaced by one that carries its comments and its note, then this one's; during a run the
|
|
1043
|
+
* drafts and the note are queued after those already queued. Every 413 (#sendLimits) is refused before anything changes.
|
|
1044
|
+
*/
|
|
1045
|
+
#send(key, payload) {
|
|
379
1046
|
const s = this.get(key);
|
|
380
1047
|
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
1048
|
+
if (s.format === "html") this.#htmlSendChecks(s, payload);
|
|
381
1049
|
const hasBytes = !!payload.bytes?.length;
|
|
382
1050
|
if (s.delivered && hasBytes) throw new HttpError(423, s.delivered.local ? "the document is locked while a change is made on this Mac" : "the document is locked while the agent works", { code: "locked" }); // before touching any state
|
|
383
|
-
|
|
1051
|
+
// the Send's own list and note (#sentPlan), every limit checked on them before anything changes; then stored as a draft save is, so a
|
|
1052
|
+
// refusal after this (the file changed, could not be read in time) leaves them as editable drafts
|
|
1053
|
+
const plan = this.#sentPlan(s, payload);
|
|
1054
|
+
this.#sendLimits(s, plan, payload);
|
|
1055
|
+
this.#commitDraft(s, plan, payload.rev, payload.tab, true);
|
|
1056
|
+
try {
|
|
1057
|
+
return this.#sendCommit(s, payload, plan);
|
|
1058
|
+
} catch (err) {
|
|
1059
|
+
this.emit("change", s, "draft"); // refused after the save: the page's tabs see the drafts it stored (a Send that commits says so once, at the end)
|
|
1060
|
+
throw err;
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
/** The rest of #send, once its list and note are stored: the batch (or the queue) and the commit. */
|
|
1065
|
+
#sendCommit(s, payload, plan) {
|
|
384
1066
|
if (s.delivered) {
|
|
385
1067
|
// queue: comments and note only. A new comment gets its durable anchor from the bytes the page shows (the pre-agent snapshot),
|
|
386
1068
|
// never from the disk, which the agent may already have changed; done() then moves it onto the post-agent file.
|
|
387
|
-
const
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
}
|
|
395
|
-
if (units) {
|
|
396
|
-
for (const c of fresh) {
|
|
397
|
-
if (c.durable) continue;
|
|
398
|
-
c.durable = buildDurable(units, c.anchor);
|
|
399
|
-
if (units.format === "pptx" && c.kind === "pin" && c.durable?.shapeId != null) c.durable.pinFallback = buildDurable(units, { location: { slideIndex: c.durable.slideIndex, slideId: c.durable.slideId } });
|
|
400
|
-
}
|
|
401
|
-
}
|
|
1069
|
+
const queuedNote = joinNotes(s.queuedNote, s.note); // before anything changes
|
|
1070
|
+
const fresh = this.#delivers(s.comments, plan);
|
|
1071
|
+
const need = fresh.filter((c) => !c.durable);
|
|
1072
|
+
if (need.length) {
|
|
1073
|
+
// no snapshot, or work that did not finish: the anchor stays unset, and done() builds it from the post-agent file
|
|
1074
|
+
const work = this.#results(s.format, () => s.snapshots.read(s.delivered.preSha), s.delivered.preSha, need.map(buildTask), payload.prepared);
|
|
1075
|
+
if (work.read?.ok) need.forEach((c, i) => (c.durable = usable(work.at(i)) ? work.at(i) : null));
|
|
402
1076
|
}
|
|
403
1077
|
for (const c of fresh) c.status = "queued";
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
s.note = "";
|
|
408
|
-
}
|
|
1078
|
+
// the note belongs to the queued batch now; it leaves the box so the next Send does not repeat it
|
|
1079
|
+
s.queuedNote = queuedNote;
|
|
1080
|
+
s.note = "";
|
|
409
1081
|
if (fresh.length || s.queuedNote) s.queued = true; // a Send with nothing new queues nothing (no empty batch after this one)
|
|
1082
|
+
this.#sentRev(s, payload, true);
|
|
410
1083
|
this.#changed(s);
|
|
411
1084
|
this.#voice(s, { sent: fresh });
|
|
412
1085
|
return s;
|
|
413
1086
|
}
|
|
414
|
-
|
|
1087
|
+
// a batch still waiting for an agent is replaced: its comments go again (they are "sent"), and so does its note, first
|
|
1088
|
+
const waiting = s.pending && !readOnlyKind(s.pending) ? s.pending.instructions?.overall_note || "" : "";
|
|
1089
|
+
const note = joinNotes(waiting, s.note); // before anything is written
|
|
1090
|
+
// the bytes that become the file are read whole before anything is written (one past a reader's cap is refused: CapError, nothing
|
|
1091
|
+
// changes), and every anchor of the batch is built from them, by the anchor worker (sendAsync) or here
|
|
1092
|
+
const newEdits = this.#editsToTake(s, payload.edits);
|
|
1093
|
+
const { eligible, edits, vs, tasks } = this.#sendTasks(s, newEdits, payload.viewState, { ...plan, comments: s.comments });
|
|
1094
|
+
// what is stored must fit its file once cut (src/retain.js SESSION_MAX): checked before the file is written or anything changes
|
|
1095
|
+
if (!this.#fits(s, { comments: s.comments, edits: [...s.edits, ...newEdits] })) throw tooLarge("This Send");
|
|
1096
|
+
let work = null;
|
|
1097
|
+
const { diskSha, writtenSha, writeInfo, lastUpdate } = this.#takePage(s, payload, (b, sha) => {
|
|
1098
|
+
work = this.#results(s.format, b, sha, tasks, payload.prepared, { check: true });
|
|
1099
|
+
this.#readVerdict(work, s.format, sha, "refusing to write");
|
|
1100
|
+
});
|
|
415
1101
|
|
|
416
1102
|
// user edits since the last Send (document-derived text) accumulate until a batch is delivered
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
const
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
}
|
|
427
|
-
for (const e of s.edits) e.durable = e.durable || resolve({ location: e.location, quote: e.after });
|
|
428
|
-
s.view = this.#resolveView(s, units, payload.viewState);
|
|
429
|
-
|
|
430
|
-
const batch = buildBatch({ file: s.file, format: s.format, baseSha: writtenSha, note: s.note, comments: eligible, edits: s.edits, style: styleNote(), guide: voiceGuide() });
|
|
1103
|
+
s.edits.push(...newEdits);
|
|
1104
|
+
// a comment whose anchor work did not finish goes out weaker (src/anchor-work.js weakDurable); a PPTX pin on a shape also remembers its slide
|
|
1105
|
+
eligible.forEach((c, i) => (c.durable = usable(work.at(i)) ? work.at(i) : weakDurable(s.format, c.anchor)));
|
|
1106
|
+
edits.forEach((e, i) => (e.durable = usable(work.at(eligible.length + i)) ? work.at(eligible.length + i) : weakDurable(s.format, editAnchor(e))));
|
|
1107
|
+
const v = vs ? work.at(eligible.length + edits.length) : null;
|
|
1108
|
+
s.view = !vs ? null : usable(v) ? v : { format: s.format, raw: vs };
|
|
1109
|
+
if (work.outline) this.#outline = { key: s.key, sha: writtenSha, all: work.outline };
|
|
1110
|
+
|
|
1111
|
+
const batch = buildBatch({ file: s.file, format: s.format, baseSha: writtenSha, note, comments: eligible, edits: s.edits, style: styleNote(), guide: voiceGuide() });
|
|
431
1112
|
const sent = eligible.filter((c) => c.status === "draft");
|
|
432
1113
|
for (const c of eligible) {
|
|
433
1114
|
c.status = "sent";
|
|
@@ -444,6 +1125,7 @@ export class Sessions extends EventEmitter {
|
|
|
444
1125
|
this.#deliverIfPolled(s);
|
|
445
1126
|
delete s.quickConfirm;
|
|
446
1127
|
delete s.skipLocalNote; // it held for this Send only (the decision for its batch is made)
|
|
1128
|
+
this.#sentRev(s, payload, true);
|
|
447
1129
|
this.#changed(s);
|
|
448
1130
|
this.#voice(s, { last: lastUpdate, diskSha, writtenSha, sent });
|
|
449
1131
|
if (this.state(s) === "stranded" && !s.pollers.length) this.emit("stranded", s); // nobody took it: the server may start its background agent
|
|
@@ -453,8 +1135,8 @@ export class Sessions extends EventEmitter {
|
|
|
453
1135
|
/**
|
|
454
1136
|
* The page's bytes become the file, as at Send (and at a quick fix's Use): the disk read, the compare-and-swap against the version
|
|
455
1137
|
* the page loaded, the package check, the guarded replace, the "sent" snapshot, the new base and the person's own history update.
|
|
456
|
-
* `check(bytes)` (optional) sees the bytes that are about to be the file, after the package check and before
|
|
457
|
-
* may throw to refuse. Returns {disk, diskSha, written, writtenSha, writeInfo, lastUpdate}.
|
|
1138
|
+
* `check(bytes, sha)` (optional) sees the bytes that are about to be the file and their sha256, after the package check and before
|
|
1139
|
+
* anything is written, and may throw to refuse. Returns {disk, diskSha, written, writtenSha, writeInfo, lastUpdate}.
|
|
458
1140
|
*/
|
|
459
1141
|
#takePage(s, payload, check = null) {
|
|
460
1142
|
const hasBytes = !!payload.bytes?.length;
|
|
@@ -466,25 +1148,33 @@ export class Sessions extends EventEmitter {
|
|
|
466
1148
|
throw new HttpError(409, `cannot read the file: ${err.message}`, { code: "gone" });
|
|
467
1149
|
}
|
|
468
1150
|
const diskSha = sha256(disk);
|
|
469
|
-
|
|
470
|
-
|
|
1151
|
+
// an HTML file's text edits were applied by its write check (#htmlSendWork) to the version the page loaded: no bytes come from the page
|
|
1152
|
+
const htmlChecked = s.format === "html" ? payload.htmlChecked ?? null : null;
|
|
1153
|
+
if (diskSha !== payload.baseSha || (htmlChecked && htmlChecked.from !== diskSha)) {
|
|
1154
|
+
// an HTML page sent no bytes: nothing of its to keep as a copy (its edits stay in the page, which reloads and places them again)
|
|
1155
|
+
throw this.#outsideChange(s, { side: "user", bytes: s.format === "html" ? null : payload.bytes, diskSha, baseSha: payload.baseSha });
|
|
471
1156
|
}
|
|
472
1157
|
let written = disk;
|
|
473
1158
|
let writeInfo = null;
|
|
474
1159
|
if (hasBytes) {
|
|
475
|
-
if (payload.bytes.length > MAX_BYTES) throw new HttpError(413,
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
}
|
|
479
|
-
|
|
1160
|
+
if (payload.bytes.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
|
|
1161
|
+
if (s.format === "html") {
|
|
1162
|
+
if (htmlChecked?.to !== sha256(payload.bytes)) throw new HttpError(500, "an HTML file is written only after its write check", { code: "unchecked_write" });
|
|
1163
|
+
} else {
|
|
1164
|
+
try {
|
|
1165
|
+
validatePackage(payload.bytes, s.format);
|
|
1166
|
+
} catch (err) {
|
|
1167
|
+
throw new HttpError(422, `refusing to write: ${err.message}`, { code: "invalid_package" });
|
|
1168
|
+
}
|
|
480
1169
|
}
|
|
481
1170
|
}
|
|
482
|
-
|
|
1171
|
+
const writtenSha = hasBytes ? sha256(payload.bytes) : diskSha;
|
|
1172
|
+
if (htmlChecked && hasBytes) readInCaps("html", writtenSha); // the write check read these bytes whole within the limits
|
|
1173
|
+
if (check) check(hasBytes ? payload.bytes : disk, writtenSha);
|
|
483
1174
|
if (hasBytes) {
|
|
484
|
-
writeInfo = this.#replace(s, payload.bytes, { expectSha: diskSha, label: "before send", side: "user" });
|
|
1175
|
+
writeInfo = this.#replace(s, payload.bytes, { expectSha: diskSha, label: "before send", side: "user", htmlChecked });
|
|
485
1176
|
written = payload.bytes;
|
|
486
1177
|
}
|
|
487
|
-
const writtenSha = sha256(written);
|
|
488
1178
|
s.snapshots.add(written, "sent");
|
|
489
1179
|
s.baseSha = writtenSha;
|
|
490
1180
|
const lastUpdate = s.history?.at(-1); // learning pairs an agent run with the person's edits right after it
|
|
@@ -508,16 +1198,21 @@ export class Sessions extends EventEmitter {
|
|
|
508
1198
|
* (DOCX paragraphs, formatting and moves: src/merge.js; XLSX cells: src/merge-xlsx.js; PPTX shape text: src/merge-pptx.js) is carried onto it as ops (same engine, expect, protected and package checks
|
|
509
1199
|
* as an agent's apply) and written as the user's own update. Refused while a batch is delivered (423): the page waits and asks
|
|
510
1200
|
* again. Returns {merged, unplaced, sha, same, mineSha, copy_name}.
|
|
511
|
-
*
|
|
1201
|
+
* The reading of the three versions within the readers' caps, the plan and the engine's run happen in a worker thread with a time and
|
|
1202
|
+
* memory limit (src/offload.js mergeOffThread): the server's thread stays free whatever the document. A merge that fails or runs out of
|
|
1203
|
+
* time writes nothing: the person's version is saved as a copy next to the file and the page keeps their edits (503 merge_failed).
|
|
1204
|
+
* One merge per session at a time (another is 423 locked; the page has only one in flight).
|
|
512
1205
|
*/
|
|
513
1206
|
async mergeUserEdits(key, { baseSha, bytes, edits = [] }) {
|
|
514
1207
|
const s = this.get(key);
|
|
515
1208
|
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
516
1209
|
if (s.delivered) throw new HttpError(423, s.delivered.local ? "a change is being made on this Mac" : "the agent is working", { code: "locked" });
|
|
517
|
-
|
|
1210
|
+
if (s.merging) throw new HttpError(423, "your edits are being merged already", { code: "locked" });
|
|
1211
|
+
// an HTML view is read only during a run (PLAN-html D5): nothing is typed to merge, and its bytes never come from the page
|
|
1212
|
+
if (s.format === "html") throw new HttpError(400, "an HTML file's edits are never merged: its view is read only while the agent works", { code: "not_for_html" });
|
|
518
1213
|
if (!/^[0-9a-f]{64}$/.test(baseSha)) throw new HttpError(400, "baseSha must be a sha256"); // it names a snapshot file
|
|
519
1214
|
if (!bytes?.length) throw new HttpError(400, "no document bytes");
|
|
520
|
-
if (bytes.length > MAX_BYTES) throw new HttpError(413,
|
|
1215
|
+
if (bytes.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
|
|
521
1216
|
try {
|
|
522
1217
|
validatePackage(bytes, s.format);
|
|
523
1218
|
} catch (err) {
|
|
@@ -539,8 +1234,19 @@ export class Sessions extends EventEmitter {
|
|
|
539
1234
|
} catch {
|
|
540
1235
|
throw new HttpError(409, "the version this page started from is no longer stored", { code: "no_base" });
|
|
541
1236
|
}
|
|
542
|
-
|
|
543
|
-
|
|
1237
|
+
// the three versions are read whole within the readers' caps in the worker first (the page's version, a file swapped on disk and an
|
|
1238
|
+
// old snapshot alike): past one, the merge is refused with the reason and nothing is written
|
|
1239
|
+
let r;
|
|
1240
|
+
s.merging = true;
|
|
1241
|
+
try {
|
|
1242
|
+
r = await mergeOffThread({ format: s.format, base, mine: bytes, theirs: disk, name: path.basename(s.file), edits: editKinds(edits), agent: s.format === "xlsx" ? this.#writesBetween(s, baseSha, diskSha, disk) : null });
|
|
1243
|
+
} finally {
|
|
1244
|
+
s.merging = false;
|
|
1245
|
+
}
|
|
1246
|
+
if (r.cap) throw new CapError(r.cap);
|
|
1247
|
+
if (r.invalid) throw new HttpError(422, `refusing to merge: ${untrusted(r.invalid, 200)}`, { code: "invalid_package" });
|
|
1248
|
+
if (!r.ok) throw this.#mergeFailed(s, bytes, r);
|
|
1249
|
+
if (r.bytes) r.bytes = Buffer.from(r.bytes.buffer, r.bytes.byteOffset, r.bytes.byteLength); // a Uint8Array once it crossed threads
|
|
544
1250
|
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
545
1251
|
if (s.delivered) throw new HttpError(423, s.delivered.local ? "a change is being made on this Mac" : "the agent is working", { code: "locked" }); // a batch was delivered while the engine ran
|
|
546
1252
|
// The page shows the first HELD_SHOWN items and drops its own version when it reloads. When the list stands for more than it
|
|
@@ -570,6 +1276,24 @@ export class Sessions extends EventEmitter {
|
|
|
570
1276
|
return { same: false, merged: r.merged, unplaced: r.unplaced.slice(0, 50), sha, mineSha: sha256(bytes), copy_name: copy?.name ?? null };
|
|
571
1277
|
}
|
|
572
1278
|
|
|
1279
|
+
/**
|
|
1280
|
+
* A merge that did not finish (out of time or memory, busy, crashed) or whose result the proof refused: nothing was written. The person's version is saved next to the
|
|
1281
|
+
* file (except when Onionskin was only busy) and the answer says so in words the page shows inside its own sentence ("could not be
|
|
1282
|
+
* merged into its version (...). Your edits are still in this page: download them before you reload.").
|
|
1283
|
+
*/
|
|
1284
|
+
#mergeFailed(s, bytes, r) {
|
|
1285
|
+
if (r.failed === "busy") return new HttpError(503, "Onionskin is merging other files right now; nothing was written, so try again in a moment", { code: "merge_busy" });
|
|
1286
|
+
let copy = null;
|
|
1287
|
+
try {
|
|
1288
|
+
copy = writeSiblingCopy(s.file, bytes);
|
|
1289
|
+
this.createdCopies.add(copy.path);
|
|
1290
|
+
} catch {}
|
|
1291
|
+
// refused: the proof of the merge (src/merge-proof.js) could not account for every edit of the person's in the result; the reason
|
|
1292
|
+
// holds counts and kinds of edits, never their text
|
|
1293
|
+
const why = r.failed === "timeout" || r.failed === "memory" ? r.why : r.refused ? `the merged file could not be checked to hold every one of your edits (${r.refused})` : "the merge stopped with an error";
|
|
1294
|
+
return new HttpError(503, `${why}; nothing was written${copy ? `, and a copy of your version was saved as ${copy.name} next to the file` : ""}`, { code: "merge_failed", copy_name: copy?.name ?? null });
|
|
1295
|
+
}
|
|
1296
|
+
|
|
573
1297
|
#logWrite(s, entry) {
|
|
574
1298
|
const list = this.writes.get(s.key) || [];
|
|
575
1299
|
list.push(entry);
|
|
@@ -610,7 +1334,9 @@ export class Sessions extends EventEmitter {
|
|
|
610
1334
|
* replacement (see replaceGuarded). `expectSha` is the hash the caller compared against; a file that is different now, or becomes
|
|
611
1335
|
* different just before the rename, is not overwritten (the newer bytes are snapshotted) and the caller gets the 409 conflict.
|
|
612
1336
|
*/
|
|
613
|
-
#replace(s, bytes, { expectSha, label, side = null }) {
|
|
1337
|
+
#replace(s, bytes, { expectSha, label, side = null, htmlChecked = null }) {
|
|
1338
|
+
// R1, R12: an HTML file is written only with bytes its write check passed in a worker (agentWriteChecked); every other path refuses
|
|
1339
|
+
if (s.format === "html" && htmlChecked?.to !== sha256(bytes)) throw new HttpError(500, "an HTML file is written only after its write check", { code: "unchecked_write" });
|
|
614
1340
|
try {
|
|
615
1341
|
return replaceGuarded(s.file, bytes, {
|
|
616
1342
|
expectSha,
|
|
@@ -684,14 +1410,6 @@ export class Sessions extends EventEmitter {
|
|
|
684
1410
|
return { stale: !s.delivered && !!loadedSha && diskSha !== loadedSha, diskSha };
|
|
685
1411
|
}
|
|
686
1412
|
|
|
687
|
-
#resolveView(s, units, vs) {
|
|
688
|
-
if (!vs) return null;
|
|
689
|
-
if (units.format === "docx" && vs.format !== "docx") return { format: s.format, raw: vs };
|
|
690
|
-
if (units.format !== "docx") return viewDurable(units, vs) || { format: s.format, raw: vs };
|
|
691
|
-
const d = durableFromEngine(units.paragraphs, { blockPath: vs.blockPath }, { quote: vs.quote });
|
|
692
|
-
return { format: "docx", durable: d, offsetPx: Number(vs.offsetPx) || 0, zoom: vs.zoom ?? "fit", fraction: vs.fraction };
|
|
693
|
-
}
|
|
694
|
-
|
|
695
1413
|
// ---- poll / delivery -------------------------------------------------------------------------------------
|
|
696
1414
|
/**
|
|
697
1415
|
* Registers a poller. Returns a handle {promise, cancel}. Resolves with the batch object, or {status:'superseded'|'closed'}.
|
|
@@ -776,17 +1494,30 @@ export class Sessions extends EventEmitter {
|
|
|
776
1494
|
* written to <state>/agents/<key>/outline/outline-<batch id>.jsonl (0600, atomic, one unit per line), the only folder that agent may read,
|
|
777
1495
|
* and the batch names it. Older outline files of the session go at each such delivery, and with the folder on Close.
|
|
778
1496
|
* Any other holder, a review (it has `units`) and a voice request (it quotes only the sources the person ticked) get the batch as stored.
|
|
779
|
-
* A file that cannot be read gets no outline (the agent declines what it cannot find); an outline file that cannot be written leaves the
|
|
780
|
-
* inline part with `outline_more`.
|
|
1497
|
+
* A file that cannot be read gets no outline, and `outline_unavailable` says why (the agent declines what it cannot find); an outline file that cannot be written leaves the
|
|
1498
|
+
* inline part with `outline_more`. The outline is built in the anchor worker at Send, at done and before a poll's delivery (outlineAhead),
|
|
1499
|
+
* and found here by the file's hash. Ceiling: a delivery that finds none (a synchronous caller, or the file changed in between) reads and
|
|
1500
|
+
* parses the file on this thread, as a review does; upgrade path as in unitsOf.
|
|
781
1501
|
*/
|
|
782
1502
|
#withOutline(s, batch, holder) {
|
|
783
1503
|
if (!isAgentHolder(holder) || batch.instructions?.review || batch.instructions?.voice_request) return batch;
|
|
784
1504
|
let all;
|
|
785
1505
|
try {
|
|
786
1506
|
checkRegularOwned(s.file);
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
1507
|
+
const bytes = readCapped(s.file);
|
|
1508
|
+
const sha = sha256(bytes);
|
|
1509
|
+
// the same version's outline is built once (an agent's polls and a Send's delivery read the same file), on the memo's reading of it
|
|
1510
|
+
if (this.#outline?.key === s.key && this.#outline.sha === sha) all = this.#outline.all;
|
|
1511
|
+
// an HTML file is never parsed on this thread: its outline comes from the anchor worker (Send, done, outlineAhead) or not at all
|
|
1512
|
+
else if (s.format === "html") return { ...batch, untrusted_document_evidence: { ...batch.untrusted_document_evidence, outline_unavailable: "Onionskin did not finish reading this page's outline in time. Use the anchors and their quotes; poll again for an outline." } };
|
|
1513
|
+
else {
|
|
1514
|
+
all = outlineOf(s.format, bytes, OUTLINE_FILE_MAX, unitsFor(s.format, bytes, sha));
|
|
1515
|
+
this.#outline = { key: s.key, sha, all };
|
|
1516
|
+
}
|
|
1517
|
+
} catch (err) {
|
|
1518
|
+
// the agent is told why it has no outline (a file changed past a reader's cap behind Onionskin, or gone), in the reader's plain words
|
|
1519
|
+
const why = err instanceof CapError ? `Onionskin cannot read this file: ${err.message}.` : "Onionskin cannot read this file as it is now.";
|
|
1520
|
+
return { ...batch, untrusted_document_evidence: { ...batch.untrusted_document_evidence, outline_unavailable: why } };
|
|
790
1521
|
}
|
|
791
1522
|
const inline = withOutline(batch, all, VOICE_BATCH_MAX_CHARS, { inlineMax: UNITS_MAX });
|
|
792
1523
|
const name = `outline-${batch.batch_id}.jsonl`;
|
|
@@ -848,7 +1579,7 @@ export class Sessions extends EventEmitter {
|
|
|
848
1579
|
#queuedToDrafts(s) {
|
|
849
1580
|
if (s.queued) {
|
|
850
1581
|
for (const c of s.comments) if (c.status === "queued") c.status = "draft";
|
|
851
|
-
s
|
|
1582
|
+
this.#giveNote(s, [s.queuedNote, s.note].filter(Boolean).join("\n\n"));
|
|
852
1583
|
s.queuedNote = "";
|
|
853
1584
|
}
|
|
854
1585
|
s.queued = false;
|
|
@@ -999,11 +1730,22 @@ export class Sessions extends EventEmitter {
|
|
|
999
1730
|
if (token != null && token !== d.lease.token) throw stale();
|
|
1000
1731
|
}
|
|
1001
1732
|
|
|
1733
|
+
/**
|
|
1734
|
+
* The delivered batch, for its lease holder only (`GET /cli/batch` with a claim): holder, batch id and token are all required and checked
|
|
1735
|
+
* before anything of the batch is handed out, so `reply` and `apply` read no anchors, comments or file text for a delivery that is not theirs.
|
|
1736
|
+
*/
|
|
1737
|
+
deliveredBatch(key, claim) {
|
|
1738
|
+
const s = this.get(key);
|
|
1739
|
+
if (!s.delivered) throw new HttpError(409, "no batch is delivered for this file, so the ticket is void", { code: "no_batch" });
|
|
1740
|
+
this.#assertLease(s, { ...claim, strict: true });
|
|
1741
|
+
return s.delivered.batch;
|
|
1742
|
+
}
|
|
1743
|
+
|
|
1002
1744
|
/**
|
|
1003
1745
|
* `apply` through a live session: the single authority for the write. Accepted only while a batch is delivered (state working or
|
|
1004
1746
|
* queued: both hold a live lease) and only for the lease holder; compare-and-swap on the disk hash; renews the lease.
|
|
1005
1747
|
*/
|
|
1006
|
-
agentWrite(key, { baseSha, bytes, holder, token, batchId, label = "before agent write", route = "commit", ops = null, partial = false }) {
|
|
1748
|
+
agentWrite(key, { baseSha, bytes, holder, token, batchId, label = "before agent write", route = "commit", ops = null, partial = false, htmlChecked = null }) {
|
|
1007
1749
|
const s = this.get(key);
|
|
1008
1750
|
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
1009
1751
|
const state = this.state(s);
|
|
@@ -1014,11 +1756,20 @@ export class Sessions extends EventEmitter {
|
|
|
1014
1756
|
// `commit` does not know the hash its candidate was edited from: the base is the last hash this server wrote or served for the delivery
|
|
1015
1757
|
if (baseSha == null) baseSha = s.delivered.knownSha ?? s.delivered.preSha;
|
|
1016
1758
|
if (!bytes?.length) throw new HttpError(400, "no bytes");
|
|
1017
|
-
if (bytes.length > MAX_BYTES) throw new HttpError(413,
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
throw new HttpError(
|
|
1759
|
+
if (bytes.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" });
|
|
1760
|
+
const newSha = sha256(bytes);
|
|
1761
|
+
if (s.format === "html") {
|
|
1762
|
+
// checked in a worker (agentWriteChecked) for exactly these bytes; the disk hash it was checked against is compared below
|
|
1763
|
+
if (htmlChecked?.to !== newSha) throw new HttpError(500, "an HTML file is written only after its write check", { code: "unchecked_write" });
|
|
1764
|
+
} else {
|
|
1765
|
+
try {
|
|
1766
|
+
validatePackage(bytes, s.format);
|
|
1767
|
+
} catch (err) {
|
|
1768
|
+
throw new HttpError(422, `refusing to write: ${err.message}`, { code: "invalid_package" });
|
|
1769
|
+
}
|
|
1770
|
+
// read whole before anything is written, as a Send's bytes are: bytes past a reader's cap are refused with the reason (CapError, 422
|
|
1771
|
+
// too_large), never written to leave the next outline, review and done without a reading. The reading is kept for them (unitsFor).
|
|
1772
|
+
readWhole(s.format, bytes, newSha);
|
|
1022
1773
|
}
|
|
1023
1774
|
let disk;
|
|
1024
1775
|
try {
|
|
@@ -1029,9 +1780,10 @@ export class Sessions extends EventEmitter {
|
|
|
1029
1780
|
}
|
|
1030
1781
|
const diskSha = sha256(disk);
|
|
1031
1782
|
if (diskSha !== baseSha) throw this.#outsideChange(s, { side: "agent", bytes, diskSha, baseSha, message: "the file changed on disk since apply read it" });
|
|
1783
|
+
// R12: the compare-and-swap again after the check's await: the version it checked against must still be the one on disk
|
|
1784
|
+
if (s.format === "html" && htmlChecked.from !== diskSha) throw this.#outsideChange(s, { side: "agent", bytes, diskSha, baseSha: htmlChecked.from, message: "the file changed on disk while the write was being checked" });
|
|
1032
1785
|
this.setStage(key, "writing"); // only once the checks passed: a refused write does not claim to be writing
|
|
1033
|
-
const info = this.#replace(s, bytes, { expectSha: diskSha, label, side: "agent" });
|
|
1034
|
-
const newSha = sha256(bytes);
|
|
1786
|
+
const info = this.#replace(s, bytes, { expectSha: diskSha, label, side: "agent", htmlChecked });
|
|
1035
1787
|
// what the merge may rely on later: the ops `apply` says it ran (checked against the bytes when a merge needs them), or that it was a commit
|
|
1036
1788
|
const checked = route === "apply" && s.format === "xlsx" && Array.isArray(ops) && validateOps({ schema: "onionskin.ops/1", ops }).ok; // only the XLSX merge reads them
|
|
1037
1789
|
this.#logWrite(s, checked ? { from: diskSha, to: newSha, route, ops, partial: partial === true, anchors: s.delivered.batch.untrusted_document_evidence?.anchors || [] } : { from: diskSha, to: newSha, route: route === "apply" ? "apply" : "commit", ops: null });
|
|
@@ -1043,6 +1795,41 @@ export class Sessions extends EventEmitter {
|
|
|
1043
1795
|
return { ok: true, sha256: newSha, lease_expires_at: s.delivered.lease.expiresAt };
|
|
1044
1796
|
}
|
|
1045
1797
|
|
|
1798
|
+
/**
|
|
1799
|
+
* The agent's write (`/cli/agent-write`, `/cli/commit`) for every format. An HTML file's bytes first go through the write check
|
|
1800
|
+
* (src/html-write.js checkedHtmlWrite: decoding, limits, a parse, the charset rule, the inventory of active content) in a worker; the
|
|
1801
|
+
* bytes it returns (the charset declaration may be added) are then written by agentWrite, which checks the lease, the state and the
|
|
1802
|
+
* disk hash again after that await (R12). Other formats: agentWrite as it is.
|
|
1803
|
+
*/
|
|
1804
|
+
async agentWriteChecked(key, args, check = htmlWriteOffThread) {
|
|
1805
|
+
const s = this.get(key);
|
|
1806
|
+
if (s.format !== "html") return this.agentWrite(key, args);
|
|
1807
|
+
// the cheap refusals first, so a write that cannot land costs no worker; agentWrite makes every one of them again after the await
|
|
1808
|
+
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
1809
|
+
if (!s.delivered) throw new HttpError(409, `no batch is delivered to an agent (state: ${this.state(s)}); run poll and wait for the user's Send`, { code: "not_working", state: this.state(s) });
|
|
1810
|
+
this.#assertLease(s, { holder: args.holder, token: args.token, batchId: args.batchId, strict: true });
|
|
1811
|
+
if (!args.bytes?.length) throw new HttpError(400, "no bytes");
|
|
1812
|
+
let disk;
|
|
1813
|
+
try {
|
|
1814
|
+
checkRegularOwned(s.file);
|
|
1815
|
+
disk = readCapped(s.file);
|
|
1816
|
+
} catch (err) {
|
|
1817
|
+
throw new HttpError(409, `cannot read the file: ${err.message}`, { code: "gone" });
|
|
1818
|
+
}
|
|
1819
|
+
// the bytes checked are the ones whose hash agentWrite compares with the base and the disk after this await (from === disk === base)
|
|
1820
|
+
const c = await check({ before: disk, after: args.bytes });
|
|
1821
|
+
if (!c.ok) {
|
|
1822
|
+
// a refused write is part of the run's result: the page lists it with the run (the R1 sentence for active content), nothing was written
|
|
1823
|
+
if (c.code !== "check_failed" && s.delivered && !s.closed) {
|
|
1824
|
+
const p = cleanProblem({ key: `write:${c.code}`, op: args.route === "apply" ? "apply" : "commit", status: "refused", code: c.code, reason: c.reason });
|
|
1825
|
+
s.delivered.applyProblems = [...(s.delivered.applyProblems || []).filter((x) => x.key !== p.key), p].slice(0, 40);
|
|
1826
|
+
this.#changed(s);
|
|
1827
|
+
}
|
|
1828
|
+
throw new HttpError(c.code === "check_failed" ? 503 : 422, c.reason, { code: c.code, ...(c.reasons ? { reasons: c.reasons } : {}) });
|
|
1829
|
+
}
|
|
1830
|
+
return this.agentWrite(key, { ...args, bytes: c.bytes, htmlChecked: { from: sha256(disk), to: sha256(c.bytes) } });
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1046
1833
|
/**
|
|
1047
1834
|
* What `apply` could not do (refused by the engine, protected, anchor or expectation mismatch), told by the lease holder after each run
|
|
1048
1835
|
* (also when nothing was written). Shown to the user with the activity entry at `done`.
|
|
@@ -1082,7 +1869,25 @@ export class Sessions extends EventEmitter {
|
|
|
1082
1869
|
return { pre, post, format: s.format, postSha: sha256(post), batchId: s.delivered.batch.batch_id, deferSoffice: env("SOFFICE_SYNC", "") !== "1" };
|
|
1083
1870
|
}
|
|
1084
1871
|
|
|
1085
|
-
|
|
1872
|
+
/**
|
|
1873
|
+
* done() as the server calls it (src/server.js finishRun): the anchors of the comments queued during the run (moved onto the file the
|
|
1874
|
+
* agent leaves), a review's cards and the outline for a waiting agent are worked out first in the anchor worker, then done() commits
|
|
1875
|
+
* with them and reads none of it on this thread. Work that did not finish leaves those comments weaker, with a flag; never lost.
|
|
1876
|
+
*/
|
|
1877
|
+
async doneAsync(key, results, claim = {}, analysis = null, opts = {}) {
|
|
1878
|
+
return this.#serial(key, async () => {
|
|
1879
|
+
const s = this.get(key);
|
|
1880
|
+
const run = s.delivered;
|
|
1881
|
+
const tasks = [];
|
|
1882
|
+
if (run && s.queued) tasks.push(...s.comments.filter((c) => c.status === "queued").map(reanchorTask));
|
|
1883
|
+
if (run?.batch.instructions?.review && s.review?.batchId === run.batch.batch_id && results && typeof results === "object") tasks.push(cardsTask(results, s.review, run));
|
|
1884
|
+
const prepared = run && (tasks.length || this.#wantsOutline(s)) ? await this.#offThread(s.format, { source: { path: s.file }, tasks, outline: this.#wantsOutline(s) }) : null;
|
|
1885
|
+
return this.done(key, results, claim, analysis, { ...opts, prepared });
|
|
1886
|
+
});
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
/** `inline: false` (the server): never analyze on this thread; an analysis for other bytes (the file changed while it ran) is a stated failure. */
|
|
1890
|
+
done(key, results, claim = {}, analysis = null, { inline = true, prepared = null } = {}) {
|
|
1086
1891
|
const s = this.get(key);
|
|
1087
1892
|
if (!s.delivered) throw new HttpError(409, "no batch is in progress for this file (taken back or already finished)", { code: "no_batch" });
|
|
1088
1893
|
this.#assertLease(s, claim, { staleOnHolder: true });
|
|
@@ -1096,10 +1901,10 @@ export class Sessions extends EventEmitter {
|
|
|
1096
1901
|
throw new HttpError(409, `cannot read the file: ${err.message}`);
|
|
1097
1902
|
}
|
|
1098
1903
|
const postSha = sha256(post);
|
|
1099
|
-
if (readOnlyKind(run.batch) === "voice") return this.#doneVoice(s, results, run, post, postSha);
|
|
1904
|
+
if (readOnlyKind(run.batch) === "voice") return this.#doneVoice(s, results, run, post, postSha, prepared);
|
|
1100
1905
|
// a review that wrote nothing ends with cards. One whose file changed anyway (a direct save past review_only) is finished as a run below:
|
|
1101
1906
|
// its write is flagged and can be reverted, and no card is kept
|
|
1102
|
-
if (run.batch.instructions?.review && postSha === run.preSha) return this.#doneReview(s, results, run, post);
|
|
1907
|
+
if (run.batch.instructions?.review && postSha === run.preSha) return this.#doneReview(s, results, run, post, prepared);
|
|
1103
1908
|
s.snapshots.add(post, "after agent");
|
|
1104
1909
|
|
|
1105
1910
|
// ---- plan: everything that can throw happens here, on locals; the session is not touched until the commit below ------
|
|
@@ -1113,7 +1918,14 @@ export class Sessions extends EventEmitter {
|
|
|
1113
1918
|
let found;
|
|
1114
1919
|
if (!pre) found = { flags: [{ code: "no_pre_snapshot", severity: "warn" }], checks: { node: "skipped", deep: { status: "skipped", reason: "no pre-agent snapshot" }, notice: "Checks skipped: the pre-agent version is no longer stored." }, changes: null };
|
|
1115
1920
|
else if (analysis && analysis.postSha === postSha && analysis.batchId === run.batch.batch_id) found = analysis;
|
|
1116
|
-
else
|
|
1921
|
+
else if (!inline) {
|
|
1922
|
+
const why = "the file changed while it was being checked";
|
|
1923
|
+
found = { flags: [{ code: "checks_failed", severity: "warn", detail: why }], checks: { node: "failed", deep: { status: "skipped", reason: why }, notice: `The package checks did not finish: ${why}. The file was not verified.` }, changes: null };
|
|
1924
|
+
} else if (s.format === "html") {
|
|
1925
|
+
// an HTML file is read in workers only: a caller without the worker's analysis gets it stated as not run, never as clean
|
|
1926
|
+
const why = "the checks of an HTML file run in a worker";
|
|
1927
|
+
found = { flags: [{ code: "checks_failed", severity: "warn", detail: why }], checks: { node: "failed", deep: { status: "skipped", reason: why }, notice: `The checks did not run: ${why}. The file was not verified.` }, changes: null };
|
|
1928
|
+
} else found = analyze(pre, post, s.format);
|
|
1117
1929
|
const flags = [...found.flags];
|
|
1118
1930
|
const { checks, changes } = found;
|
|
1119
1931
|
// the file must be what this server last wrote (apply/commit) or delivered; anything else changed it behind the lease (a direct save by
|
|
@@ -1154,7 +1966,8 @@ export class Sessions extends EventEmitter {
|
|
|
1154
1966
|
// style.json changed by something else while this batch was out (an agent with a shell, say): already put back by src/style.js
|
|
1155
1967
|
this.#guardFlags(run, flags);
|
|
1156
1968
|
if (results.voice_draft !== undefined) flags.push({ code: "voice_draft_not_allowed", severity: "warn" }); // only a voice batch takes a draft; nothing is stored
|
|
1157
|
-
const queuedPlan = this.#planQueued(s, post, postSha, flags);
|
|
1969
|
+
const queuedPlan = this.#planQueued(s, post, postSha, flags, prepared);
|
|
1970
|
+
if (prepared?.ok && prepared.sha === postSha && prepared.outline) this.#outline = { key: s.key, sha: postSha, all: prepared.outline };
|
|
1158
1971
|
const cleanFlags = flags.map(cleanFlag);
|
|
1159
1972
|
const { list: ignored, more: ignoredMore } = ignoredInstructions(results.document_instructions_ignored);
|
|
1160
1973
|
const entry = {
|
|
@@ -1280,27 +2093,21 @@ export class Sessions extends EventEmitter {
|
|
|
1280
2093
|
}
|
|
1281
2094
|
|
|
1282
2095
|
/** Comments queued while the agent worked become the next pending batch, anchored on the file the agent leaves. Pure planning: touches no session state; re-anchoring problems are pushed onto `flags`. */
|
|
1283
|
-
|
|
2096
|
+
// `prepared`: the anchor worker's answer for the same tasks (doneAsync); without it the anchors are moved on this thread.
|
|
2097
|
+
#planQueued(s, post, postSha, flags, prepared = null) {
|
|
1284
2098
|
let plan = null;
|
|
1285
2099
|
if (s.queued) {
|
|
1286
2100
|
const queued = s.comments.filter((c) => c.status === "queued");
|
|
1287
2101
|
if (queued.length || s.queuedNote) {
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
} catch (err) {
|
|
1292
|
-
flags.push({ code: "queued_reanchor_failed", severity: "warn", detail: err.message });
|
|
1293
|
-
}
|
|
2102
|
+
const work = this.#results(s.format, post, postSha, queued.map(reanchorTask), prepared);
|
|
2103
|
+
const read = work.read;
|
|
2104
|
+
if (!read?.ok) flags.push({ code: "queued_reanchor_failed", severity: "warn", detail: read?.invalid || read?.cap || "the anchors could not be worked out in time" });
|
|
1294
2105
|
const durables = new Map();
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
flags.push({ code: "queued_reanchor_failed", severity: "warn", comment_id: c.id, detail: err.message });
|
|
1301
|
-
}
|
|
1302
|
-
durables.set(c.id, d);
|
|
1303
|
-
}
|
|
2106
|
+
queued.forEach((c, i) => {
|
|
2107
|
+
const r = work.at(i);
|
|
2108
|
+
if (read?.ok && !usable(r)) flags.push({ code: "queued_reanchor_failed", severity: "warn", comment_id: c.id, detail: r?.error ?? "the anchor could not be worked out in time" });
|
|
2109
|
+
durables.set(c.id, read?.ok && usable(r) ? r : weakDurable(s.format, c.anchor));
|
|
2110
|
+
});
|
|
1304
2111
|
const batch = buildBatch({ file: s.file, format: s.format, baseSha: postSha, note: s.queuedNote, comments: queued.map((c) => ({ ...c, durable: durables.get(c.id) })), edits: [], style: styleNote(), guide: voiceGuide() });
|
|
1305
2112
|
plan = { queued, durables, batch };
|
|
1306
2113
|
}
|
|
@@ -1318,6 +2125,7 @@ export class Sessions extends EventEmitter {
|
|
|
1318
2125
|
const s = this.get(key);
|
|
1319
2126
|
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
1320
2127
|
if (!REVIEW_KINDS.includes(kind)) throw new HttpError(400, `kind must be one of ${REVIEW_KINDS.join(", ")}`, { code: "bad_kind" });
|
|
2128
|
+
if (s.format === "html") throw new HttpError(409, "Review first is not available for HTML files in this version. Comment on the page and press Send.", { code: "not_for_html" }); // PLAN-html 2a: the page hides the button
|
|
1321
2129
|
if (s.delivered || s.pending) throw new HttpError(409, s.delivered?.local ? "Wait for the change on this Mac to finish, then try again." : "Wait for the agent to finish, then try again.", { code: "busy" });
|
|
1322
2130
|
let disk;
|
|
1323
2131
|
try {
|
|
@@ -1334,7 +2142,7 @@ export class Sessions extends EventEmitter {
|
|
|
1334
2142
|
}
|
|
1335
2143
|
|
|
1336
2144
|
#reviewBatch(s, kind, bytes, baseSha) {
|
|
1337
|
-
const { units, more, index } = unitsOf(s.format, bytes);
|
|
2145
|
+
const { units, more, index } = unitsOf(s.format, bytes, unitsFor(s.format, bytes, baseSha));
|
|
1338
2146
|
const batch = buildBatch({ file: s.file, format: s.format, baseSha, note: "", comments: [], edits: [], review: { kind, ask: REVIEW_ASKS[kind] }, units, unitsMore: more });
|
|
1339
2147
|
s.review = { batchId: batch.batch_id, kind, units: units.map((u) => ({ id: u.id, protected: u.protected })), index };
|
|
1340
2148
|
s.decidedBatch = batch.batch_id; // never the local runner's: only an agent reads the file
|
|
@@ -1342,13 +2150,19 @@ export class Sessions extends EventEmitter {
|
|
|
1342
2150
|
}
|
|
1343
2151
|
|
|
1344
2152
|
/** `done` of a review that wrote nothing: the suggestions become cards (checked against the file), an activity entry of kind "review", no history update. */
|
|
1345
|
-
#doneReview(s, results, run, post) {
|
|
2153
|
+
#doneReview(s, results, run, post, prepared = null) {
|
|
1346
2154
|
const flags = [];
|
|
1347
2155
|
const ix = s.review?.batchId === run.batch.batch_id ? s.review : null;
|
|
1348
|
-
const
|
|
2156
|
+
const gone = (why) => ({ cards: [], dropped: (Array.isArray(results.suggestions) ? results.suggestions : []).map((_, i) => ({ index: i, reason: why })) });
|
|
2157
|
+
let checked = gone("the review's outline is gone");
|
|
2158
|
+
if (ix) {
|
|
2159
|
+
const r = this.#results(s.format, post, run.preSha, [cardsTask(results, ix, run)], prepared).at(0);
|
|
2160
|
+
checked = usable(r) && r ? r : gone("the suggestions could not be checked against the file in time");
|
|
2161
|
+
}
|
|
2162
|
+
const { cards, dropped } = checked;
|
|
1349
2163
|
for (const d of dropped) flags.push({ code: "bad_suggestion", severity: "warn", ...(d.index != null ? { index: d.index } : {}), reason: d.reason });
|
|
1350
2164
|
if (results.voice_draft !== undefined) flags.push({ code: "voice_draft_not_allowed", severity: "warn" });
|
|
1351
|
-
const queuedPlan = this.#planQueued(s, post, run.preSha, flags);
|
|
2165
|
+
const queuedPlan = this.#planQueued(s, post, run.preSha, flags, prepared);
|
|
1352
2166
|
const cleanFlags = flags.map(cleanFlag);
|
|
1353
2167
|
const { list: ignored, more: ignoredMore } = ignoredInstructions(results.document_instructions_ignored); // a review reads the whole file: its report counts most here
|
|
1354
2168
|
const entry = {
|
|
@@ -1428,8 +2242,25 @@ export class Sessions extends EventEmitter {
|
|
|
1428
2242
|
* and Revert, as for any local run. The text written is the stored suggestion, never text from the page. Every refusal happens before
|
|
1429
2243
|
* anything is written. Other comments and the overall note stay for the next Send; typed edits ride with it too.
|
|
1430
2244
|
*/
|
|
1431
|
-
|
|
2245
|
+
/**
|
|
2246
|
+
* useSuggestion() as the server's route calls it (src/suggest.js): where the suggestion goes in the bytes about to be the file, and
|
|
2247
|
+
* the one reply that writes it, are worked out in the anchor worker first. Work that did not finish writes nothing: the comment stays a
|
|
2248
|
+
* draft and says it can go to the agent, as when the sentence is not found.
|
|
2249
|
+
*/
|
|
2250
|
+
async useSuggestionAsync(key, args) {
|
|
1432
2251
|
const s = this.get(key);
|
|
2252
|
+
this.#useChecks(s, args); // refused before it waits its turn
|
|
2253
|
+
return this.#serial(key, async () => {
|
|
2254
|
+
const { id } = this.#useChecks(s, args);
|
|
2255
|
+
const { c } = this.#usePlan(s, args, id); // nothing is stored before the commit, as a Send (sendAsync)
|
|
2256
|
+
const bytes = args.bytes?.length && args.bytes.length <= MAX_BYTES ? args.bytes : null;
|
|
2257
|
+
const prepared = await this.#offThread(s.format, { source: bytes ? { bytes } : { path: s.file }, tasks: [useTask(c.anchor, args.suggestion, id)] });
|
|
2258
|
+
return this.useSuggestion(key, { ...args, prepared });
|
|
2259
|
+
});
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
/** The refusals of a quick fix's Use that need nothing read (useSuggestion runs them first). Returns {id}. */
|
|
2263
|
+
#useChecks(s, { suggestion: rec, commentId, comments }) {
|
|
1433
2264
|
if (s.closed) throw new HttpError(409, "session closed", { code: "closed" });
|
|
1434
2265
|
if (s.delivered || s.pending) throw new HttpError(409, "The agent started working on this file, so nothing was written. Ask the agent instead, or try again when it is done.", { code: "busy" });
|
|
1435
2266
|
const words = rec && Object.hasOwn(STAMP_WORDS, rec.task) && rec.task !== "translate" ? STAMP_WORDS[rec.task] : null;
|
|
@@ -1439,22 +2270,37 @@ export class Sessions extends EventEmitter {
|
|
|
1439
2270
|
const old = s.comments.find((c) => c.id === id);
|
|
1440
2271
|
const sameAnchor = (a) => a && typeof a === "object" && a.quote === rec.quote && JSON.stringify(a.location ?? null) === JSON.stringify(rec.location ?? null);
|
|
1441
2272
|
if (!mine || userText(mine.text) !== words || !sameAnchor(mine.anchor) || (old && old.status !== "draft")) throw new HttpError(409, "This comment is not the suggestion's stamp.", { code: "not_this_comment" });
|
|
2273
|
+
return { id };
|
|
2274
|
+
}
|
|
1442
2275
|
|
|
1443
|
-
|
|
2276
|
+
/** What a Use stores before it commits (#sentPlan with its stamp), the stamp as planned, and every limit checked: nothing changed. */
|
|
2277
|
+
#usePlan(s, { comments, note, edits = [], rev, tab, seen, noteSeen, bytes = null }, id) {
|
|
2278
|
+
if (bytes?.length > MAX_BYTES) throw new HttpError(413, TOO_BIG, { code: "too_large" }); // before anything is stored, as a Send
|
|
2279
|
+
const saved = this.#sentPlan(s, { comments, note, rev, tab, seen, noteSeen }, id);
|
|
2280
|
+
const c = saved.comments.find((x) => x.id === id);
|
|
2281
|
+
if (c?.status !== "draft") throw new HttpError(409, "This comment is not the suggestion's stamp.", { code: "not_this_comment" });
|
|
2282
|
+
if (!this.#fits(s, { comments: saved.comments, edits: [...s.edits, ...this.#editsToTake(s, edits)] })) throw tooLarge("This request");
|
|
2283
|
+
return { saved, c };
|
|
2284
|
+
}
|
|
2285
|
+
|
|
2286
|
+
useSuggestion(key, { suggestion: rec, commentId, comments, note, baseSha, bytes = null, edits = [], prepared = null, rev, tab, seen, noteSeen }) {
|
|
2287
|
+
const s = this.get(key);
|
|
2288
|
+
const { id } = this.#useChecks(s, { suggestion: rec, commentId, comments });
|
|
2289
|
+
// the page's list and note are saved first (only the stamp goes out), after every limit is checked on them
|
|
2290
|
+
this.#commitDraft(s, this.#usePlan(s, { comments, note, edits, rev, tab, seen, noteSeen, bytes }, id).saved, rev, tab, true);
|
|
1444
2291
|
const c = s.comments.find((x) => x.id === id);
|
|
1445
|
-
const anchor = { location: c.anchor.location, quote: c.anchor.quote, prefix: c.anchor.prefix, suffix: c.anchor.suffix };
|
|
1446
2292
|
let plan = null;
|
|
1447
|
-
// the place and the reply are worked out on the very bytes that are about to be the file
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
const
|
|
1451
|
-
const place =
|
|
2293
|
+
// the place and the reply are worked out on the very bytes that are about to be the file (by the anchor worker, or here); any
|
|
2294
|
+
// refusal here writes nothing, and work that did not finish is "not here"
|
|
2295
|
+
const check = (b, sha) => {
|
|
2296
|
+
const r = this.#results(s.format, b, sha, [useTask(c.anchor, rec, id)], prepared).at(0);
|
|
2297
|
+
const { durable, place, reply } = usable(r) ? r : { durable: null, place: { ok: false, code: "not_here" }, reply: null };
|
|
1452
2298
|
if (!place.ok) throw new HttpError(409, place.code === "protected" ? "This part of the file is protected, so it cannot be changed here." : LOCAL_HINT, { code: place.code });
|
|
1453
|
-
const
|
|
1454
|
-
const batch = buildBatch({ file: s.file, format: s.format, baseSha: sha256(b), note: "", comments: [{ ...c, durable }], edits: [] });
|
|
2299
|
+
const batch = buildBatch({ file: s.file, format: s.format, baseSha: sha, note: "", comments: [{ ...c, durable }], edits: [] });
|
|
1455
2300
|
if (!reply || batch.instructions.comments[0]?.reply_with !== "find_replace_or_text") throw new HttpError(409, LOCAL_HINT, { code: "not_here" });
|
|
1456
2301
|
plan = { durable, reply, batch };
|
|
1457
2302
|
};
|
|
2303
|
+
if (!this.#fits(s, { comments: s.comments, edits: [...s.edits, ...this.#editsToTake(s, edits)] })) throw tooLarge("This request"); // before anything is written
|
|
1458
2304
|
let taken;
|
|
1459
2305
|
try {
|
|
1460
2306
|
taken = this.#takePage(s, { baseSha, bytes }, check);
|
|
@@ -1463,15 +2309,16 @@ export class Sessions extends EventEmitter {
|
|
|
1463
2309
|
if (["not_here", "not_eligible", "protected"].includes(err?.extra?.code)) {
|
|
1464
2310
|
c.localHint = LOCAL_HINT;
|
|
1465
2311
|
this.#changed(s);
|
|
1466
|
-
}
|
|
2312
|
+
} else this.emit("change", s, "draft"); // the drafts it stored
|
|
1467
2313
|
throw err;
|
|
1468
2314
|
}
|
|
1469
|
-
|
|
2315
|
+
s.edits.push(...this.#editsToTake(s, edits));
|
|
1470
2316
|
c.durable = plan.durable;
|
|
1471
2317
|
c.status = "sent";
|
|
1472
2318
|
c.batchId = plan.batch.batch_id;
|
|
1473
2319
|
delete c.localHint;
|
|
1474
2320
|
s.lastWrite = taken.writeInfo;
|
|
2321
|
+
this.#sentRev(s, { rev, tab });
|
|
1475
2322
|
try {
|
|
1476
2323
|
this.startLocal(key, plan.batch, { reply: plan.reply, summary: USE_SUMMARY, model: true }); // saves, tells the page, starts the runner
|
|
1477
2324
|
} catch (err) {
|
|
@@ -1530,13 +2377,13 @@ export class Sessions extends EventEmitter {
|
|
|
1530
2377
|
* "voice" with a fixed summary, no history update and no analysis. A file that changed anyway (a direct save past voice_only) is
|
|
1531
2378
|
* recorded as a withdrawn run would be: flagged, kept Revert-able.
|
|
1532
2379
|
*/
|
|
1533
|
-
#doneVoice(s, results, run, post, postSha) {
|
|
2380
|
+
#doneVoice(s, results, run, post, postSha, prepared = null) {
|
|
1534
2381
|
const flags = [];
|
|
1535
2382
|
for (const r of (Array.isArray(results.results) ? results.results : []).slice(0, 1000)) flags.push({ code: "unknown_result", severity: "warn", comment_id: String(r?.comment_id ?? "").slice(0, 64) });
|
|
1536
2383
|
const check = cleanVoiceDraft(results.voice_draft); // size first, then hidden characters, links and markup, the layout, the caps
|
|
1537
2384
|
if (check.code) flags.push({ code: "bad_voice_draft", severity: "warn", reason: check.code });
|
|
1538
2385
|
this.#guardFlags(run, flags);
|
|
1539
|
-
const queuedPlan = this.#planQueued(s, post, postSha, flags);
|
|
2386
|
+
const queuedPlan = this.#planQueued(s, post, postSha, flags, prepared);
|
|
1540
2387
|
const { list: ignored, more: ignoredMore } = ignoredInstructions(results.document_instructions_ignored);
|
|
1541
2388
|
// ---- commit: the draft file first (a failed write keeps nothing and says so), then the session
|
|
1542
2389
|
let kept = false;
|
|
@@ -1589,7 +2436,7 @@ export class Sessions extends EventEmitter {
|
|
|
1589
2436
|
if (!batch) throw new HttpError(409, "nothing to take back");
|
|
1590
2437
|
// the note travels back with the comments: the withdrawn batch's note, then a queued one, then whatever was typed since
|
|
1591
2438
|
const notes = [batch.instructions?.overall_note, s.queuedNote, s.note].filter(Boolean);
|
|
1592
|
-
s
|
|
2439
|
+
this.#giveNote(s, [...new Set(notes)].join("\n\n"));
|
|
1593
2440
|
s.queuedNote = "";
|
|
1594
2441
|
let event = "state";
|
|
1595
2442
|
if (s.delivered) {
|
|
@@ -1640,7 +2487,27 @@ export class Sessions extends EventEmitter {
|
|
|
1640
2487
|
return true;
|
|
1641
2488
|
}
|
|
1642
2489
|
|
|
1643
|
-
revert(
|
|
2490
|
+
/** revert() as the server's route calls it: a waiting batch's anchors are moved onto the restored version in the anchor worker first. */
|
|
2491
|
+
async revertAsync(key, activityId) {
|
|
2492
|
+
return this.#serial(key, async () => {
|
|
2493
|
+
const s = this.get(key);
|
|
2494
|
+
const entry = s.activity.find((a) => a.id === activityId && a.kind === "agent");
|
|
2495
|
+
let prepared = null;
|
|
2496
|
+
if (entry && !entry.reverted && !s.delivered && s.pending && !readOnlyKind(s.pending) && !s.pending.instructions?.review && s.snapshots.has(entry.preSha)) {
|
|
2497
|
+
const tasks = this.#revertComments(s, entry).map(reanchorTask);
|
|
2498
|
+
if (tasks.length) prepared = await this.#offThread(s.format, { source: { path: s.snapshots.blob(entry.preSha), sha: entry.preSha }, tasks });
|
|
2499
|
+
}
|
|
2500
|
+
return this.revert(key, activityId, { prepared });
|
|
2501
|
+
});
|
|
2502
|
+
}
|
|
2503
|
+
|
|
2504
|
+
/** The comments of the waiting batch, as revert() moves them: the run's comments become drafts first, so only the others are left. */
|
|
2505
|
+
#revertComments(s, entry) {
|
|
2506
|
+
const ids = new Set(s.pending.instructions.comments.map((c) => c.id));
|
|
2507
|
+
return s.comments.filter((c) => ids.has(c.id) && !(c.batchId === entry.batchId && ["done", "declined", "asked"].includes(c.status)));
|
|
2508
|
+
}
|
|
2509
|
+
|
|
2510
|
+
revert(key, activityId, { prepared = null } = {}) {
|
|
1644
2511
|
const s = this.get(key);
|
|
1645
2512
|
if (s.delivered) throw new HttpError(423, s.delivered.local ? "a change is being made on this Mac" : "the agent is working", { code: "locked" });
|
|
1646
2513
|
const entry = s.activity.find((a) => a.id === activityId && a.kind === "agent");
|
|
@@ -1652,7 +2519,15 @@ export class Sessions extends EventEmitter {
|
|
|
1652
2519
|
if (sha256(disk) !== entry.postSha) throw new HttpError(409, mac ? "the file changed after the change was made; revert refused" : "the file changed after the agent finished; revert refused", { code: "changed_since" });
|
|
1653
2520
|
if (!s.snapshots.has(entry.preSha)) throw new HttpError(410, mac ? "the version before this change is no longer stored" : "the pre-agent version is no longer stored");
|
|
1654
2521
|
const pre = s.snapshots.read(entry.preSha);
|
|
1655
|
-
|
|
2522
|
+
// an HTML file: a revert restores the version before the run, by identity (the stored bytes must hash to the run's preSha), never
|
|
2523
|
+
// through the write check: that version was the file before (opened by the person or written after its own check), and checking it
|
|
2524
|
+
// against the run's result would refuse undoing a run that removed active content (the revert would "add" it back)
|
|
2525
|
+
let htmlChecked = null;
|
|
2526
|
+
if (s.format === "html") {
|
|
2527
|
+
if (sha256(pre) !== entry.preSha) throw new HttpError(410, mac ? "the version before this change is no longer stored" : "the pre-agent version is no longer stored");
|
|
2528
|
+
htmlChecked = { from: entry.postSha, to: entry.preSha };
|
|
2529
|
+
}
|
|
2530
|
+
this.#replace(s, pre, { expectSha: entry.postSha, label: "before revert", htmlChecked }); // a revert is a new version
|
|
1656
2531
|
s.snapshots.add(pre, "reverted");
|
|
1657
2532
|
s.baseSha = entry.preSha;
|
|
1658
2533
|
addUpdate(s, { kind: "revert", before: entry.postSha, after: entry.preSha, local: mac }, this.now());
|
|
@@ -1670,7 +2545,7 @@ export class Sessions extends EventEmitter {
|
|
|
1670
2545
|
}
|
|
1671
2546
|
for (const g of s.suggestions || []) if (g.status === "accepted" && g.acceptedBatch === entry.batchId) g.status = "open"; // the text is back: it can be accepted again
|
|
1672
2547
|
if (entry.by === "local" && entry.localNote && !s.note) {
|
|
1673
|
-
s
|
|
2548
|
+
this.#giveNote(s, entry.localNote); // the request comes back to the box, and the same words go to the agent next time
|
|
1674
2549
|
s.skipLocalNote = entry.localNote;
|
|
1675
2550
|
}
|
|
1676
2551
|
// a batch built against the reverted version is rebuilt against the restored bytes (a review: its outline is read again; a voice
|
|
@@ -1680,19 +2555,9 @@ export class Sessions extends EventEmitter {
|
|
|
1680
2555
|
else if (s.pending) {
|
|
1681
2556
|
const ids = new Set(s.pending.instructions.comments.map((c) => c.id));
|
|
1682
2557
|
const cs = s.comments.filter((c) => ids.has(c.id));
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
} catch {
|
|
1687
|
-
units = null;
|
|
1688
|
-
}
|
|
1689
|
-
for (const c of cs) {
|
|
1690
|
-
try {
|
|
1691
|
-
if (units) c.durable = reanchor(units, c);
|
|
1692
|
-
} catch {
|
|
1693
|
-
c.durable = null;
|
|
1694
|
-
}
|
|
1695
|
-
}
|
|
2558
|
+
// moved onto the restored version by the anchor worker (revertAsync) or here; one whose work did not finish goes out weaker
|
|
2559
|
+
const work = this.#results(s.format, pre, entry.preSha, cs.map(reanchorTask), prepared);
|
|
2560
|
+
if (work.read?.ok) cs.forEach((c, i) => (c.durable = usable(work.at(i)) ? work.at(i) : work.at(i)?.error ? null : weakDurable(s.format, c.anchor)));
|
|
1696
2561
|
s.pending = buildBatch({ file: s.file, format: s.format, baseSha: entry.preSha, note: s.pending.instructions.overall_note, comments: cs, edits: [], style: styleNote(), guide: voiceGuide() });
|
|
1697
2562
|
for (const c of cs) c.batchId = s.pending.batch_id;
|
|
1698
2563
|
}
|
|
@@ -1704,6 +2569,7 @@ export class Sessions extends EventEmitter {
|
|
|
1704
2569
|
close(key) {
|
|
1705
2570
|
const s = this.get(key);
|
|
1706
2571
|
s.closed = true;
|
|
2572
|
+
this.#views.delete(key); // the render of an HTML file (htmlView)
|
|
1707
2573
|
this.editors.delete(key); // the closed tab may stay on screen with its event stream open: it must not hold the file against the tab that reopens it
|
|
1708
2574
|
for (const p of s.pollers.splice(0)) p.resolve({ status: "closed" });
|
|
1709
2575
|
fs.rmSync(path.join(stateDir(), "agents", s.key, "outline"), { recursive: true, force: true }); // the outline files of Onionskin's own agent
|
|
@@ -1733,6 +2599,37 @@ export class Sessions extends EventEmitter {
|
|
|
1733
2599
|
return out;
|
|
1734
2600
|
}
|
|
1735
2601
|
|
|
2602
|
+
// ---- the view of an HTML file (GET /api/html-view) ------------------------------------------------------------------------------
|
|
2603
|
+
// key -> {sha, full, plain}: the render of one version (promises, so two requests for it share one worker run); one version per file,
|
|
2604
|
+
// dropped when the file changes or the session closes. Ceiling: one entry per open HTML file, each up to ~43 MB (the render copy at the
|
|
2605
|
+
// limits): a person with many large pages open holds them all; an LRU over files would be the upgrade.
|
|
2606
|
+
#views = new Map();
|
|
2607
|
+
/**
|
|
2608
|
+
* The page's view of an HTML file (PLAN-html 3.2, R12): the disk bytes are taken as /api/file takes them (loadForPage: an outside change
|
|
2609
|
+
* is recorded, the watcher told), then the render copy of exactly those bytes is built in the render worker (src/offload.js
|
|
2610
|
+
* renderOffThread), or the plain copy with `plain`. Resolves to {sha, plain, reason, stats, texts, editable, ident, bytes}; throws HttpError
|
|
2611
|
+
* 422 with the encoding's sentence and code (and `copyable`) or a limit's, 503 when the worker was busy or did not finish.
|
|
2612
|
+
*/
|
|
2613
|
+
async htmlView(key, { plain = false } = {}) {
|
|
2614
|
+
const s = this.get(key);
|
|
2615
|
+
if (s.format !== "html") throw new HttpError(400, "this file is not an HTML file", { code: "not_html" });
|
|
2616
|
+
const { bytes, diskSha } = this.loadForPage(key);
|
|
2617
|
+
let c = this.#views.get(key);
|
|
2618
|
+
if (!c || c.sha !== diskSha) this.#views.set(key, (c = { sha: diskSha, full: null, plain: null }));
|
|
2619
|
+
const k = plain ? "plain" : "full";
|
|
2620
|
+
if (!c[k]) {
|
|
2621
|
+
const p = renderOffThread({ bytes, fileDir: path.dirname(s.file), plain }).then((r) => {
|
|
2622
|
+
if (r.ok) return { sha: diskSha, plain: r.plain, reason: r.reason, stats: r.stats, texts: r.texts, editable: r.editable, ident: r.ident, bytes: r.bytes };
|
|
2623
|
+
if (c[k] === p) c[k] = null; // a failure is not kept: the next request tries again
|
|
2624
|
+
if (r.failed) throw new HttpError(503, `Onionskin could not prepare this page: ${r.why}. Try again in a moment.`, { code: r.failed === "busy" ? "busy" : "render_failed" });
|
|
2625
|
+
if (r.code === "not_utf8" || ["utf16", "invalid_utf8", "roundtrip"].includes(r.code)) throw encodingRefusal(s.file, { error: r.error, code: r.code, copyable: r.copyable });
|
|
2626
|
+
throw new HttpError(422, `Onionskin cannot read this file: ${r.error}.`, { code: "too_large" });
|
|
2627
|
+
});
|
|
2628
|
+
c[k] = p;
|
|
2629
|
+
}
|
|
2630
|
+
return c[k];
|
|
2631
|
+
}
|
|
2632
|
+
|
|
1736
2633
|
/** The history list and one update (browser routes, read-only; see src/history.js). */
|
|
1737
2634
|
history(key) {
|
|
1738
2635
|
return historyList(this.get(key));
|
|
@@ -1847,25 +2744,58 @@ export class Sessions extends EventEmitter {
|
|
|
1847
2744
|
*/
|
|
1848
2745
|
resolveAll(key) {
|
|
1849
2746
|
const s = this.get(key);
|
|
2747
|
+
// an HTML file is read in workers only: on this thread every anchor is "moved or removed" (the server's route resolves in the worker)
|
|
2748
|
+
if (s.format === "html") return { baseSha: sha256(readCapped(s.file)), format: s.format, anchors: Object.fromEntries(this.#resolveTask(s).comments.map((c) => [c.id, { method: "none" }])), view: null };
|
|
1850
2749
|
const bytes = readCapped(s.file);
|
|
1851
|
-
const
|
|
1852
|
-
const
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
2750
|
+
const baseSha = sha256(bytes);
|
|
2751
|
+
const r = anchorWork({ format: s.format, bytes, sha: baseSha, tasks: [this.#resolveTask(s)], budget: SCAN_BUDGET });
|
|
2752
|
+
if (!r.read.ok) throw r.read.cap ? new CapError(r.read.cap) : new Error(r.read.invalid);
|
|
2753
|
+
if (r.results[0]?.error) throw new Error(r.results[0].error);
|
|
2754
|
+
return { baseSha, format: s.format, ...r.results[0] };
|
|
2755
|
+
}
|
|
2756
|
+
|
|
2757
|
+
/**
|
|
2758
|
+
* resolveAll() as the server's route calls it: resolved in the anchor worker against the file on disk. Work that did not finish
|
|
2759
|
+
* answers as a spent budget does: every anchor "moved or removed" (the page keeps the comments in its list), the view unknown.
|
|
2760
|
+
*/
|
|
2761
|
+
async resolveAllAsync(key) {
|
|
2762
|
+
const s = this.get(key);
|
|
2763
|
+
const task = this.#resolveTask(s);
|
|
2764
|
+
if (s.format === "html") await this.#htmlDrafts(s, task);
|
|
2765
|
+
const r = await anchorsOffThread({ format: s.format, source: { path: s.file }, tasks: [task] });
|
|
2766
|
+
if (r.ok && r.r.read.cap) throw new CapError(r.r.read.cap);
|
|
2767
|
+
if (r.ok && r.r.read.ok && usable(r.r.results[0])) return { baseSha: r.sha, format: s.format, ...r.r.results[0] };
|
|
2768
|
+
return { baseSha: r.ok ? r.sha : sha256(readCapped(s.file)), format: s.format, anchors: Object.fromEntries(task.comments.map((c) => [c.id, { method: "none" }])), view: null };
|
|
2769
|
+
}
|
|
2770
|
+
|
|
2771
|
+
/**
|
|
2772
|
+
* HTML: a draft (no durable yet) whose page anchor was made on another version than the file on disk (an outside change, a revert, a
|
|
2773
|
+
* save) would be drawn at a node id of that other version. It is made durable on its own version first (a stored snapshot) and resolved
|
|
2774
|
+
* with the others; one whose version is no longer stored resolves by its quote. Drafts made on the file's version need nothing: the page
|
|
2775
|
+
* draws them at their own place.
|
|
2776
|
+
*/
|
|
2777
|
+
async #htmlDrafts(s, task) {
|
|
2778
|
+
let cur;
|
|
2779
|
+
try {
|
|
2780
|
+
cur = sha256(readCapped(s.file));
|
|
2781
|
+
} catch {
|
|
2782
|
+
return;
|
|
2783
|
+
}
|
|
2784
|
+
const other = s.comments.filter((c) => !c.durable && c.anchor?.location?.version && c.anchor.location.version !== cur);
|
|
2785
|
+
const byVersion = new Map();
|
|
2786
|
+
for (const c of other) (byVersion.get(c.anchor.location.version) || byVersion.set(c.anchor.location.version, []).get(c.anchor.location.version)).push(c);
|
|
2787
|
+
for (const [v, list] of byVersion) {
|
|
2788
|
+
const r = s.snapshots.has(v) ? await this.#offThread("html", { source: { path: s.snapshots.blob(v), sha: v }, tasks: list.map(buildTask) }) : null;
|
|
2789
|
+
for (const c of list) {
|
|
2790
|
+
const d = r?.ok && r.read?.ok ? r.map.get(taskKey(buildTask(c))) : undefined;
|
|
2791
|
+
const durable = usable(d) && d ? d : weakDurable("html", c.anchor);
|
|
2792
|
+
if (durable) task.comments.push({ id: c.id, durable });
|
|
1859
2793
|
}
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
if (s.view?.durable) out.view = { ...resolveDurable(units.paragraphs, s.view.durable), offsetPx: s.view.offsetPx, zoom: s.view.zoom, fraction: s.view.fraction };
|
|
1866
|
-
} else out.view = resolveView(units, s.view);
|
|
1867
|
-
out.sheets = s.format === "xlsx" ? units.wb.sheets.map((x) => ({ index: x.index, name: x.name, sheetId: x.sheetId })) : undefined;
|
|
1868
|
-
return out;
|
|
2794
|
+
}
|
|
2795
|
+
}
|
|
2796
|
+
|
|
2797
|
+
#resolveTask(s) {
|
|
2798
|
+
return { op: "resolveAll", comments: s.comments.filter((c) => c.durable).map((c) => ({ id: c.id, durable: c.durable })), view: s.view };
|
|
1869
2799
|
}
|
|
1870
2800
|
}
|
|
1871
2801
|
|
|
@@ -1906,7 +2836,7 @@ function exportInWorker(job) {
|
|
|
1906
2836
|
};
|
|
1907
2837
|
const timer = setTimeout(() => finish({ timeout: true }), EXPORT_TIMEOUT_MS());
|
|
1908
2838
|
try {
|
|
1909
|
-
worker =
|
|
2839
|
+
worker = startWorker("redline-worker.js", { workerData: job });
|
|
1910
2840
|
} catch (err) {
|
|
1911
2841
|
return finish({ check_failed: String(err?.message || err) });
|
|
1912
2842
|
}
|