onionskin-cli 0.0.3 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/CHANGELOG.md +100 -0
  2. package/INSTALL.md +5 -1
  3. package/NOTICE +34 -0
  4. package/README.md +101 -33
  5. package/docs/contracts/batch.schema.json +27 -4
  6. package/docs/contracts/ops.schema.json +117 -5
  7. package/package.json +4 -2
  8. package/skills/onionskin/SKILL.md +9 -6
  9. package/skills/onionskin/reference/batch.md +5 -3
  10. package/skills/onionskin/reference/formats.md +5 -1
  11. package/skills/onionskin/reference/ops.md +47 -3
  12. package/skills/onionskin/reference/troubleshooting.md +31 -0
  13. package/src/agent-detect.js +28 -2
  14. package/src/agent-prompt.js +12 -1
  15. package/src/agent-runner.js +16 -5
  16. package/src/anchor-work.js +154 -0
  17. package/src/anchor-worker.js +38 -0
  18. package/src/anchors.js +61 -11
  19. package/src/apply.js +119 -47
  20. package/src/batch.js +32 -8
  21. package/src/check-worker.js +8 -5
  22. package/src/cli.js +91 -25
  23. package/src/constants.js +34 -1
  24. package/src/diff.js +3 -1
  25. package/src/docx.js +368 -58
  26. package/src/edit-hash.js +19 -0
  27. package/src/fsutil.js +1 -1
  28. package/src/history-worker.js +8 -8
  29. package/src/history.js +135 -31
  30. package/src/html-active.js +384 -0
  31. package/src/html-envelope.js +51 -0
  32. package/src/html-parser.js +8 -0
  33. package/src/html-render-worker.js +25 -0
  34. package/src/html-render.js +880 -0
  35. package/src/html-utf8.js +65 -0
  36. package/src/html-write-worker.js +14 -0
  37. package/src/html-write.js +380 -0
  38. package/src/html.js +1257 -0
  39. package/src/local.js +68 -4
  40. package/src/merge-package.js +1110 -0
  41. package/src/merge-pptx.js +98 -38
  42. package/src/merge-proof.js +1019 -0
  43. package/src/merge-worker.js +60 -0
  44. package/src/merge-xlsx.js +84 -29
  45. package/src/merge.js +286 -83
  46. package/src/nfc.js +46 -0
  47. package/src/notes.js +122 -0
  48. package/src/offload.js +400 -1
  49. package/src/open-worker.js +27 -0
  50. package/src/ops/common.js +2 -1
  51. package/src/ops/docx.js +58 -29
  52. package/src/ops/html.js +394 -0
  53. package/src/ops/pptx.js +74 -21
  54. package/src/ops/run.js +23 -3
  55. package/src/ops/runs.js +215 -13
  56. package/src/ops/schema.js +8 -1
  57. package/src/ops/xlsx.js +12 -7
  58. package/src/ops-worker.js +17 -0
  59. package/src/package.js +26 -2
  60. package/src/person.js +2 -1
  61. package/src/pptx.js +85 -21
  62. package/src/quick-plan.js +13 -6
  63. package/src/redline-worker.js +10 -14
  64. package/src/reply.js +46 -19
  65. package/src/retain.js +223 -0
  66. package/src/review.js +32 -18
  67. package/src/rewrite-check.js +2 -2
  68. package/src/sanitize.js +16 -3
  69. package/src/server.js +80 -36
  70. package/src/session.js +1138 -208
  71. package/src/stop-gate.js +34 -0
  72. package/src/suggest-reply.js +42 -7
  73. package/src/suggest.js +14 -29
  74. package/src/text.js +161 -11
  75. package/src/update.js +288 -0
  76. package/src/voice-guide.js +4 -3
  77. package/src/voice-learn.js +4 -3
  78. package/src/voice-sample-worker.js +7 -6
  79. package/src/voice-worker.js +4 -6
  80. package/src/worker-host.js +23 -0
  81. package/src/xlsx.js +355 -94
  82. package/src/xml-tree.js +6 -3
  83. package/src/xml.js +82 -34
  84. package/src/zip-write.js +2 -1
  85. package/src/zip.js +9 -5
  86. package/vendor/parse5/LICENSE.txt +32 -0
  87. package/vendor/parse5/parse5.mjs +7822 -0
  88. package/vendor/parse5.lock.json +23 -0
  89. package/web/dist/assets/index-DkNx0LSH.css +1 -0
  90. package/web/dist/assets/{index-DS-WST1E.js → index-DytKkMdi.js} +112 -108
  91. package/web/dist/index.html +2 -2
  92. package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/notes.js ADDED
@@ -0,0 +1,122 @@
1
+ // The overall note, joined and merged the same way on the server and in the page (web/src/drafts.js, web/src/stage.js). Pure, no imports.
2
+
3
+ /** `back`, then `mine` after a blank line, unless every paragraph of `mine` is in `back` already as whole paragraphs (never part of one). */
4
+ export function joinNote(back, mine) {
5
+ back = back || "";
6
+ const paras = (x) => x.split(/\n\s*\n/).map((p) => p.trim()).filter(Boolean);
7
+ const m = paras(mine || "");
8
+ if (!m.length) return back;
9
+ if (`\n\n${paras(back).join("\n\n")}\n\n`.includes(`\n\n${m.join("\n\n")}\n\n`)) return back;
10
+ return back ? `${back}\n\n${mine}` : mine;
11
+ }
12
+
13
+ // Tokens of a note: runs of letters and digits, runs of white space, and each other character on its own.
14
+ const tokens = (s) => s.match(/[\p{L}\p{N}\p{M}_]+|\s+|[^\p{L}\p{N}\p{M}_\s]/gu) || [];
15
+ // Ceiling of the word diff: a table of this many cells (about 16 MB) at most; a longer changed middle is one change (then the merge falls back).
16
+ const DIFF_CELLS = 4_000_000;
17
+
18
+ /** The changes that make token list `b` from `a`: [{start, end, ins}] in order, `a[start, end)` replaced by the tokens `ins`. */
19
+ function changes(a, b) {
20
+ let p = 0;
21
+ while (p < a.length && p < b.length && a[p] === b[p]) p++;
22
+ let s = 0;
23
+ while (s < a.length - p && s < b.length - p && a[a.length - 1 - s] === b[b.length - 1 - s]) s++;
24
+ const n = a.length - p - s;
25
+ const m = b.length - p - s;
26
+ if (n === 0 && m === 0) return [];
27
+ if (n === 0 || m === 0 || (n + 1) * (m + 1) > DIFF_CELLS) return [{ start: p, end: p + n, ins: b.slice(p, p + m) }];
28
+ // longest common subsequence of the middles, by a table filled from the end
29
+ const w = m + 1;
30
+ const L = new Int32Array((n + 1) * w);
31
+ for (let i = n - 1; i >= 0; i--)
32
+ for (let j = m - 1; j >= 0; j--) L[i * w + j] = a[p + i] === b[p + j] ? L[(i + 1) * w + j + 1] + 1 : Math.max(L[(i + 1) * w + j], L[i * w + j + 1]);
33
+ const out = [];
34
+ let cur = null;
35
+ const flush = () => {
36
+ if (cur) out.push(cur);
37
+ cur = null;
38
+ };
39
+ let i = 0;
40
+ let j = 0;
41
+ while (i < n || j < m) {
42
+ if (i < n && j < m && a[p + i] === b[p + j]) {
43
+ flush();
44
+ i++;
45
+ j++;
46
+ } else if (j < m && (i === n || L[i * w + j + 1] >= L[(i + 1) * w + j])) {
47
+ cur ??= { start: p + i, end: p + i, ins: [] };
48
+ cur.ins.push(b[p + j++]);
49
+ } else {
50
+ cur ??= { start: p + i, end: p + i, ins: [] };
51
+ cur.end = p + ++i;
52
+ }
53
+ }
54
+ flush();
55
+ return out;
56
+ }
57
+
58
+ const startsWithTokens = (x, y) => y.length <= x.length && y.every((t, k) => x[k] === t);
59
+ const blank = (t) => !t || /^\s/.test(t);
60
+
61
+ /**
62
+ * The note when one side holds `theirs` and the other `mine`, both from `base`: a three-way merge by words. What `mine` deleted from `base`
63
+ * stays deleted, what `mine` added stays, what `theirs` added (to `base`) is kept, each once; changes of the two sides that do not touch
64
+ * the same words both apply. Where both changed the same words, the merge cannot decide: `mine` is kept whole and the words `theirs` added
65
+ * follow it after a blank line (the caller then holds a note that is not the server's, so it stays unsaved). Both adding at the same place:
66
+ * the same words once; one adding more after the same words (merged before: a second save from the same state) the longer; at the start
67
+ * of `base` the server's first (it only puts returned notes in front: Take back, Stop); anywhere else, undecided as above. Unchanged on
68
+ * either side takes the other; the server emptying the box (a Send took the note) keeps `mine`.
69
+ */
70
+ export function mergeNote(base, theirs, mine) {
71
+ base = base || "";
72
+ theirs = theirs || "";
73
+ mine = mine || "";
74
+ if (mine === base || mine === theirs) return theirs;
75
+ if (theirs === base || !theirs) return mine;
76
+ const B = tokens(base);
77
+ const T = tokens(theirs);
78
+ const M = tokens(mine);
79
+ const cm = changes(B, M);
80
+ const ct = changes(B, T);
81
+ const fallback = () => joinNote(mine, ct.map((c) => c.ins.join("").trim()).filter(Boolean).join("\n\n"));
82
+ const out = [];
83
+ let at = 0; // the next token of `base` to copy
84
+ let i = 0;
85
+ let j = 0;
86
+ const take = (c) => {
87
+ for (; at < c.start; at++) out.push(B[at]);
88
+ out.push(...c.ins);
89
+ at = c.end;
90
+ };
91
+ // `x` is all before `y`: it ends before `y` starts, or where `y` starts unless both only insert there
92
+ const before = (x, y) => x.end < y.start || (x.end === y.start && !(x.start === x.end && y.start === y.end));
93
+ while (i < cm.length || j < ct.length) {
94
+ const a = cm[i];
95
+ const b = ct[j];
96
+ if (!b || (a && before(a, b))) {
97
+ take(a);
98
+ i++;
99
+ continue;
100
+ }
101
+ if (!a || before(b, a)) {
102
+ take(b);
103
+ j++;
104
+ continue;
105
+ }
106
+ // both change at the same place
107
+ const same = a.start === b.start && a.end === b.end;
108
+ if (same && a.ins.join("") === b.ins.join("")) take(a);
109
+ else if (same && a.start === a.end && startsWithTokens(b.ins, a.ins)) take(b);
110
+ else if (same && a.start === a.end && startsWithTokens(a.ins, b.ins)) take(a);
111
+ else if (same && a.start === 0 && a.end === 0) take({ start: 0, end: 0, ins: [...b.ins, ...(blank(b.ins.at(-1)) || blank(a.ins[0]) ? [] : ["\n\n"]), ...a.ins] });
112
+ else return fallback();
113
+ i++;
114
+ j++;
115
+ }
116
+ for (; at < B.length; at++) out.push(B[at]);
117
+ let r = out.join("");
118
+ // white space the merge left at an end the person's text does not have there
119
+ if (!/^\s/.test(mine)) r = r.replace(/^\s+/, "");
120
+ if (!/\s$/.test(mine)) r = r.replace(/\s+$/, "");
121
+ return r;
122
+ }
package/src/offload.js CHANGED
@@ -1,5 +1,37 @@
1
1
  import { Worker } from "node:worker_threads";
2
2
  import { env } from "./constants.js";
3
+ import { STOPPING } from "./stop-gate.js";
4
+
5
+ /**
6
+ * A worker thread running src/<name>'s work(workerData) through src/worker-host.js. Its terminate() waits until the worker has loaded its
7
+ * modules (they are evaluated asynchronously, and stopping a worker then aborts the whole process on Node 26), then marks the worker as
8
+ * being stopped and waits for the zlib call it may be inside to end (stopping it inside zlib aborts the process too: src/stop-gate.js),
9
+ * then stops it; a worker that exits first is not waited for. The wait is the module loading and one zlib call at most, never the work.
10
+ */
11
+ export function startWorker(name, { workerData, ...opts } = {}) {
12
+ const loaded = new SharedArrayBuffer(8);
13
+ const flag = new Int32Array(loaded);
14
+ const worker = new Worker(new URL("./worker-host.js", import.meta.url), { ...opts, workerData: { data: workerData, loaded }, argv: [name] });
15
+ let exited = false;
16
+ worker.once("exit", () => (exited = true));
17
+ const stop = worker.terminate.bind(worker);
18
+ let stopping = null;
19
+ worker.terminate = () =>
20
+ (stopping ??= new Promise((resolve) => {
21
+ let marked = false;
22
+ const tryStop = () => {
23
+ if (exited) return resolve();
24
+ if (Atomics.load(flag, 0) === 1) {
25
+ marked = true;
26
+ // from here on the worker enters no zlib call; it may still be inside one
27
+ if ((Atomics.or(flag, 1, STOPPING) & ~STOPPING) === 0) return stop().then(resolve, resolve);
28
+ }
29
+ setTimeout(tryStop, marked ? 2 : 10);
30
+ };
31
+ tryStop();
32
+ }));
33
+ return worker;
34
+ }
3
35
 
4
36
  const WORKER_TIMEOUT_MS = () => Number(env("CHECK_TIMEOUT_MS", 180_000)) + 60_000;
5
37
 
@@ -18,7 +50,7 @@ export function analyzeOffThread(inputs) {
18
50
  });
19
51
  return new Promise((resolve) => {
20
52
  let settled = false;
21
- const worker = new Worker(new URL("./check-worker.js", import.meta.url), { workerData: { pre, post, format, deferSoffice } });
53
+ const worker = startWorker("check-worker.js", { workerData: { pre, post, format, deferSoffice } });
22
54
  const finish = (r) => {
23
55
  if (settled) return;
24
56
  settled = true;
@@ -32,3 +64,370 @@ export function analyzeOffThread(inputs) {
32
64
  worker.once("exit", (code) => finish(failed(`the check worker exited (${code})`)));
33
65
  });
34
66
  }
67
+
68
+ const OPEN_TIMEOUT_MS = () => Number(env("OPEN_TIMEOUT_MS", 60_000));
69
+ // The open worker's heap: the largest documents the readers accept (their element, paragraph, shape and cell caps) took up to about
70
+ // 550 MB. ONIONSKIN_OPEN_HEAP_MB lowers it in tests.
71
+ const OPEN_HEAP_MB = () => Number(env("OPEN_HEAP_MB", 1536));
72
+
73
+ // At most OPEN_AT_ONCE open readings run at a time (each may use up to OPEN_HEAP_MB); the rest wait their turn, at most OPEN_WAIT_MS each
74
+ // and OPEN_QUEUE of them, then get "busy". So the worst wait for an answer is OPEN_WAIT_MS + the open time limit (the CLI waits longer).
75
+ const OPEN_AT_ONCE = 2;
76
+ const OPEN_QUEUE = 16;
77
+ export const OPEN_WAIT_MS = 60_000;
78
+ let running = 0;
79
+ const waiting = [];
80
+ function nextOpen() {
81
+ while (running < OPEN_AT_ONCE && waiting.length) {
82
+ const w = waiting.shift();
83
+ clearTimeout(w.timer);
84
+ running++;
85
+ w.go();
86
+ }
87
+ }
88
+
89
+ /**
90
+ * Reads a file being opened in a worker thread (src/open-worker.js): the package check and every part the server reads later, so a slow or
91
+ * hostile file stalls only that thread. Resolves to {ok: true, sha} or {ok: false, stage, error}: stage "invalid" (not a package of this
92
+ * format), "unreadable" (a reader refused it, a cap included), "timeout", "memory" or "busy" (too many files are being opened at once);
93
+ * any other worker failure is "unreadable". Never rejects.
94
+ */
95
+ export function readOffThread(bytes, format) {
96
+ return new Promise((resolve) => {
97
+ const busy = { ok: false, stage: "busy", error: `${OPEN_AT_ONCE} files are being read already` };
98
+ if (waiting.length >= OPEN_QUEUE) return resolve(busy);
99
+ const w = {
100
+ go: () =>
101
+ readNow(bytes, format).then((r) => {
102
+ running--;
103
+ nextOpen();
104
+ resolve(r);
105
+ }),
106
+ timer: setTimeout(() => {
107
+ waiting.splice(waiting.indexOf(w), 1);
108
+ resolve(busy);
109
+ }, OPEN_WAIT_MS),
110
+ };
111
+ w.timer.unref?.();
112
+ waiting.push(w);
113
+ nextOpen();
114
+ });
115
+ }
116
+
117
+ function readNow(bytes, format) {
118
+ return new Promise((resolve) => {
119
+ let settled = false;
120
+ let worker;
121
+ const finish = (r) => {
122
+ if (settled) return;
123
+ settled = true;
124
+ clearTimeout(timer);
125
+ worker?.terminate();
126
+ resolve(r);
127
+ };
128
+ const ms = OPEN_TIMEOUT_MS();
129
+ const secs = Math.max(1, Math.round(ms / 1000));
130
+ const timer = setTimeout(() => finish({ ok: false, stage: "timeout", error: `reading it took longer than ${secs} second${secs === 1 ? "" : "s"}` }), ms);
131
+ try {
132
+ worker = startWorker("open-worker.js", { workerData: { bytes, format }, resourceLimits: { maxOldGenerationSizeMb: OPEN_HEAP_MB() } });
133
+ } catch (err) {
134
+ return finish({ ok: false, stage: "unreadable", error: String(err?.message || err) });
135
+ }
136
+ worker.once("message", finish);
137
+ worker.once("error", (err) => finish(err?.code === "ERR_WORKER_OUT_OF_MEMORY" ? { ok: false, stage: "memory", error: "reading it needs more memory than Onionskin allows" } : { ok: false, stage: "unreadable", error: String(err?.message || err) }));
138
+ worker.once("exit", (code) => finish({ ok: false, stage: "unreadable", error: `the reading worker exited (${code})` }));
139
+ });
140
+ }
141
+
142
+ // The merge of what the person typed while the agent worked (src/merge-worker.js). The time limit: the longest merge at the readers'
143
+ // caps measured here, a workbook of 1,000,000 cells (three versions read, the agent's run checked again, the plan and the engine's
144
+ // run), took 14 to 15 s at a load average of 20 (e2e/stall-cap.e2e.test.js), so 120 s leaves room for a slower or busier Mac. The heap:
145
+ // that merge needed more than 1,024 MB and less than 1,536 MB; 2,048 MB. ONIONSKIN_MERGE_TIMEOUT_MS and _MERGE_HEAP_MB set them.
146
+ // At most MERGE_AT_ONCE run at a time; another is refused as busy (the page keeps the person's edits and says so).
147
+ const MERGE_TIMEOUT_MS = () => Number(env("MERGE_TIMEOUT_MS", 120_000));
148
+ const MERGE_HEAP_MB = () => Number(env("MERGE_HEAP_MB", 2048));
149
+ const MERGE_AT_ONCE = 2;
150
+ let merging = 0;
151
+
152
+ /**
153
+ * Runs one merge in a worker thread with a time and memory limit, so a large or crafted document never holds the server's thread.
154
+ * `job` = {format, base, mine, theirs, name, edits, agent}. Resolves to what the worker answered ({ok: true, ...} | {cap} | {invalid} |
155
+ * {error}), or {failed: "timeout" | "memory" | "busy" | "crashed", why}. Never rejects.
156
+ */
157
+ export function mergeOffThread(job) {
158
+ return new Promise((resolve) => {
159
+ if (merging >= MERGE_AT_ONCE) return resolve({ failed: "busy", why: `${MERGE_AT_ONCE} merges are running already` });
160
+ merging++;
161
+ let settled = false;
162
+ let worker;
163
+ const finish = (r) => {
164
+ if (settled) return;
165
+ settled = true;
166
+ merging--;
167
+ clearTimeout(timer);
168
+ worker?.terminate();
169
+ resolve(r);
170
+ };
171
+ const ms = MERGE_TIMEOUT_MS();
172
+ const secs = Math.max(1, Math.round(ms / 1000));
173
+ const timer = setTimeout(() => finish({ failed: "timeout", why: `it took longer than ${secs} second${secs === 1 ? "" : "s"}` }), ms);
174
+ try {
175
+ worker = startWorker("merge-worker.js", { workerData: job, resourceLimits: { maxOldGenerationSizeMb: MERGE_HEAP_MB() } });
176
+ } catch (err) {
177
+ return finish({ failed: "crashed", why: String(err?.message || err) });
178
+ }
179
+ worker.once("message", finish);
180
+ worker.once("error", (err) => finish(err?.code === "ERR_WORKER_OUT_OF_MEMORY" ? { failed: "memory", why: "it needs more memory than Onionskin allows" } : { failed: "crashed", why: String(err?.message || err).slice(0, 200) }));
181
+ worker.once("exit", (code) => finish({ failed: "crashed", why: `the merge worker exited (${code})` }));
182
+ });
183
+ }
184
+
185
+ // The engine's work for a quick change (src/ops-worker.js): the preview's count, the plan's count and the change itself. Time limit: a
186
+ // count over a document at the readers' caps (8 million characters, 4 million matches) took about 2 s here, the engine's open included,
187
+ // and a change of every match a few times that; 120 s leaves room for a slower or busier Mac. Heap: as a merge. ONIONSKIN_OPS_TIMEOUT_MS
188
+ // and _OPS_HEAP_MB set them. At most OPS_AT_ONCE run at a time; another is answered as busy (a preview is then not shown, a change is
189
+ // left to the agent).
190
+ const OPS_TIMEOUT_MS = () => Number(env("OPS_TIMEOUT_MS", 120_000));
191
+ const OPS_HEAP_MB = () => Number(env("OPS_HEAP_MB", 2048));
192
+ const OPS_AT_ONCE = 2;
193
+ let opsRunning = 0;
194
+
195
+ /**
196
+ * Runs `job` ({kind: "preview", bytes, format, op, name} | {kind: "run", bytes, format, name, ops, partial, anchors: [[k, v]]}) in a
197
+ * worker thread. Resolves to what the worker answered ({ok: true, p} | {ok: true, r}) or {failed: "timeout" | "memory" | "busy" |
198
+ * "crashed", why}. Never rejects.
199
+ */
200
+ export function opsOffThread(job) {
201
+ return new Promise((resolve) => {
202
+ if (opsRunning >= OPS_AT_ONCE) return resolve({ failed: "busy", why: `${OPS_AT_ONCE} changes are being counted or made already` });
203
+ opsRunning++;
204
+ let settled = false;
205
+ let worker;
206
+ const finish = (r) => {
207
+ if (settled) return;
208
+ settled = true;
209
+ opsRunning--;
210
+ clearTimeout(timer);
211
+ worker?.terminate();
212
+ resolve(r);
213
+ };
214
+ const ms = OPS_TIMEOUT_MS();
215
+ const secs = Math.max(1, Math.round(ms / 1000));
216
+ const timer = setTimeout(() => finish({ failed: "timeout", why: `it took longer than ${secs} second${secs === 1 ? "" : "s"}` }), ms);
217
+ try {
218
+ worker = startWorker("ops-worker.js", { workerData: job, resourceLimits: { maxOldGenerationSizeMb: OPS_HEAP_MB() } });
219
+ } catch (err) {
220
+ return finish({ failed: "crashed", why: String(err?.message || err).slice(0, 200) });
221
+ }
222
+ worker.once("message", finish);
223
+ worker.once("error", (err) => finish(err?.code === "ERR_WORKER_OUT_OF_MEMORY" ? { failed: "memory", why: "it needs more memory than Onionskin allows" } : { failed: "crashed", why: String(err?.message || err).slice(0, 200) }));
224
+ worker.once("exit", (code) => finish({ failed: "crashed", why: `the worker exited (${code})` }));
225
+ });
226
+ }
227
+
228
+ // The render copy of an HTML file (src/html-render-worker.js, PLAN-html R12). Time limit: a page at the HTML limits (9.2 MB, 198,000
229
+ // elements, 30 MB of images) took 2.6 s in this worker, its reading included, at a load average of 17 (test/html-render-limits.test.js);
230
+ // 60 s leaves room for a slower or busier Mac. Heap: 1 GB, the HTML limit (R22; at most two trees alive). ONIONSKIN_RENDER_TIMEOUT_MS and _RENDER_HEAP_MB set them. At most
231
+ // RENDER_AT_ONCE run at a time; another is answered as busy.
232
+ const RENDER_TIMEOUT_MS = () => Number(env("RENDER_TIMEOUT_MS", 60_000));
233
+ const RENDER_HEAP_MB = () => Number(env("RENDER_HEAP_MB", 1024));
234
+ const RENDER_AT_ONCE = 2;
235
+ let rendering = 0;
236
+
237
+ /**
238
+ * Builds the render copy of `bytes` (one version of an HTML file in folder `fileDir`, the folder of its real path) in a worker thread.
239
+ * Resolves to {ok: true, plain, reason, bytes (Buffer), texts, stats}, {ok: false, code, error} (the file was refused: a limit, its
240
+ * encoding) or {failed: "timeout" | "memory" | "busy" | "crashed", why}. Never rejects.
241
+ */
242
+ export function renderOffThread({ bytes, fileDir, plain = false }) {
243
+ return new Promise((resolve) => {
244
+ if (rendering >= RENDER_AT_ONCE) return resolve({ failed: "busy", why: `${RENDER_AT_ONCE} pages are being prepared already` });
245
+ rendering++;
246
+ let settled = false;
247
+ let worker;
248
+ const finish = (r) => {
249
+ if (settled) return;
250
+ settled = true;
251
+ rendering--;
252
+ clearTimeout(timer);
253
+ worker?.terminate();
254
+ resolve(r?.ok && r.bytes ? { ...r, bytes: Buffer.from(r.bytes.buffer, r.bytes.byteOffset, r.bytes.byteLength) } : r);
255
+ };
256
+ const ms = RENDER_TIMEOUT_MS();
257
+ const secs = Math.max(1, Math.round(ms / 1000));
258
+ const timer = setTimeout(() => finish({ failed: "timeout", why: `it took longer than ${secs} second${secs === 1 ? "" : "s"}` }), ms);
259
+ try {
260
+ worker = startWorker("html-render-worker.js", { workerData: { bytes, fileDir, plain }, resourceLimits: { maxOldGenerationSizeMb: RENDER_HEAP_MB() } });
261
+ } catch (err) {
262
+ return finish({ failed: "crashed", why: String(err?.message || err).slice(0, 200) });
263
+ }
264
+ worker.once("message", finish);
265
+ worker.once("error", (err) => finish(err?.code === "ERR_WORKER_OUT_OF_MEMORY" ? { failed: "memory", why: "it needs more memory than Onionskin allows" } : { failed: "crashed", why: String(err?.message || err).slice(0, 200) }));
266
+ worker.once("exit", (code) => finish({ failed: "crashed", why: `the worker exited (${code})` }));
267
+ });
268
+ }
269
+
270
+ /** previewAll (src/ops/run.js) in the ops worker: its answer, or a thrown Error when the worker did not finish (nothing was counted). */
271
+ export async function previewOffThread(bytes, format, op) {
272
+ const r = await opsOffThread({ kind: "preview", bytes, format, op, name: `preview.${format}` });
273
+ if (!r.ok) throw new Error(`the count did not finish: ${r.why}`);
274
+ return r.p;
275
+ }
276
+
277
+ /**
278
+ * runOps (src/ops/run.js) in the ops worker, with its answer's shape. A worker that does not finish writes nothing: every op reads
279
+ * not_applied with the reason, and there are no bytes.
280
+ */
281
+ export async function runOpsOffThread({ bytes, format, name, ops, partial = false, anchors = new Map() }) {
282
+ const r = await opsOffThread({ kind: "run", bytes, format, name, ops, partial, anchors: [...anchors] });
283
+ if (r.ok) return { ...r.r, bytes: r.r.bytes ? Buffer.from(r.r.bytes.buffer, r.r.bytes.byteOffset, r.r.bytes.byteLength) : null };
284
+ const reason = `the change did not finish: ${r.why}; nothing was written`;
285
+ return { results: ops.map((op, index) => ({ index, op: op.op, ...(op.id ? { id: op.id } : {}), status: "not_applied", reason })), bytes: null, warnings: [], blocked: null };
286
+ }
287
+
288
+ // ---- anchors: the document reading and anchor work of a request (src/anchor-work.js) -----------------------------------------
289
+ // Warm workers (src/anchor-worker.js), each one job at a time: at most ANCHOR_POOL at once, so two requests for different documents (or
290
+ // a crafted one and an ordinary one) never wait for each other; a request past them waits its turn within its own time limit, at most
291
+ // ANCHOR_QUEUE of them, then is answered "busy". A worker that runs past its time limit, runs out of memory or crashes is stopped and
292
+ // replaced; the request it held is answered as failed, and its caller makes its comments weaker the way a spent budget does (never an
293
+ // error that loses one). One worker stays warm; the others stop after ANCHOR_IDLE_MS idle, with the reading they keep.
294
+ // Time limit: a reading at the readers' caps took under 1 s (docs/GUIDE.md "Size limits"; the open worker allows 60 s); the work after it is bounded by the
295
+ // worker's budget (all of it: 0.4 to 1.3 s of CPU here). ONIONSKIN_ANCHOR_TIMEOUT_MS and _ANCHOR_HEAP_MB set them.
296
+ const ANCHOR_TIMEOUT_MS = () => Number(env("ANCHOR_TIMEOUT_MS", 60_000));
297
+ /** The anchor worker's time limit for one job, in ms (src/session.js settled bounds a page load's wait by it). */
298
+ export const anchorTimeoutMs = ANCHOR_TIMEOUT_MS;
299
+ const ANCHOR_HEAP_MB = () => Number(env("ANCHOR_HEAP_MB", 2048));
300
+ const ANCHOR_POOL = 3;
301
+ const ANCHOR_QUEUE = 16;
302
+ const ANCHOR_IDLE_MS = 60_000;
303
+ const anchorPool = []; // {worker, job: the one in progress or null, sha: of the version it read last, idle: timer}
304
+ const anchorWaiting = [];
305
+ let anchorIds = 0;
306
+
307
+ function anchorWorker() {
308
+ const w = { worker: startWorker("anchor-worker.js", { resourceLimits: { maxOldGenerationSizeMb: ANCHOR_HEAP_MB() } }), job: null, sha: null, idle: null };
309
+ w.worker.unref(); // a warm worker never keeps the process alive
310
+ const fail = (failed, why) => {
311
+ const i = anchorPool.indexOf(w);
312
+ if (i >= 0) anchorPool.splice(i, 1);
313
+ w.dead = true;
314
+ clearTimeout(w.idle);
315
+ w.job?.finish({ failed, why });
316
+ w.worker.terminate();
317
+ nextAnchor();
318
+ };
319
+ w.worker.on("message", (m) => {
320
+ if (!w.job || m?.id !== w.job.id) return;
321
+ if (m.ok) w.sha = m.sha;
322
+ w.job.finish(m.ok ? { ok: true, sha: m.sha, r: m.r } : { failed: m.stale ? "stale" : "crashed", why: m.error });
323
+ });
324
+ w.worker.on("error", (err) => fail(err?.code === "ERR_WORKER_OUT_OF_MEMORY" ? "memory" : "crashed", String(err?.message || err).slice(0, 200)));
325
+ w.worker.on("exit", (code) => fail("crashed", `the anchor worker exited (${code})`));
326
+ w.fail = fail;
327
+ anchorPool.push(w);
328
+ return w;
329
+ }
330
+
331
+ function runAnchor(w, job) {
332
+ clearTimeout(w.idle);
333
+ w.job = job;
334
+ const msg = { id: job.id, job: job.data };
335
+ w.worker.postMessage(msg);
336
+ job.onDone = () => {
337
+ w.job = null;
338
+ if (w.dead) return;
339
+ if (anchorPool.filter((x) => !x.dead).length > 1) {
340
+ w.idle = setTimeout(() => !w.job && w.fail("idle", "idle"), ANCHOR_IDLE_MS);
341
+ w.idle.unref?.();
342
+ }
343
+ nextAnchor();
344
+ };
345
+ }
346
+
347
+ function nextAnchor() {
348
+ while (anchorWaiting.length) {
349
+ const job = anchorWaiting[0];
350
+ const idle = anchorPool.filter((w) => !w.job && !w.dead);
351
+ const w = idle.find((x) => x.sha && x.sha === job.data.source.sha) || idle[0] || (anchorPool.length < ANCHOR_POOL ? anchorWorker() : null);
352
+ if (!w) return;
353
+ anchorWaiting.shift();
354
+ runAnchor(w, job);
355
+ }
356
+ }
357
+
358
+ /**
359
+ * src/anchor-work.js anchorWork in a warm worker. `data` = {format, source: {bytes} | {path, sha?}, check, tasks, outline}. Resolves to
360
+ * {ok: true, sha, r} or {failed: "timeout" | "memory" | "busy" | "crashed" | "stale", why}. Never rejects. A job that times out stops its
361
+ * worker (the next job gets a fresh one).
362
+ */
363
+ export function anchorsOffThread(data) {
364
+ return new Promise((resolve) => {
365
+ if (anchorWaiting.length >= ANCHOR_QUEUE) return resolve({ failed: "busy", why: `${ANCHOR_QUEUE} requests are waiting already` });
366
+ const job = { id: ++anchorIds, data, settled: false };
367
+ const ms = ANCHOR_TIMEOUT_MS();
368
+ const timer = setTimeout(() => {
369
+ const i = anchorWaiting.indexOf(job);
370
+ if (i >= 0) {
371
+ anchorWaiting.splice(i, 1);
372
+ return job.finish({ failed: "busy", why: "the other requests did not finish in time" });
373
+ }
374
+ const w = anchorPool.find((x) => x.job === job);
375
+ if (w) w.fail("timeout", `it took longer than ${Math.max(1, Math.round(ms / 1000))} s`);
376
+ else job.finish({ failed: "timeout", why: "timed out" });
377
+ }, ms);
378
+ job.finish = (r) => {
379
+ if (job.settled) return;
380
+ job.settled = true;
381
+ clearTimeout(timer);
382
+ resolve(r);
383
+ job.onDone?.();
384
+ };
385
+ anchorWaiting.push(job);
386
+ nextAnchor();
387
+ });
388
+ }
389
+
390
+ /** Starts the warm anchor worker ahead of the first request (the server calls it at start), so a first Send does not wait for its start. */
391
+ export function warmAnchors() {
392
+ if (!anchorPool.length) anchorWorker();
393
+ }
394
+
395
+ // ---- the write check of an HTML file (src/html-write-worker.js) -------------------------------------------------------------------
396
+ // One parse of each version and an inventory of each: about 1.5 s of worker CPU at the limits (10 MB, 200,000 elements), with at most
397
+ // two trees alive (about 600 MB). Time limit and heap as the ops worker's (ONIONSKIN_OPS_TIMEOUT_MS, _OPS_HEAP_MB). At most HTML_AT_ONCE
398
+ // at a time; another is answered as busy, and the write is refused (nothing is written unchecked).
399
+ const HTML_AT_ONCE = 2;
400
+ let htmlRunning = 0;
401
+
402
+ /**
403
+ * src/html-write.js checkedHtmlWrite in a worker thread: `job` = {before, after, edits?} (bytes; edits [{nodeId, text}]). Resolves to its
404
+ * answer, with `bytes` as a Buffer: {ok: true, bytes, charset_added, links} | {ok: false, code, reason, reasons?}; a worker that does not
405
+ * finish resolves to {ok: false, code: "check_failed", reason}. Never rejects.
406
+ */
407
+ export function htmlWriteOffThread(job) {
408
+ return new Promise((resolve) => {
409
+ const failed = (why) => ({ ok: false, code: "check_failed", reason: `the write check did not finish: ${why}; nothing was written` });
410
+ if (htmlRunning >= HTML_AT_ONCE) return resolve(failed(`${HTML_AT_ONCE} checks are running already`));
411
+ htmlRunning++;
412
+ let settled = false;
413
+ let worker;
414
+ const finish = (r) => {
415
+ if (settled) return;
416
+ settled = true;
417
+ htmlRunning--;
418
+ clearTimeout(timer);
419
+ worker?.terminate();
420
+ resolve(r?.ok && r.bytes ? { ...r, bytes: Buffer.from(r.bytes.buffer, r.bytes.byteOffset, r.bytes.byteLength) } : r);
421
+ };
422
+ const ms = OPS_TIMEOUT_MS();
423
+ const timer = setTimeout(() => finish(failed(`it took longer than ${Math.max(1, Math.round(ms / 1000))} s`)), ms);
424
+ try {
425
+ worker = startWorker("html-write-worker.js", { workerData: job, resourceLimits: { maxOldGenerationSizeMb: OPS_HEAP_MB() } });
426
+ } catch (err) {
427
+ return finish(failed(String(err?.message || err).slice(0, 200)));
428
+ }
429
+ worker.once("message", finish);
430
+ worker.once("error", (err) => finish(failed(err?.code === "ERR_WORKER_OUT_OF_MEMORY" ? "it needed more memory than Onionskin allows" : String(err?.message || err).slice(0, 200))));
431
+ worker.once("exit", (code) => finish(failed(`the worker exited (${code})`)));
432
+ });
433
+ }
@@ -0,0 +1,27 @@
1
+ import { env } from "./constants.js";
2
+ import { sha256 } from "./fsutil.js";
3
+ import { validatePackage } from "./package.js";
4
+ import { readAll, readUnits } from "./anchors.js";
5
+ import "./html-parser.js"; // an HTML file is read whole here too: the parser lives in worker threads only
6
+ import { HtmlEncodingError } from "./html.js";
7
+
8
+ // The first reading of a file being opened (src/offload.js readOffThread), run by src/worker-host.js: the package check and every part the
9
+ // server later reads (paragraphs, every sheet, every slide), so a damaged or hostile file is refused here, on this thread, within a time
10
+ // and memory limit. ONIONSKIN_TEST_OPEN_DELAY_MS (tests only) blocks this thread for that long, the way a slow parse would.
11
+ export function work({ bytes: raw, format }) {
12
+ const delay = Number(env("TEST_OPEN_DELAY_MS", 0));
13
+ if (delay > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, delay);
14
+ const bytes = Buffer.from(raw);
15
+ let stage = "invalid";
16
+ try {
17
+ validatePackage(bytes, format);
18
+ stage = "unreadable";
19
+ // every slide and every sheet in one reading, as a review or an agent's outline reads them: the caps count over all of them
20
+ readAll(readUnits(format, bytes));
21
+ return { ok: true, sha: sha256(bytes) };
22
+ } catch (err) {
23
+ // an HTML file not saved as UTF-8 (PLAN-html R10): its fixed sentence and whether a UTF-8 copy can be made, never mangled
24
+ if (err instanceof HtmlEncodingError) return { ok: false, stage: "encoding", error: err.message, code: err.code, copyable: err.copyable };
25
+ return { ok: false, stage, error: String(err?.message || err) };
26
+ }
27
+ }
package/src/ops/common.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { nfc } from "../nfc.js";
1
2
  import { exactHash, normText, textHash } from "../text.js";
2
3
  import { untrusted } from "../sanitize.js";
3
4
 
@@ -21,7 +22,7 @@ export function checkExpect(expect, text, mode = "contains") {
21
22
  const exact = mode === "exact";
22
23
  const hash = exact ? exactHash(text) : textHash(text);
23
24
  if ("quote" in expect) {
24
- const ok = exact ? text.normalize("NFC") === expect.quote.normalize("NFC") : normText(text).includes(normText(expect.quote));
25
+ const ok = exact ? nfc(text) === nfc(expect.quote) : normText(text).includes(normText(expect.quote));
25
26
  if (!ok) return fail("expect_mismatch", "the target no longer says what `expect.quote` says", { actual_excerpt: excerpt(text) });
26
27
  }
27
28
  if ("text_hash" in expect && expect.text_hash !== hash) return fail("expect_mismatch", "the target's text hash differs from `expect.text_hash`", { actual_excerpt: excerpt(text), actual_text_hash: hash });