onionskin-cli 0.0.3 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CHANGELOG.md +198 -0
  2. package/INSTALL.md +5 -1
  3. package/NOTICE +34 -0
  4. package/README.md +102 -34
  5. package/docs/contracts/batch.schema.json +27 -4
  6. package/docs/contracts/ops.schema.json +117 -5
  7. package/package.json +4 -2
  8. package/scripts/checker/check.py +24 -5
  9. package/scripts/checker/requirements.txt +482 -5
  10. package/scripts/requirements.txt +482 -5
  11. package/scripts/rewriter/apple/main.swift +1 -1
  12. package/scripts/rewriter/apple/onionskin-fm +0 -0
  13. package/scripts/rewriter/apple/onionskin-fm.json +1 -1
  14. package/skills/onionskin/SKILL.md +12 -9
  15. package/skills/onionskin/reference/batch.md +5 -3
  16. package/skills/onionskin/reference/formats.md +5 -1
  17. package/skills/onionskin/reference/ops.md +48 -4
  18. package/skills/onionskin/reference/troubleshooting.md +31 -0
  19. package/src/agent-detect.js +28 -2
  20. package/src/agent-prompt.js +12 -1
  21. package/src/agent-runner.js +110 -20
  22. package/src/anchor-work.js +154 -0
  23. package/src/anchor-worker.js +38 -0
  24. package/src/anchors.js +61 -11
  25. package/src/apply.js +119 -47
  26. package/src/batch.js +32 -8
  27. package/src/check-worker.js +8 -5
  28. package/src/checker.js +238 -33
  29. package/src/cli.js +248 -42
  30. package/src/constants.js +34 -1
  31. package/src/diff.js +3 -1
  32. package/src/doctor.js +12 -3
  33. package/src/docx.js +464 -87
  34. package/src/edit-hash.js +19 -0
  35. package/src/fsutil.js +14 -6
  36. package/src/history-worker.js +8 -8
  37. package/src/history.js +144 -31
  38. package/src/html-active.js +384 -0
  39. package/src/html-envelope.js +51 -0
  40. package/src/html-parser.js +8 -0
  41. package/src/html-render-worker.js +25 -0
  42. package/src/html-render.js +939 -0
  43. package/src/html-utf8.js +65 -0
  44. package/src/html-write-worker.js +14 -0
  45. package/src/html-write.js +380 -0
  46. package/src/html.js +1259 -0
  47. package/src/local.js +68 -4
  48. package/src/merge-package.js +1166 -0
  49. package/src/merge-pptx.js +98 -38
  50. package/src/merge-proof.js +1019 -0
  51. package/src/merge-worker.js +60 -0
  52. package/src/merge-xlsx.js +84 -29
  53. package/src/merge.js +286 -83
  54. package/src/nfc.js +46 -0
  55. package/src/notes.js +122 -0
  56. package/src/offload.js +419 -1
  57. package/src/open-worker.js +27 -0
  58. package/src/ops/common.js +2 -1
  59. package/src/ops/docx-table.js +5 -3
  60. package/src/ops/docx.js +58 -29
  61. package/src/ops/html.js +394 -0
  62. package/src/ops/pptx.js +82 -23
  63. package/src/ops/run.js +23 -3
  64. package/src/ops/runs.js +215 -13
  65. package/src/ops/schema.js +8 -1
  66. package/src/ops/xlsx.js +121 -10
  67. package/src/ops-worker.js +17 -0
  68. package/src/package.js +43 -4
  69. package/src/person.js +2 -1
  70. package/src/pptx.js +323 -23
  71. package/src/question.js +10 -3
  72. package/src/quick-plan.js +13 -6
  73. package/src/quick.js +1 -1
  74. package/src/redline-check.js +12 -2
  75. package/src/redline-worker.js +10 -14
  76. package/src/reply.js +46 -19
  77. package/src/retain.js +223 -0
  78. package/src/review.js +32 -18
  79. package/src/rewrite-check.js +2 -2
  80. package/src/rewriter.js +38 -8
  81. package/src/sanitize.js +16 -3
  82. package/src/server-entry.js +4 -3
  83. package/src/server.js +175 -62
  84. package/src/session.js +1151 -210
  85. package/src/setup.js +20 -2
  86. package/src/stop-gate.js +34 -0
  87. package/src/suggest-reply.js +42 -7
  88. package/src/suggest.js +14 -29
  89. package/src/text.js +161 -11
  90. package/src/update.js +517 -0
  91. package/src/voice-guide.js +15 -9
  92. package/src/voice-learn.js +15 -4
  93. package/src/voice-sample-worker.js +7 -6
  94. package/src/voice-store.js +11 -4
  95. package/src/voice-worker.js +4 -6
  96. package/src/voice.js +2 -1
  97. package/src/worker-host.js +23 -0
  98. package/src/xlsx.js +378 -97
  99. package/src/xml-tree.js +6 -3
  100. package/src/xml.js +82 -34
  101. package/src/zip-write.js +10 -5
  102. package/src/zip.js +73 -18
  103. package/vendor/parse5/LICENSE.txt +32 -0
  104. package/vendor/parse5/parse5.mjs +7822 -0
  105. package/vendor/parse5.lock.json +23 -0
  106. package/web/dist/assets/{index-DS-WST1E.js → index-CjsTf5gj.js} +58 -52
  107. package/web/dist/assets/index-DnbLtHhG.css +1 -0
  108. package/web/dist/index.html +2 -2
  109. package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/reply.js CHANGED
@@ -1,9 +1,9 @@
1
1
  import fs from "node:fs";
2
2
  import path from "node:path";
3
- import { FORMATS, PRODUCT } from "./constants.js";
4
- import { ApplyError, APPLY_EXIT, applyCommand } from "./apply.js";
5
- import { parseTicket } from "./lease.js";
6
- import { isAgentHolder } from "./agent-detect.js";
3
+ import { PRODUCT, unknownFormat } from "./constants.js";
4
+ import { ApplyError, APPLY_EXIT, applyCommand, formatOfPath } from "./apply.js";
5
+ import { loadTicketRecord, parseTicket } from "./lease.js";
6
+ import { NOT_YOUR_FILE, agentMayUse, isAgentHolder } from "./agent-detect.js";
7
7
  import { QUESTION_FIX, QUESTIONS_PER_BATCH, cleanQuestion } from "./question.js";
8
8
  import { SUGGESTIONS_MAX, SUGGEST_FIX, cleanSuggestion } from "./review.js";
9
9
  import { VOICE_FIX, cleanVoiceDraft } from "./voice-guide.js";
@@ -186,13 +186,16 @@ export function planReplies(batch, input, { partial = false } = {}) {
186
186
  if (mode === "find") {
187
187
  if (typeof r.find !== "string" || !r.find || typeof r.replace !== "string") return refuse("bad_reply", "find needs a non-empty string and replace needs a string");
188
188
  const extras = Object.fromEntries(SHORTHAND_EXTRAS.filter((k) => k in r).map((k) => [k, r[k]]));
189
- if (format === "docx") op = { op: "replace_text", ...base, find: r.find, replace: r.replace, ...extras };
189
+ if (format === "docx" || format === "html") op = { op: "replace_text", ...base, find: r.find, replace: r.replace, ...extras };
190
190
  else if (format === "pptx") op = { op: "replace_text_in_shape", ...base, find: r.find, replace: r.replace, ...extras };
191
+ else if (format !== "xlsx") return unknownFormat(format, "reply find");
191
192
  else return refuse("shorthand_not_for_format", "find/replace does not apply to a .xlsx cell", "use `set` for a cell or range");
192
193
  } else if (mode === "text") {
193
194
  if (typeof r.text !== "string") return refuse("bad_reply", "text must be a string");
194
195
  if (format === "docx") op = { op: "replace_text", ...base, text: r.text };
196
+ else if (format === "html") op = { op: "set_text", ...base, text: r.text };
195
197
  else if (format === "pptx") op = { op: "set_shape_text", ...base, paragraphs: r.text.split("\n") };
198
+ else if (format !== "xlsx") return unknownFormat(format, "reply text");
196
199
  else return refuse("shorthand_not_for_format", "text does not apply to a .xlsx cell", "use `set` for a cell or range");
197
200
  } else if (mode === "set") {
198
201
  if (format !== "xlsx") return refuse("shorthand_not_for_format", `set applies to a .xlsx cell or range, not a .${format}`, "use `find`+`replace` or `text`");
@@ -264,7 +267,7 @@ const FIX = {
264
267
  skipped_protected: "Protected place (field, equation, content control or similar): it cannot be rewritten losslessly. Decline this comment with a note.",
265
268
  };
266
269
  const fixFor = (r) => FIX[r.status] || (r.status === "refused" ? `Refused by the engine${r.code ? ` (${r.code})` : ""}. Change the op as the reason says, or decline.` : undefined);
267
- const brief = (comment, r) => ({ comment, op: r.op, status: r.status, ...(r.code ? { code: r.code } : {}), ...(r.reason ? { reason: r.reason } : {}), ...(r.actual_excerpt ? { actual_excerpt: r.actual_excerpt } : {}), ...(fixFor(r) ? { fix: fixFor(r) } : {}) });
270
+ const brief = (comment, r) => ({ comment, op: r.op, status: r.status, ...(r.code ? { code: r.code } : {}), ...(r.reason ? { reason: r.reason } : {}), ...(Array.isArray(r.reasons) ? { reasons: r.reasons } : {}), ...(r.actual_excerpt ? { actual_excerpt: r.actual_excerpt } : {}), ...(fixFor(r) ? { fix: fixFor(r) } : {}) });
268
271
 
269
272
  /**
270
273
  * The command. `deps` = {server: {healthy, req, json}, finish(file, results, lease) -> done's JSON, pollNext(file, timeout) -> string, readStdin(), holder?}.
@@ -279,32 +282,56 @@ export async function replyCommand(rest, deps) {
279
282
  throw new ReplyError(err.message);
280
283
  }
281
284
  const { server } = deps;
285
+ // who calls: the local runner names itself; otherwise ONIONSKIN_HOLDER (an agent Onionskin started cannot change it), else the holder the
286
+ // server saved with this ticket. It goes with every request, so the server checks the binding and the lease before it hands out anything.
287
+ const envHolder = process.env[`${PRODUCT.envPrefix}_HOLDER`];
288
+ const ticketOfArgs = parseTicket(a.lease);
289
+ const holder = deps.holder || envHolder || (ticketOfArgs ? loadTicketRecord(ticketOfArgs.batch_id)?.holder : null) || "";
290
+ const notYours = () => new ReplyError(NOT_YOUR_FILE, { exit: APPLY_EXIT.write_refused, body: { code: "not_your_file" } });
282
291
  if (!a.file) {
283
- // no file given: the ticket's batch id names the session (the token is still checked by apply and done)
292
+ // no file given: the whole ticket (batch id and token) names the session; a wrong token names none
284
293
  if (!(await server.healthy())) throw new ReplyError(`the ${PRODUCT.name} server is not running, so the delivery named by --lease cannot be checked. Nothing was written.`, { exit: APPLY_EXIT.write_refused });
285
- const w = await server.req({ method: "GET", path: `/cli/batch-file?batch_id=${encodeURIComponent(parseTicket(a.lease)?.batch_id || "")}` });
294
+ const w = await server.req({ method: "GET", path: `/cli/batch-file?batch_id=${encodeURIComponent(ticketOfArgs?.batch_id || "")}&lease_token=${encodeURIComponent(ticketOfArgs?.token || "")}&holder=${encodeURIComponent(holder)}` });
286
295
  a.file = w.status === 200 ? server.json(w.text).file || "" : "";
287
296
  if (!a.file) throw new ReplyError(`No delivered batch matches this ticket: your delivery was taken back, finished or handed on. Nothing was written. Run \`${PRODUCT.bin} poll <file>\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: "stale_batch" } });
288
297
  }
298
+ // an agent Onionskin started: its own input first (only its one reply file, content-free checks), then its binding to its one file,
299
+ // refused before that file is looked at, in the same words whether it exists or is open
300
+ const bg = !deps.holder && isAgentHolder(envHolder);
301
+ const ticketRequired = () => new ReplyError(`--lease <ticket> is required: it is the lease_ticket of the batch (\`${PRODUCT.bin} status <file> --ticket\` prints it again)`);
302
+ let input;
303
+ if (bg) {
304
+ if (!ticketOfArgs) throw ticketRequired();
305
+ checkBackgroundInput(a, process.env[`${PRODUCT.envPrefix}_AGENT_DIR`], ticketOfArgs.batch_id);
306
+ input = await readInput(a, deps.readStdin, { quiet: true });
307
+ if (!agentMayUse(envHolder, a.file)) throw notYours();
308
+ }
289
309
  let real;
290
310
  try {
291
311
  real = fs.realpathSync(path.resolve(a.file));
292
312
  } catch {
293
313
  throw new ReplyError(`File not found: ${JSON.stringify(a.file.slice(0, 200))}`, { exit: APPLY_EXIT.error });
294
314
  }
295
- if (!FORMATS[path.extname(real).toLowerCase().slice(1)]) throw new ReplyError("Only .docx, .xlsx and .pptx files are supported");
296
- const ticket = parseTicket(a.lease);
297
- if (!ticket) throw new ReplyError(`--lease <ticket> is required: it is the lease_ticket of the batch (\`${PRODUCT.bin} status <file> --ticket\` prints it again)`);
298
- const bg = !deps.holder && isAgentHolder(process.env[`${PRODUCT.envPrefix}_HOLDER`]);
299
- if (bg) checkBackgroundInput(a, process.env[`${PRODUCT.envPrefix}_AGENT_DIR`], ticket.batch_id);
300
- const input = await readInput(a, deps.readStdin, { quiet: bg });
315
+ if (!formatOfPath(real)) throw new ReplyError("Only .docx, .xlsx, .pptx, .html and .htm files are supported");
316
+ const ticket = ticketOfArgs;
317
+ if (!ticket) throw ticketRequired();
318
+ if (!bg) input = await readInput(a, deps.readStdin, { quiet: false });
301
319
 
302
- // the batch the session holds for this delivery: the only source of anchors and hashes
320
+ // authorization first: the server hands out the batch the session holds for this delivery (the only source of anchors and hashes) only
321
+ // to this ticket's holder, checked before the reply is planned or run against the file
303
322
  if (!(await server.healthy())) throw new ReplyError(`the ${PRODUCT.name} server is not running, so the delivery named by --lease cannot be checked. Nothing was written.`, { exit: APPLY_EXIT.write_refused });
304
- const r = await server.req({ method: "GET", path: `/cli/batch?file=${encodeURIComponent(real)}` });
305
- const held = r.status === 200 ? server.json(r.text).batch : null;
306
- if (!held) throw new ReplyError(`No batch is delivered for this file, so the ticket is void. Nothing was written. Run \`${PRODUCT.bin} poll\` and wait for the user's Send.`, { exit: APPLY_EXIT.write_refused, body: { code: "no_batch" } });
307
- if (held.batch_id !== ticket.batch_id) throw new ReplyError(`The ticket belongs to batch ${ticket.batch_id}, but the delivered batch is ${held.batch_id}: your delivery was taken back, finished or handed on. Nothing was written. Run \`${PRODUCT.bin} poll\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: "stale_batch" } });
323
+ const r = await server.req({ method: "GET", path: `/cli/batch?file=${encodeURIComponent(real)}&holder=${encodeURIComponent(holder)}&batch_id=${encodeURIComponent(ticket.batch_id)}&lease_token=${encodeURIComponent(ticket.token)}` });
324
+ let refused = null;
325
+ try {
326
+ refused = r.status === 200 ? null : server.json(r.text);
327
+ } catch {
328
+ refused = {};
329
+ }
330
+ if (refused?.code === "not_your_file") throw notYours();
331
+ if (refused?.code === "no_batch") throw new ReplyError(`No batch is delivered for this file, so the ticket is void. Nothing was written. Run \`${PRODUCT.bin} poll\` and wait for the user's Send.`, { exit: APPLY_EXIT.write_refused, body: { code: "no_batch" } });
332
+ if (refused) throw new ReplyError(`This ticket is not the delivery the session holds: your delivery was taken back, finished or handed on, or the ticket is wrong. Nothing was written. Run \`${PRODUCT.bin} poll\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: refused.code === "not_lease_holder" ? "not_lease_holder" : "stale_batch" } });
333
+ const held = server.json(r.text).batch;
334
+ if (!held || held.batch_id !== ticket.batch_id) throw new ReplyError(`This ticket is not the delivery the session holds. Nothing was written. Run \`${PRODUCT.bin} poll\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: "stale_batch" } });
308
335
 
309
336
  const plan = planReplies(held, input, { partial: a.partial });
310
337
  if (plan.errors.length) throw new ReplyError("the reply cannot be applied; nothing was written and the batch is still open", { body: { code: "bad_reply", lease_kept: true, problems: plan.errors } });
package/src/retain.js ADDED
@@ -0,0 +1,223 @@
1
+ // Retained size: what the server keeps of a request (the page's anchors, typed edits, view state) and of the anchors it builds, cut to
2
+ // fixed caps where it is stored, so a crafted file or page inside the documented size limits cannot make a session file, a Send's batch
3
+ // or a draft save hundreds of MB. Over a cap a value is cut and marked (`cut: true`), never refused: an anchor keeps a prefix of its
4
+ // text and its hash of the full text where one was paid for. Only a whole request past REQUEST_MAX is refused (src/server.js), with a
5
+ // plain message the page shows. Chosen from measurement (test/retain.test.js): ordinary use is untouched, see the numbers below.
6
+
7
+ /**
8
+ * One string field of an anchor (a quote, a paragraph's text, a typed edit's before and after). A whole-paragraph selection of an
9
+ * ordinary document is well under it: the longest paragraph of the sample files is under 300 characters and a long report paragraph
10
+ * 3,000 to 6,000. UTF-16 units.
11
+ */
12
+ export const STRING_MAX = 20_000;
13
+ /** A location's own strings (a sheet or shape name, a range, a paragraph id): Excel caps a sheet name at 31 characters. */
14
+ const NAME_MAX = 1_000;
15
+ /** A page anchor's context (prefix, suffix): the server keeps 40 characters of each (src/docx.js, src/pptx.js CTX). */
16
+ const CONTEXT_MAX = 1_000;
17
+ /**
18
+ * One stored anchor, as JSON. Measured: a whole-paragraph Word anchor about 2 to 4 KB plus its quote, a 1,000 x 20 Excel range of
19
+ * ordinary values about 11 KB, a PowerPoint shape 3 KB; a quote at STRING_MAX makes about 25 KB. Past it an Excel anchor's values are cut
20
+ * to VALUE_SHORT characters a cell, then the optional evidence (values, formats, kinds, names, row cells) goes; the hashes stay.
21
+ */
22
+ export const ANCHOR_MAX = 48 * 1024;
23
+ const VALUE_SHORT = 64;
24
+ /** One comment as stored (its words, the page's anchor, the anchor built from the file): past it the page's anchor is cut further. */
25
+ export const COMMENT_MAX = 128 * 1024;
26
+ /** One request's comments, note, typed edits and view state, as JSON in UTF-8 (not the document's bytes): past it the request is refused. */
27
+ export const REQUEST_MAX = 32 * 1024 * 1024;
28
+ /** One session file, in bytes. Past it every comment's page anchor and every typed edit are cut further before it is written (the waiting
29
+ * batch's evidence is already capped per anchor by src/batch.js); nothing is dropped. */
30
+ export const SESSION_MAX = 64 * 1024 * 1024;
31
+ /** Comments and pins of one file (drafts, sent, queued and finished): one more is refused in words (src/session.js), never dropped. */
32
+ export const COMMENTS_MAX = 500;
33
+ /** What the server says (413 too_many_comments) and the page shows when one more comment would pass COMMENTS_MAX. */
34
+ export const TOO_MANY_COMMENTS = `This file has ${COMMENTS_MAX} comments waiting; send or delete some before adding more.`;
35
+ /** The overall note, in code points: the box, the notes queued during a run and a waiting batch's note together. Past it a save or a
36
+ * Send is refused in words (src/session.js), never cut. The page's box takes no more (web/src/Sidebar.jsx). */
37
+ export const NOTE_MAX = 20_000;
38
+ /** One comment's words, in code points (src/sanitize.js userText): the page's box takes no more, so it is never cut from the page. */
39
+ export const COMMENT_TEXT_MAX = 5_000;
40
+ /** Typed edits kept for the next batch (they accumulate over Sends until one is delivered); later ones are counted, not kept. */
41
+ export const EDITS_MAX = 2_000;
42
+ /** A Word block path: the body index, then table row, cell and paragraph, deeper for nested tables. */
43
+ const PATH_MAX = 16;
44
+
45
+ const isHigh = (c) => c >= 0xd800 && c <= 0xdbff;
46
+ const isLow = (c) => c >= 0xdc00 && c <= 0xdfff;
47
+ // a slice keeps the whole string alive; this copy does not
48
+ const flat = (s) => Buffer.from(s, "utf16le").toString("utf16le");
49
+
50
+ /** `s` cut to its first `max` UTF-16 units (never inside a surrogate pair), as a flat string that does not hold on to the long one. */
51
+ export function cutText(s, max) {
52
+ if (typeof s !== "string" || s.length <= max) return s;
53
+ let n = max;
54
+ if (n > 0 && isHigh(s.charCodeAt(n - 1)) && isLow(s.charCodeAt(n))) n--;
55
+ return flat(s.slice(0, n));
56
+ }
57
+
58
+ /** `s` cut to its last `max` UTF-16 units (never inside a surrogate pair): the text just before a selection keeps the end nearest to it. */
59
+ export function cutTail(s, max) {
60
+ if (typeof s !== "string" || s.length <= max) return s;
61
+ let n = s.length - max;
62
+ if (isLow(s.charCodeAt(n)) && isHigh(s.charCodeAt(n - 1))) n++;
63
+ return flat(s.slice(n));
64
+ }
65
+
66
+ // Every object the page sends that is stored is rebuilt from the fields Onionskin reads, each of a known type and size; anything else is
67
+ // dropped, so the work and the size are bounded by this list, never by what the request carries.
68
+ const int = (v) => (Number.isSafeInteger(v) ? v : undefined);
69
+ const finite = (v) => (typeof v === "number" && Number.isFinite(v) ? v : undefined);
70
+ const str = (v, max) => (typeof v === "string" ? cutText(v, max) : undefined);
71
+ const idOf = (v) => int(v) ?? str(v, 64); // a PowerPoint slide or shape id: a number, or a string from older pages
72
+ const blockPath = (v) => (Array.isArray(v) && v.length > 0 && v.length <= PATH_MAX && v.every((x) => Number.isSafeInteger(x) && x >= 0) ? [...v] : undefined);
73
+ const put = (out, k, v) => {
74
+ if (v !== undefined) out[k] = v;
75
+ };
76
+
77
+ /**
78
+ * Where a comment, a pin or a typed edit sits, as the engine reports it (vendor/engine: docx Sz and the edit records, xlsx un, pptx AV):
79
+ * Word {blockPath, paraId, start, end, protected}, Excel {sheetIndex, sheetName, range}, PowerPoint {slideIndex, slideId, shapeId,
80
+ * shapeName, protected, protectedReason, cell: {row, col}}. Null for anything that is not an object.
81
+ */
82
+ export function cleanLocation(l) {
83
+ if (!l || typeof l !== "object" || Array.isArray(l)) return null;
84
+ const out = {};
85
+ put(out, "blockPath", blockPath(l.blockPath));
86
+ put(out, "paraId", str(l.paraId, 64));
87
+ put(out, "start", int(l.start));
88
+ put(out, "end", int(l.end));
89
+ put(out, "protected", typeof l.protected === "boolean" ? l.protected : str(l.protected, 200));
90
+ put(out, "protectedReason", str(l.protectedReason, 200));
91
+ put(out, "sheetIndex", int(l.sheetIndex));
92
+ put(out, "sheetName", str(l.sheetName, 256)); // Excel caps a sheet name at 31 characters
93
+ put(out, "range", str(l.range, 64)); // "XFD1048576:XFD1048576" is 21
94
+ put(out, "slideIndex", int(l.slideIndex));
95
+ put(out, "slideId", idOf(l.slideId));
96
+ put(out, "shapeId", idOf(l.shapeId));
97
+ put(out, "shapeName", str(l.shapeName, NAME_MAX));
98
+ if (l.cell && typeof l.cell === "object" && int(l.cell.row) !== undefined && int(l.cell.col) !== undefined) out.cell = { row: l.cell.row, col: l.cell.col };
99
+ // HTML (web/src/html-view.js): the located node's id in one version of the file, and that version's sha256 (an id means nothing in another)
100
+ put(out, "nodeId", int(l.nodeId));
101
+ if (typeof l.version === "string" && /^[0-9a-f]{64}$/.test(l.version)) out.version = l.version;
102
+ return out;
103
+ }
104
+
105
+ /**
106
+ * The page's view at Send (the engine's getViewState): Word {blockPath, textHash, quote, offsetPx, zoom, fraction}, Excel {sheetIndex,
107
+ * sheetName, topLeft, selection}, PowerPoint {slideIndex, zoom}, each with its `format`. Null for anything else.
108
+ */
109
+ export function cleanView(v) {
110
+ if (!v || typeof v !== "object" || Array.isArray(v) || !["docx", "xlsx", "pptx", "html"].includes(v.format)) return null;
111
+ if (v.format === "html") return cleanHtmlView(v);
112
+ const out = { format: v.format };
113
+ put(out, "blockPath", blockPath(v.blockPath));
114
+ put(out, "textHash", str(v.textHash, 64));
115
+ put(out, "quote", str(v.quote, 200)); // the engine sends 80
116
+ put(out, "offsetPx", finite(v.offsetPx));
117
+ put(out, "zoom", finite(v.zoom) ?? str(v.zoom, 16)); // a number or "fit"
118
+ put(out, "fraction", finite(v.fraction));
119
+ put(out, "sheetIndex", int(v.sheetIndex));
120
+ put(out, "sheetName", str(v.sheetName, 256));
121
+ put(out, "topLeft", str(v.topLeft, 64));
122
+ put(out, "selection", str(v.selection, 64));
123
+ put(out, "slideIndex", int(v.slideIndex));
124
+ return out;
125
+ }
126
+
127
+ /** An HTML view (web/src/html-frame.js getViewState): {version, x, y, fraction, top: {nodeId, quote, offsetPx}}; hidden elements stay in the page. */
128
+ function cleanHtmlView(v) {
129
+ const out = { format: "html" };
130
+ if (typeof v.version === "string" && /^[0-9a-f]{64}$/.test(v.version)) out.version = v.version;
131
+ put(out, "x", finite(v.x));
132
+ put(out, "y", finite(v.y));
133
+ put(out, "fraction", finite(v.fraction));
134
+ const t = v.top;
135
+ if (t && typeof t === "object" && int(t.nodeId) !== undefined) out.top = { nodeId: t.nodeId, quote: str(t.quote, 200) ?? "", offsetPx: finite(t.offsetPx) ?? 0 };
136
+ return out;
137
+ }
138
+
139
+ /**
140
+ * The page's anchor of a comment, as stored: the fields the server and the page read (location, quote, prefix, suffix, paraText,
141
+ * format, hitKind), each cut. A quote or paragraph text cut here is marked `cut`, and the anchor is then built from the file as the
142
+ * paragraph or shape it sits in (src/anchors.js), never as a range of the cut text. Null for anything that is not an object.
143
+ */
144
+ export function cleanPageAnchor(a) {
145
+ if (!a || typeof a !== "object" || Array.isArray(a)) return null;
146
+ const out = {};
147
+ const loc = cleanLocation(a.location);
148
+ if (loc) out.location = loc;
149
+ let cut = false;
150
+ for (const k of ["quote", "paraText"]) {
151
+ if (typeof a[k] !== "string") continue;
152
+ out[k] = cutText(a[k], STRING_MAX);
153
+ if (out[k].length < a[k].length) cut = true;
154
+ }
155
+ put(out, "prefix", typeof a.prefix === "string" ? cutTail(a.prefix, CONTEXT_MAX) : undefined); // the end nearest the selection
156
+ put(out, "suffix", str(a.suffix, CONTEXT_MAX));
157
+ for (const k of ["format", "hitKind"]) put(out, k, str(a[k], 40));
158
+ if (cut || a.cut === true) out.cut = true;
159
+ return out;
160
+ }
161
+
162
+ /** A typed edit the page reported ({kind, before, after, label, location}), as stored for the next batch. */
163
+ export function cleanEdit(e) {
164
+ const side = (v) => (typeof v === "string" ? cutText(v, STRING_MAX) : v == null ? v : typeof v === "number" || typeof v === "boolean" ? cutText(String(v), 100) : null);
165
+ const before = side(e?.before);
166
+ const after = side(e?.after);
167
+ const out = { kind: str(e?.kind, 40) ?? null, before, after, label: str(e?.label, NAME_MAX) ?? null, location: cleanLocation(e?.location) };
168
+ if ((typeof e?.before === "string" && before.length < e.before.length) || (typeof e?.after === "string" && after.length < e.after.length)) out.cut = true;
169
+ return out;
170
+ }
171
+
172
+ // in UTF-8 bytes, as the file is written (a CJK character is 3)
173
+ export const jsonSize = (v) => {
174
+ try {
175
+ return Buffer.byteLength(JSON.stringify(v) ?? "");
176
+ } catch {
177
+ return Infinity;
178
+ }
179
+ };
180
+ const OPTIONAL = ["values", "types", "formats", "names"];
181
+ /**
182
+ * An anchor built from the file (src/anchors.js buildDurable, src/session.js reanchor), as stored and handed on: every string at
183
+ * STRING_MAX; past ANCHOR_MAX as JSON, an Excel anchor's values cut to VALUE_SHORT characters a cell, then the optional evidence dropped.
184
+ * Hashes, places and the kept text prefix stay. Marked `cut` when anything was cut. Returns the same object when nothing was.
185
+ */
186
+ export function capDurable(d) {
187
+ if (!d || typeof d !== "object") return d;
188
+ let out = d;
189
+ const own = () => (out === d ? (out = { ...d }) : out);
190
+ for (const [k, v] of Object.entries(d)) if (typeof v === "string" && v.length > STRING_MAX) own()[k] = cutText(v, STRING_MAX);
191
+ if (jsonSize(out) > ANCHOR_MAX) {
192
+ own();
193
+ if (Array.isArray(out.values)) out.values = out.values.map((r) => (Array.isArray(r) ? r.map((x) => (typeof x === "string" ? cutText(x, VALUE_SHORT) : x)) : r));
194
+ if (jsonSize(out) > ANCHOR_MAX) {
195
+ for (const k of OPTIONAL) if (Array.isArray(out[k])) out[k] = [];
196
+ if (out.ctx && typeof out.ctx === "object") out.ctx = { ...out.ctx, rowCells: [] };
197
+ for (const [k, v] of Object.entries(out)) {
198
+ if (typeof v !== "string" || v.length <= 2000 || /hash/i.test(k)) continue;
199
+ // a quote is a range of the text: cut, it would point at a shorter one, so the anchor becomes its whole paragraph or shape
200
+ if (k === "quote") Object.assign(out, { quote: "", start: null, end: null, ...("nth" in out ? { nth: null } : {}) });
201
+ else out[k] = cutText(v, 2000);
202
+ }
203
+ }
204
+ }
205
+ if (d.pinFallback) {
206
+ const p = capDurable(d.pinFallback);
207
+ if (p !== d.pinFallback) own().pinFallback = p;
208
+ }
209
+ if (out !== d) out.cut = true;
210
+ return out;
211
+ }
212
+
213
+ /** A comment as stored, past COMMENT_MAX: the page's anchor is cut to 2,000 characters a field (the anchor built from the file holds the rest). */
214
+ export function capComment(c) {
215
+ if (jsonSize(c) <= COMMENT_MAX) return c;
216
+ const out = { ...c, cut: true };
217
+ if (out.anchor) out.anchor = { ...out.anchor, cut: true, ...Object.fromEntries(["quote", "paraText"].filter((k) => typeof out.anchor[k] === "string").map((k) => [k, cutText(out.anchor[k], 2000)])) };
218
+ if (out.durable) out.durable = capDurable(out.durable);
219
+ return out;
220
+ }
221
+
222
+ /** The size of a request's own data (comments, note, edits, view state) as JSON; Infinity when it cannot be written as JSON. */
223
+ export const requestSize = (parts) => jsonSize(parts);
package/src/review.js CHANGED
@@ -1,9 +1,12 @@
1
1
  import crypto from "node:crypto";
2
- import { readUnits } from "./anchors.js";
2
+ import { readUnits, unitsFor } from "./anchors.js";
3
3
  import { findAll } from "./ops/runs.js";
4
4
  import { untrusted } from "./sanitize.js";
5
5
  import { exactHash, normText } from "./text.js";
6
- import { a1, a1Range } from "./xlsx.js";
6
+ import { parseA1 } from "./ops/common.js";
7
+ import { a1, a1Range, cellAt, mergeAt } from "./xlsx.js";
8
+ import { unknownFormat } from "./constants.js";
9
+ import { htmlRawUnits } from "./html.js";
7
10
 
8
11
  // Idea 21: the agent reviews first. The person taps one of three fixed requests; the batch carries that fixed sentence in
9
12
  // `instructions.review` and a read-only outline of the file (`untrusted_document_evidence.units[]`, built here by code). The agent answers
@@ -50,11 +53,12 @@ const isObj = (v) => v && typeof v === "object" && !Array.isArray(v);
50
53
  * {units: [{id, label, text, protected}] (at most UNITS_MAX, for the batch), more, index: {id -> where the unit is and its text hash}} (the
51
54
  * index stays on the server). A unit an edit cannot address (a DOCX paragraph with no body path, a PPTX shape whose id is shared on its
52
55
  * slide) is marked protected: suggestions on it are refused.
53
- * Ceiling: every paragraph, shape or cell is read and sorted on the server's main thread before the cut to UNITS_MAX, so a very large
54
- * workbook pauses other requests while a review is built. Upgrade path: stop reading at UNITS_MAX + a count, or build in a worker.
56
+ * `reading` (optional) is readUnits of `bytes` that the caller already has (src/anchors.js unitsFor).
57
+ * Ceiling: every paragraph, shape or cell is read and sorted on the server's main thread before the cut to UNITS_MAX; the readers' caps
58
+ * (src/docx.js, src/xlsx.js, src/pptx.js) bound that to a few hundred milliseconds. Upgrade path: build in a worker.
55
59
  */
56
- export function unitsOf(format, bytes) {
57
- const raw = rawUnits(format, bytes);
60
+ export function unitsOf(format, bytes, reading = null) {
61
+ const raw = rawUnits(format, bytes, reading);
58
62
  const units = [];
59
63
  const index = {};
60
64
  raw.slice(0, UNITS_MAX).forEach((x, i) => {
@@ -67,9 +71,9 @@ export function unitsOf(format, bytes) {
67
71
  }
68
72
 
69
73
  /** Every unit of the file in document order: {label, text (the full text, not cleaned), protected, at (where it is and its text hash)}. */
70
- function rawUnits(format, bytes) {
74
+ function rawUnits(format, bytes, reading = null) {
71
75
  const raw = [];
72
- const u = readUnits(format, bytes);
76
+ const u = reading || readUnits(format, bytes);
73
77
  if (format === "docx") {
74
78
  for (const p of u.paragraphs) {
75
79
  if (!normText(p.text)) continue;
@@ -93,11 +97,13 @@ function rawUnits(format, bytes) {
93
97
  const sd = u.wb.sheetCells(sheet);
94
98
  for (const c of cells) {
95
99
  const text = c.k === "formula" && c.cached ? `${c.text} (shows ${c.cached})` : c.text;
96
- const m = sd.merges.find((q) => q.r0 === c.r && q.c0 === c.c);
100
+ const m = mergeAt(sd, c.r, c.c);
97
101
  raw.push({ label: `${sheet.name}!${a1(c.r, c.c)}`, text, protected: false, ...(m ? { merged: a1Range(m) } : {}), at: { sheet: sheet.name, range: a1(c.r, c.c), text_hash: exactHash(c.text) } });
98
102
  }
99
103
  }
100
- } else throw new Error(`unknown format ${format}`);
104
+ } else if (format === "html") {
105
+ for (const x of htmlRawUnits(u.read)) raw.push(x); // src/html.js: elements that hold text and leaf elements, tag names and ids per R7
106
+ } else unknownFormat(format, "rawUnits");
101
107
  return raw;
102
108
  }
103
109
 
@@ -112,8 +118,8 @@ function rawUnits(format, bytes) {
112
118
  * Every entry, as JSON, stays under OUTLINE_LINE_MAX characters (its text is cut further when quotes or backslashes double in JSON), so
113
119
  * the outline file, one entry per line, can be paged with Claude Code's Read tool (which cuts lines at 2,000 characters).
114
120
  */
115
- export function outlineOf(format, bytes, max = UNITS_MAX) {
116
- const raw = rawUnits(format, bytes);
121
+ export function outlineOf(format, bytes, max = UNITS_MAX, reading = null) {
122
+ const raw = rawUnits(format, bytes, reading);
117
123
  const outline = raw.slice(0, max).map((x, i) => {
118
124
  const target = x.protected ? null : targetOf(x.at, format);
119
125
  const entry = (n) => {
@@ -183,9 +189,11 @@ function textNow(units, at) {
183
189
  const sh = units.deck.slides.find((s) => String(s.id) === at.slide_id)?.shapes.filter((x) => x.id === at.shape_id);
184
190
  return sh?.length === 1 && sh[0].hash === at.text_hash ? sh[0].text : null;
185
191
  }
192
+ if (units.format !== "xlsx") return unknownFormat(units.format, "textNow");
186
193
  const sheet = units.wb.sheets.find((s) => s.name === at.sheet);
187
194
  if (!sheet) return null;
188
- const c = [...units.wb.sheetCells(sheet).cells.values()].find((x) => a1(x.r, x.c) === at.range);
195
+ const q = parseA1(String(at.range || ""));
196
+ const c = q && q.r0 === q.r1 && q.c0 === q.c1 ? cellAt(units.wb.sheetCells(sheet), q.r0, q.c0) : null;
189
197
  return c && exactHash(c.text) === at.text_hash ? c.text : null;
190
198
  }
191
199
 
@@ -227,34 +235,40 @@ export function cardFor(item, units, index, { reviewId, format }) {
227
235
  function placeOf(at, format) {
228
236
  if (format === "docx") return { format, ordinal: at.ordinal, para_id: at.para_id, body_path: at.body_path };
229
237
  if (format === "pptx") return { format, slide_id: at.slide_id, slide_index: at.slide_index, shape_id: at.shape_id };
230
- return { format, sheet: at.sheet, range: at.range };
238
+ if (format === "xlsx") return { format, sheet: at.sheet, range: at.range };
239
+ return unknownFormat(format, "review placeOf");
231
240
  }
232
241
 
233
242
  /** The op Accept runs: one find-and-replace in one paragraph or shape, with the unit's text hash as `expect`. */
234
243
  export function opFor(at, format, id, { find, replace }) {
235
244
  if (format === "docx") return { op: "replace_text", id, target: targetOf(at, format), expect: { text_hash: at.text_hash }, find, replace };
236
245
  if (format === "pptx") return { op: "replace_text_in_shape", id, target: targetOf(at, format), expect: { text_hash: at.text_hash }, find, replace };
237
- return null;
246
+ if (format === "xlsx") return null; // a spreadsheet takes notes only (cleanSuggestion)
247
+ return unknownFormat(format, "opFor");
238
248
  }
239
249
 
240
250
  /** The explicit op target of a unit (ops.schema.json docx_target, pptx_target, xlsx_range_target). */
241
251
  function targetOf(at, format) {
242
252
  if (format === "docx") return at.para_id ? { para_id: at.para_id, body_path: at.body_path, text_hash: at.text_hash } : { body_path: at.body_path, text_hash: at.text_hash };
243
253
  if (format === "pptx") return { slide_id: at.slide_id, shape_id: at.shape_id };
244
- return { sheet: at.sheet, range: at.range };
254
+ if (format === "xlsx") return { sheet: at.sheet, range: at.range };
255
+ // HTML: exactly one form, as the op schema takes it (src/ops/schema.js): the element's id when it is unique (it survives elements
256
+ // added before it), else its path; the path is in the outline's `at` either way
257
+ if (format === "html") return at.element_id ? { element_id: at.element_id } : { path: at.path };
258
+ return unknownFormat(format, "targetOf");
245
259
  }
246
260
 
247
261
  /**
248
262
  * The agent's suggestions -> {cards, dropped: [{index, reason}]}. More than SUGGESTIONS_MAX stores none (refused, never the first 20).
249
263
  * `bytes` is the file now; `units` the batch's outline; `index` the server's record.
250
264
  */
251
- export function cardsFrom(list, { bytes, format, units, index, reviewId }) {
265
+ export function cardsFrom(list, { bytes, sha = null, format, units, index, reviewId }) {
252
266
  if (list === undefined || list === null) return { cards: [], dropped: [] };
253
267
  if (!Array.isArray(list)) return { cards: [], dropped: [{ index: null, reason: "suggestions must be a list" }] };
254
268
  if (list.length > SUGGESTIONS_MAX) return { cards: [], dropped: list.map((_, i) => ({ index: i, reason: `more than ${SUGGESTIONS_MAX} suggestions` })) };
255
269
  let now;
256
270
  try {
257
- now = readUnits(format, bytes);
271
+ now = sha ? unitsFor(format, bytes, sha) : readUnits(format, bytes); // `sha`: the bytes' own hash, to reuse the last reading of them
258
272
  } catch {
259
273
  return { cards: [], dropped: list.map((_, i) => ({ index: i, reason: "the file could not be read" })) };
260
274
  }
@@ -176,7 +176,7 @@ function expand(s) {
176
176
  function tokens(s) {
177
177
  const out = [];
178
178
  // a number keeps a sign written right before it ("-5", "+3", "~240"), but not the dash of a range ("3-5")
179
- const re = /((?:(?<![\p{L}\p{N}])[-+~<>\u2212\u2248\u00b1])?[$€£¥₹]?\p{N}[\p{N},.:/%]*(?:\s?%)?)|[\p{L}][\p{L}\p{M}'-]*/gu;
179
+ const re = /((?:(?<![\p{L}\p{N}])[-+~<>\u2212\u2248\u00b1])?[$€£¥₹]?\p{N}[\p{N},.:/%]*(?:\s?%)?)|[\p{L}][\p{L}\p{M}'-]{0,4095}/gu; // a word at most 4,096 characters (a run of 7.9 million marks overflowed the regex stack)
180
180
  for (const m of expand(s).matchAll(re)) {
181
181
  if (m[1] !== undefined) {
182
182
  const kept = m[0].replace(/[.,:/]+$/, "");
@@ -661,7 +661,7 @@ const factText = (s) =>
661
661
  .replace(/(\p{N})\s*(?:percent|per cent)\b/giu, "$1%")
662
662
  .replace(/(\p{N})\s*([ap])\.?\s?m\.?(?![\p{L}\p{N}])/giu, (_, n, x) => `${n} ${x.toLowerCase()}m`);
663
663
 
664
- const APOS_WORD = /\p{L}[\p{L}\p{M}']*/gu;
664
+ const APOS_WORD = /\p{L}[\p{L}\p{M}']{0,4095}/gu; // bounded, as in tokens() above
665
665
  /**
666
666
  * Words that differ only by an apostrophe are written alike on both sides, with the apostrophe ("Its" and "It's" both "It's"), but only
667
667
  * a word that one side has more often than the other (a word in both is no misspelling of the other's word), and never a pair whose
package/src/rewriter.js CHANGED
@@ -71,18 +71,27 @@ export const APPLE_STATES = {
71
71
  /** The probe's words (scripts/rewriter/apple/main.swift --probe) to the state codes above. */
72
72
  export const PROBE_REASONS = { deviceNotEligible: "device_not_eligible", appleIntelligenceNotEnabled: "apple_intelligence_not_enabled", modelNotReady: "model_not_ready" };
73
73
 
74
+ const shaCache = new Map(); // `${bin}\0${mtimeMs}\0${size}\0${ino}` -> sha256
75
+ const quarantineCache = new Map(); // `${bin}\0${mtimeMs}\0${ctimeMs}\0${size}\0${ino}` -> boolean
76
+ const cacheSet = (m, k, v) => {
77
+ if (m.size >= 32) m.clear();
78
+ m.set(k, v);
79
+ return v;
80
+ };
81
+
74
82
  /**
75
83
  * Does the file carry the com.apple.quarantine mark? Asked of /usr/bin/xattr, never by running the file. true | false; anything but a clean
76
84
  * "No such xattr" answer counts as marked (fail closed).
77
85
  */
78
86
  export function isQuarantined(file) {
79
- const r = spawnSync(XATTR, ["-p", QUARANTINE, file], { encoding: "utf8", timeout: 5000 });
87
+ const r = spawnSync(XATTR, ["-p", QUARANTINE, file], { encoding: "utf8", timeout: 5000, env: { ...process.env, LC_ALL: "C" } });
80
88
  if (r.status === 0) return true;
81
89
  return !(r.status === 1 && /No such xattr/.test(r.stderr || ""));
82
90
  }
83
91
  /** Removes the mark from exactly this file. true when the file carries no mark afterwards. */
84
92
  export function clearQuarantine(file) {
85
- spawnSync(XATTR, ["-d", QUARANTINE, file], { encoding: "utf8", timeout: 5000 });
93
+ spawnSync(XATTR, ["-d", QUARANTINE, file], { encoding: "utf8", timeout: 5000, env: { ...process.env, LC_ALL: "C" } });
94
+ for (const k of quarantineCache.keys()) if (k.startsWith(`${file}\0`)) quarantineCache.delete(k);
86
95
  return !isQuarantined(file);
87
96
  }
88
97
 
@@ -112,20 +121,25 @@ export function hostFacts() {
112
121
  export function appleHelperState(o = {}) {
113
122
  const dir = o.dir ?? (env("APPLE_HELPER_DIR", "") || APPLE_HELPER_DIR); // the folder variable is for the browser tests, with a stand-in helper
114
123
  const host = o.host ?? hostFacts();
115
- const marked = o.quarantined ?? isQuarantined;
116
124
  const major = Number(String(host.macos ?? "").split(".")[0]);
117
125
  if (!Number.isFinite(major) || major < 26) return { ok: false, reason: "macos_too_old" };
118
126
  if (!host.appleSilicon) return { ok: false, reason: "device_not_eligible" };
119
127
  const bin = path.join(dir, "onionskin-fm");
120
128
  let rec;
129
+ let st;
121
130
  try {
122
131
  rec = JSON.parse(fs.readFileSync(path.join(dir, "onionskin-fm.json"), "utf8"));
123
- if (!fs.lstatSync(bin).isFile()) return { ok: false, reason: "helper_missing" };
132
+ st = fs.lstatSync(bin);
133
+ if (!st.isFile()) return { ok: false, reason: "helper_missing" };
124
134
  } catch {
125
135
  return { ok: false, reason: "helper_missing" };
126
136
  }
127
- if (typeof rec?.sha256 !== "string" || fileSha256(bin) !== rec.sha256) return { ok: false, reason: "helper_changed" };
128
- if (marked(bin)) return { ok: false, reason: "quarantined", bin };
137
+ const shaKey = `${bin}\0${st.mtimeMs}\0${st.size}\0${st.ino}`;
138
+ const sha = shaCache.get(shaKey) ?? cacheSet(shaCache, shaKey, fileSha256(bin));
139
+ if (typeof rec?.sha256 !== "string" || sha !== rec.sha256) return { ok: false, reason: "helper_changed" };
140
+ const qKey = `${bin}\0${st.mtimeMs}\0${st.ctimeMs}\0${st.size}\0${st.ino}`;
141
+ const isMarked = o.quarantined ? o.quarantined(bin) : (quarantineCache.get(qKey) ?? cacheSet(quarantineCache, qKey, isQuarantined(bin)));
142
+ if (isMarked) return { ok: false, reason: "quarantined", bin };
129
143
  return { ok: true, bin, sha256: rec.sha256 };
130
144
  }
131
145
  /** The person's words for a reason code, or null. helper_changed reads as helper_missing. */
@@ -175,7 +189,7 @@ export function workerEnv() {
175
189
  * Options: idleMs (default 15 minutes), startMs, stuckMs, retryMs.
176
190
  */
177
191
  export function createWorker(launch, opts = {}) {
178
- const o = { idleMs: idleMs(), startMs: START_LIMIT_MS, stuckMs: STUCK_MS, retryMs: RETRY_AFTER_FAILURE_MS, ...opts };
192
+ const o = { idleMs: idleMs(), startMs: START_LIMIT_MS, stuckMs: STUCK_MS, retryMs: RETRY_AFTER_FAILURE_MS, killGraceMs: 500, ...opts };
179
193
  let child = null;
180
194
  let ready = false;
181
195
  let failedAt = 0;
@@ -197,7 +211,23 @@ export function createWorker(launch, opts = {}) {
197
211
  try {
198
212
  c.stdin.end();
199
213
  } catch {}
200
- c.kill("SIGTERM");
214
+ if (c.exitCode === null && c.signalCode === null) {
215
+ try {
216
+ c.kill("SIGTERM");
217
+ } catch {}
218
+ const forceKill = () => {
219
+ try {
220
+ if (c.exitCode === null && c.signalCode === null) c.kill("SIGKILL");
221
+ } catch {}
222
+ };
223
+ const killTimer = setTimeout(forceKill, o.killGraceMs);
224
+ killTimer.unref?.();
225
+ process.once("exit", forceKill);
226
+ c.once("exit", () => {
227
+ clearTimeout(killTimer);
228
+ process.off("exit", forceKill);
229
+ });
230
+ }
201
231
  }
202
232
  if (busy) {
203
233
  busy.settle({ ok: false, code: "worker_exited" });
package/src/sanitize.js CHANGED
@@ -1,5 +1,6 @@
1
1
  // Built with new RegExp from escaped strings so no raw separator characters live in this source file.
2
2
  // C0 (\n \t \r become spaces first), DEL, C1, bidi controls (LRM/RLM/ALM, embeddings, overrides, isolates), zero-width and BOM.
3
+ import { nfc } from "./nfc.js";
3
4
  const STRIP = new RegExp("[\\u0000-\\u001F\\u007F-\\u009F\\u061C\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u2069\\uFEFF]", "g");
4
5
  const WS = new RegExp("[\\t\\n\\r\\u2028\\u2029]+", "g");
5
6
  // the user's own words keep newlines (\u000A) and tabs; everything else control/bidi is dropped
@@ -8,21 +9,33 @@ const MAX_UNTRUSTED = 500;
8
9
  const MARK = "…[truncated]";
9
10
 
10
11
  /** Untrusted document-derived text: strip control and bidi characters, NFC, collapse whitespace, cap at 500 chars including the marker. */
12
+ // Of a longer value only the first READ_PER_KEPT x max characters are cleaned: what is kept is at most `max` characters, and cleaning all
13
+ // of a paragraph of 7.9 million characters, for each of 50 comments quoting it, held the server's thread 2.5 s at Send.
14
+ const READ_PER_KEPT = 16;
11
15
  export function untrusted(value, max = MAX_UNTRUSTED) {
12
16
  if (value == null) return "";
13
- let s = String(value).replace(WS, " ").replace(STRIP, "").normalize("NFC");
17
+ let raw = String(value);
18
+ const cut = raw.length > READ_PER_KEPT * max;
19
+ if (cut) raw = raw.slice(0, READ_PER_KEPT * max);
20
+ let s = nfc(raw.replace(WS, " ").replace(STRIP, ""));
14
21
  s = s.replace(/ {2,}/g, " ").trim();
15
22
  const cps = [...s];
16
23
  if (cps.length > max) s = cps.slice(0, max - [...MARK].length).join("") + MARK;
24
+ else if (cut) s = cps.slice(0, Math.max(0, max - [...MARK].length)).join("") + MARK; // cut before it was cleaned: still says so
17
25
  return s;
18
26
  }
19
27
 
20
28
  /** The user's own words: keep newlines, strip other control and bidi characters, cap generously. */
29
+ // Of a longer value only the first READ_PER_KEPT x max characters are read, as in untrusted(): 50 comments of 7.9 million characters
30
+ // each were normalized whole at every draft save.
21
31
  export function userText(value, max = 5000) {
22
32
  if (value == null) return "";
23
- const s = String(value).replace(/\r\n?/g, "\n").replace(USER_STRIP, "").normalize("NFC");
33
+ let raw = String(value);
34
+ const cut = raw.length > READ_PER_KEPT * max;
35
+ if (cut) raw = raw.slice(0, READ_PER_KEPT * max);
36
+ const s = nfc(raw.replace(/\r\n?/g, "\n").replace(USER_STRIP, ""));
24
37
  const cps = [...s];
25
- return cps.length > max ? cps.slice(0, max).join("") + MARK : s;
38
+ return cps.length > max ? cps.slice(0, max).join("") + MARK : cut ? s + MARK : s;
26
39
  }
27
40
 
28
41
  const FLAG_KEEP = new Set(["code", "severity"]);