onionskin-cli 0.0.3 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +198 -0
- package/INSTALL.md +5 -1
- package/NOTICE +34 -0
- package/README.md +102 -34
- package/docs/contracts/batch.schema.json +27 -4
- package/docs/contracts/ops.schema.json +117 -5
- package/package.json +4 -2
- package/scripts/checker/check.py +24 -5
- package/scripts/checker/requirements.txt +482 -5
- package/scripts/requirements.txt +482 -5
- package/scripts/rewriter/apple/main.swift +1 -1
- package/scripts/rewriter/apple/onionskin-fm +0 -0
- package/scripts/rewriter/apple/onionskin-fm.json +1 -1
- package/skills/onionskin/SKILL.md +12 -9
- package/skills/onionskin/reference/batch.md +5 -3
- package/skills/onionskin/reference/formats.md +5 -1
- package/skills/onionskin/reference/ops.md +48 -4
- package/skills/onionskin/reference/troubleshooting.md +31 -0
- package/src/agent-detect.js +28 -2
- package/src/agent-prompt.js +12 -1
- package/src/agent-runner.js +110 -20
- package/src/anchor-work.js +154 -0
- package/src/anchor-worker.js +38 -0
- package/src/anchors.js +61 -11
- package/src/apply.js +119 -47
- package/src/batch.js +32 -8
- package/src/check-worker.js +8 -5
- package/src/checker.js +238 -33
- package/src/cli.js +248 -42
- package/src/constants.js +34 -1
- package/src/diff.js +3 -1
- package/src/doctor.js +12 -3
- package/src/docx.js +464 -87
- package/src/edit-hash.js +19 -0
- package/src/fsutil.js +14 -6
- package/src/history-worker.js +8 -8
- package/src/history.js +144 -31
- package/src/html-active.js +384 -0
- package/src/html-envelope.js +51 -0
- package/src/html-parser.js +8 -0
- package/src/html-render-worker.js +25 -0
- package/src/html-render.js +939 -0
- package/src/html-utf8.js +65 -0
- package/src/html-write-worker.js +14 -0
- package/src/html-write.js +380 -0
- package/src/html.js +1259 -0
- package/src/local.js +68 -4
- package/src/merge-package.js +1166 -0
- package/src/merge-pptx.js +98 -38
- package/src/merge-proof.js +1019 -0
- package/src/merge-worker.js +60 -0
- package/src/merge-xlsx.js +84 -29
- package/src/merge.js +286 -83
- package/src/nfc.js +46 -0
- package/src/notes.js +122 -0
- package/src/offload.js +419 -1
- package/src/open-worker.js +27 -0
- package/src/ops/common.js +2 -1
- package/src/ops/docx-table.js +5 -3
- package/src/ops/docx.js +58 -29
- package/src/ops/html.js +394 -0
- package/src/ops/pptx.js +82 -23
- package/src/ops/run.js +23 -3
- package/src/ops/runs.js +215 -13
- package/src/ops/schema.js +8 -1
- package/src/ops/xlsx.js +121 -10
- package/src/ops-worker.js +17 -0
- package/src/package.js +43 -4
- package/src/person.js +2 -1
- package/src/pptx.js +323 -23
- package/src/question.js +10 -3
- package/src/quick-plan.js +13 -6
- package/src/quick.js +1 -1
- package/src/redline-check.js +12 -2
- package/src/redline-worker.js +10 -14
- package/src/reply.js +46 -19
- package/src/retain.js +223 -0
- package/src/review.js +32 -18
- package/src/rewrite-check.js +2 -2
- package/src/rewriter.js +38 -8
- package/src/sanitize.js +16 -3
- package/src/server-entry.js +4 -3
- package/src/server.js +175 -62
- package/src/session.js +1151 -210
- package/src/setup.js +20 -2
- package/src/stop-gate.js +34 -0
- package/src/suggest-reply.js +42 -7
- package/src/suggest.js +14 -29
- package/src/text.js +161 -11
- package/src/update.js +517 -0
- package/src/voice-guide.js +15 -9
- package/src/voice-learn.js +15 -4
- package/src/voice-sample-worker.js +7 -6
- package/src/voice-store.js +11 -4
- package/src/voice-worker.js +4 -6
- package/src/voice.js +2 -1
- package/src/worker-host.js +23 -0
- package/src/xlsx.js +378 -97
- package/src/xml-tree.js +6 -3
- package/src/xml.js +82 -34
- package/src/zip-write.js +10 -5
- package/src/zip.js +73 -18
- package/vendor/parse5/LICENSE.txt +32 -0
- package/vendor/parse5/parse5.mjs +7822 -0
- package/vendor/parse5.lock.json +23 -0
- package/web/dist/assets/{index-DS-WST1E.js → index-CjsTf5gj.js} +58 -52
- package/web/dist/assets/index-DnbLtHhG.css +1 -0
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/reply.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import {
|
|
4
|
-
import { ApplyError, APPLY_EXIT, applyCommand } from "./apply.js";
|
|
5
|
-
import { parseTicket } from "./lease.js";
|
|
6
|
-
import { isAgentHolder } from "./agent-detect.js";
|
|
3
|
+
import { PRODUCT, unknownFormat } from "./constants.js";
|
|
4
|
+
import { ApplyError, APPLY_EXIT, applyCommand, formatOfPath } from "./apply.js";
|
|
5
|
+
import { loadTicketRecord, parseTicket } from "./lease.js";
|
|
6
|
+
import { NOT_YOUR_FILE, agentMayUse, isAgentHolder } from "./agent-detect.js";
|
|
7
7
|
import { QUESTION_FIX, QUESTIONS_PER_BATCH, cleanQuestion } from "./question.js";
|
|
8
8
|
import { SUGGESTIONS_MAX, SUGGEST_FIX, cleanSuggestion } from "./review.js";
|
|
9
9
|
import { VOICE_FIX, cleanVoiceDraft } from "./voice-guide.js";
|
|
@@ -186,13 +186,16 @@ export function planReplies(batch, input, { partial = false } = {}) {
|
|
|
186
186
|
if (mode === "find") {
|
|
187
187
|
if (typeof r.find !== "string" || !r.find || typeof r.replace !== "string") return refuse("bad_reply", "find needs a non-empty string and replace needs a string");
|
|
188
188
|
const extras = Object.fromEntries(SHORTHAND_EXTRAS.filter((k) => k in r).map((k) => [k, r[k]]));
|
|
189
|
-
if (format === "docx") op = { op: "replace_text", ...base, find: r.find, replace: r.replace, ...extras };
|
|
189
|
+
if (format === "docx" || format === "html") op = { op: "replace_text", ...base, find: r.find, replace: r.replace, ...extras };
|
|
190
190
|
else if (format === "pptx") op = { op: "replace_text_in_shape", ...base, find: r.find, replace: r.replace, ...extras };
|
|
191
|
+
else if (format !== "xlsx") return unknownFormat(format, "reply find");
|
|
191
192
|
else return refuse("shorthand_not_for_format", "find/replace does not apply to a .xlsx cell", "use `set` for a cell or range");
|
|
192
193
|
} else if (mode === "text") {
|
|
193
194
|
if (typeof r.text !== "string") return refuse("bad_reply", "text must be a string");
|
|
194
195
|
if (format === "docx") op = { op: "replace_text", ...base, text: r.text };
|
|
196
|
+
else if (format === "html") op = { op: "set_text", ...base, text: r.text };
|
|
195
197
|
else if (format === "pptx") op = { op: "set_shape_text", ...base, paragraphs: r.text.split("\n") };
|
|
198
|
+
else if (format !== "xlsx") return unknownFormat(format, "reply text");
|
|
196
199
|
else return refuse("shorthand_not_for_format", "text does not apply to a .xlsx cell", "use `set` for a cell or range");
|
|
197
200
|
} else if (mode === "set") {
|
|
198
201
|
if (format !== "xlsx") return refuse("shorthand_not_for_format", `set applies to a .xlsx cell or range, not a .${format}`, "use `find`+`replace` or `text`");
|
|
@@ -264,7 +267,7 @@ const FIX = {
|
|
|
264
267
|
skipped_protected: "Protected place (field, equation, content control or similar): it cannot be rewritten losslessly. Decline this comment with a note.",
|
|
265
268
|
};
|
|
266
269
|
const fixFor = (r) => FIX[r.status] || (r.status === "refused" ? `Refused by the engine${r.code ? ` (${r.code})` : ""}. Change the op as the reason says, or decline.` : undefined);
|
|
267
|
-
const brief = (comment, r) => ({ comment, op: r.op, status: r.status, ...(r.code ? { code: r.code } : {}), ...(r.reason ? { reason: r.reason } : {}), ...(r.actual_excerpt ? { actual_excerpt: r.actual_excerpt } : {}), ...(fixFor(r) ? { fix: fixFor(r) } : {}) });
|
|
270
|
+
const brief = (comment, r) => ({ comment, op: r.op, status: r.status, ...(r.code ? { code: r.code } : {}), ...(r.reason ? { reason: r.reason } : {}), ...(Array.isArray(r.reasons) ? { reasons: r.reasons } : {}), ...(r.actual_excerpt ? { actual_excerpt: r.actual_excerpt } : {}), ...(fixFor(r) ? { fix: fixFor(r) } : {}) });
|
|
268
271
|
|
|
269
272
|
/**
|
|
270
273
|
* The command. `deps` = {server: {healthy, req, json}, finish(file, results, lease) -> done's JSON, pollNext(file, timeout) -> string, readStdin(), holder?}.
|
|
@@ -279,32 +282,56 @@ export async function replyCommand(rest, deps) {
|
|
|
279
282
|
throw new ReplyError(err.message);
|
|
280
283
|
}
|
|
281
284
|
const { server } = deps;
|
|
285
|
+
// who calls: the local runner names itself; otherwise ONIONSKIN_HOLDER (an agent Onionskin started cannot change it), else the holder the
|
|
286
|
+
// server saved with this ticket. It goes with every request, so the server checks the binding and the lease before it hands out anything.
|
|
287
|
+
const envHolder = process.env[`${PRODUCT.envPrefix}_HOLDER`];
|
|
288
|
+
const ticketOfArgs = parseTicket(a.lease);
|
|
289
|
+
const holder = deps.holder || envHolder || (ticketOfArgs ? loadTicketRecord(ticketOfArgs.batch_id)?.holder : null) || "";
|
|
290
|
+
const notYours = () => new ReplyError(NOT_YOUR_FILE, { exit: APPLY_EXIT.write_refused, body: { code: "not_your_file" } });
|
|
282
291
|
if (!a.file) {
|
|
283
|
-
// no file given: the ticket
|
|
292
|
+
// no file given: the whole ticket (batch id and token) names the session; a wrong token names none
|
|
284
293
|
if (!(await server.healthy())) throw new ReplyError(`the ${PRODUCT.name} server is not running, so the delivery named by --lease cannot be checked. Nothing was written.`, { exit: APPLY_EXIT.write_refused });
|
|
285
|
-
const w = await server.req({ method: "GET", path: `/cli/batch-file?batch_id=${encodeURIComponent(
|
|
294
|
+
const w = await server.req({ method: "GET", path: `/cli/batch-file?batch_id=${encodeURIComponent(ticketOfArgs?.batch_id || "")}&lease_token=${encodeURIComponent(ticketOfArgs?.token || "")}&holder=${encodeURIComponent(holder)}` });
|
|
286
295
|
a.file = w.status === 200 ? server.json(w.text).file || "" : "";
|
|
287
296
|
if (!a.file) throw new ReplyError(`No delivered batch matches this ticket: your delivery was taken back, finished or handed on. Nothing was written. Run \`${PRODUCT.bin} poll <file>\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: "stale_batch" } });
|
|
288
297
|
}
|
|
298
|
+
// an agent Onionskin started: its own input first (only its one reply file, content-free checks), then its binding to its one file,
|
|
299
|
+
// refused before that file is looked at, in the same words whether it exists or is open
|
|
300
|
+
const bg = !deps.holder && isAgentHolder(envHolder);
|
|
301
|
+
const ticketRequired = () => new ReplyError(`--lease <ticket> is required: it is the lease_ticket of the batch (\`${PRODUCT.bin} status <file> --ticket\` prints it again)`);
|
|
302
|
+
let input;
|
|
303
|
+
if (bg) {
|
|
304
|
+
if (!ticketOfArgs) throw ticketRequired();
|
|
305
|
+
checkBackgroundInput(a, process.env[`${PRODUCT.envPrefix}_AGENT_DIR`], ticketOfArgs.batch_id);
|
|
306
|
+
input = await readInput(a, deps.readStdin, { quiet: true });
|
|
307
|
+
if (!agentMayUse(envHolder, a.file)) throw notYours();
|
|
308
|
+
}
|
|
289
309
|
let real;
|
|
290
310
|
try {
|
|
291
311
|
real = fs.realpathSync(path.resolve(a.file));
|
|
292
312
|
} catch {
|
|
293
313
|
throw new ReplyError(`File not found: ${JSON.stringify(a.file.slice(0, 200))}`, { exit: APPLY_EXIT.error });
|
|
294
314
|
}
|
|
295
|
-
if (!
|
|
296
|
-
const ticket =
|
|
297
|
-
if (!ticket) throw
|
|
298
|
-
|
|
299
|
-
if (bg) checkBackgroundInput(a, process.env[`${PRODUCT.envPrefix}_AGENT_DIR`], ticket.batch_id);
|
|
300
|
-
const input = await readInput(a, deps.readStdin, { quiet: bg });
|
|
315
|
+
if (!formatOfPath(real)) throw new ReplyError("Only .docx, .xlsx, .pptx, .html and .htm files are supported");
|
|
316
|
+
const ticket = ticketOfArgs;
|
|
317
|
+
if (!ticket) throw ticketRequired();
|
|
318
|
+
if (!bg) input = await readInput(a, deps.readStdin, { quiet: false });
|
|
301
319
|
|
|
302
|
-
// the batch the session holds for this delivery
|
|
320
|
+
// authorization first: the server hands out the batch the session holds for this delivery (the only source of anchors and hashes) only
|
|
321
|
+
// to this ticket's holder, checked before the reply is planned or run against the file
|
|
303
322
|
if (!(await server.healthy())) throw new ReplyError(`the ${PRODUCT.name} server is not running, so the delivery named by --lease cannot be checked. Nothing was written.`, { exit: APPLY_EXIT.write_refused });
|
|
304
|
-
const r = await server.req({ method: "GET", path: `/cli/batch?file=${encodeURIComponent(real)}` });
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
323
|
+
const r = await server.req({ method: "GET", path: `/cli/batch?file=${encodeURIComponent(real)}&holder=${encodeURIComponent(holder)}&batch_id=${encodeURIComponent(ticket.batch_id)}&lease_token=${encodeURIComponent(ticket.token)}` });
|
|
324
|
+
let refused = null;
|
|
325
|
+
try {
|
|
326
|
+
refused = r.status === 200 ? null : server.json(r.text);
|
|
327
|
+
} catch {
|
|
328
|
+
refused = {};
|
|
329
|
+
}
|
|
330
|
+
if (refused?.code === "not_your_file") throw notYours();
|
|
331
|
+
if (refused?.code === "no_batch") throw new ReplyError(`No batch is delivered for this file, so the ticket is void. Nothing was written. Run \`${PRODUCT.bin} poll\` and wait for the user's Send.`, { exit: APPLY_EXIT.write_refused, body: { code: "no_batch" } });
|
|
332
|
+
if (refused) throw new ReplyError(`This ticket is not the delivery the session holds: your delivery was taken back, finished or handed on, or the ticket is wrong. Nothing was written. Run \`${PRODUCT.bin} poll\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: refused.code === "not_lease_holder" ? "not_lease_holder" : "stale_batch" } });
|
|
333
|
+
const held = server.json(r.text).batch;
|
|
334
|
+
if (!held || held.batch_id !== ticket.batch_id) throw new ReplyError(`This ticket is not the delivery the session holds. Nothing was written. Run \`${PRODUCT.bin} poll\` again.`, { exit: APPLY_EXIT.write_refused, body: { code: "stale_batch" } });
|
|
308
335
|
|
|
309
336
|
const plan = planReplies(held, input, { partial: a.partial });
|
|
310
337
|
if (plan.errors.length) throw new ReplyError("the reply cannot be applied; nothing was written and the batch is still open", { body: { code: "bad_reply", lease_kept: true, problems: plan.errors } });
|
package/src/retain.js
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
// Retained size: what the server keeps of a request (the page's anchors, typed edits, view state) and of the anchors it builds, cut to
|
|
2
|
+
// fixed caps where it is stored, so a crafted file or page inside the documented size limits cannot make a session file, a Send's batch
|
|
3
|
+
// or a draft save hundreds of MB. Over a cap a value is cut and marked (`cut: true`), never refused: an anchor keeps a prefix of its
|
|
4
|
+
// text and its hash of the full text where one was paid for. Only a whole request past REQUEST_MAX is refused (src/server.js), with a
|
|
5
|
+
// plain message the page shows. Chosen from measurement (test/retain.test.js): ordinary use is untouched, see the numbers below.
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* One string field of an anchor (a quote, a paragraph's text, a typed edit's before and after). A whole-paragraph selection of an
|
|
9
|
+
* ordinary document is well under it: the longest paragraph of the sample files is under 300 characters and a long report paragraph
|
|
10
|
+
* 3,000 to 6,000. UTF-16 units.
|
|
11
|
+
*/
|
|
12
|
+
export const STRING_MAX = 20_000;
|
|
13
|
+
/** A location's own strings (a sheet or shape name, a range, a paragraph id): Excel caps a sheet name at 31 characters. */
|
|
14
|
+
const NAME_MAX = 1_000;
|
|
15
|
+
/** A page anchor's context (prefix, suffix): the server keeps 40 characters of each (src/docx.js, src/pptx.js CTX). */
|
|
16
|
+
const CONTEXT_MAX = 1_000;
|
|
17
|
+
/**
|
|
18
|
+
* One stored anchor, as JSON. Measured: a whole-paragraph Word anchor about 2 to 4 KB plus its quote, a 1,000 x 20 Excel range of
|
|
19
|
+
* ordinary values about 11 KB, a PowerPoint shape 3 KB; a quote at STRING_MAX makes about 25 KB. Past it an Excel anchor's values are cut
|
|
20
|
+
* to VALUE_SHORT characters a cell, then the optional evidence (values, formats, kinds, names, row cells) goes; the hashes stay.
|
|
21
|
+
*/
|
|
22
|
+
export const ANCHOR_MAX = 48 * 1024;
|
|
23
|
+
const VALUE_SHORT = 64;
|
|
24
|
+
/** One comment as stored (its words, the page's anchor, the anchor built from the file): past it the page's anchor is cut further. */
|
|
25
|
+
export const COMMENT_MAX = 128 * 1024;
|
|
26
|
+
/** One request's comments, note, typed edits and view state, as JSON in UTF-8 (not the document's bytes): past it the request is refused. */
|
|
27
|
+
export const REQUEST_MAX = 32 * 1024 * 1024;
|
|
28
|
+
/** One session file, in bytes. Past it every comment's page anchor and every typed edit are cut further before it is written (the waiting
|
|
29
|
+
* batch's evidence is already capped per anchor by src/batch.js); nothing is dropped. */
|
|
30
|
+
export const SESSION_MAX = 64 * 1024 * 1024;
|
|
31
|
+
/** Comments and pins of one file (drafts, sent, queued and finished): one more is refused in words (src/session.js), never dropped. */
|
|
32
|
+
export const COMMENTS_MAX = 500;
|
|
33
|
+
/** What the server says (413 too_many_comments) and the page shows when one more comment would pass COMMENTS_MAX. */
|
|
34
|
+
export const TOO_MANY_COMMENTS = `This file has ${COMMENTS_MAX} comments waiting; send or delete some before adding more.`;
|
|
35
|
+
/** The overall note, in code points: the box, the notes queued during a run and a waiting batch's note together. Past it a save or a
|
|
36
|
+
* Send is refused in words (src/session.js), never cut. The page's box takes no more (web/src/Sidebar.jsx). */
|
|
37
|
+
export const NOTE_MAX = 20_000;
|
|
38
|
+
/** One comment's words, in code points (src/sanitize.js userText): the page's box takes no more, so it is never cut from the page. */
|
|
39
|
+
export const COMMENT_TEXT_MAX = 5_000;
|
|
40
|
+
/** Typed edits kept for the next batch (they accumulate over Sends until one is delivered); later ones are counted, not kept. */
|
|
41
|
+
export const EDITS_MAX = 2_000;
|
|
42
|
+
/** A Word block path: the body index, then table row, cell and paragraph, deeper for nested tables. */
|
|
43
|
+
const PATH_MAX = 16;
|
|
44
|
+
|
|
45
|
+
const isHigh = (c) => c >= 0xd800 && c <= 0xdbff;
|
|
46
|
+
const isLow = (c) => c >= 0xdc00 && c <= 0xdfff;
|
|
47
|
+
// a slice keeps the whole string alive; this copy does not
|
|
48
|
+
const flat = (s) => Buffer.from(s, "utf16le").toString("utf16le");
|
|
49
|
+
|
|
50
|
+
/** `s` cut to its first `max` UTF-16 units (never inside a surrogate pair), as a flat string that does not hold on to the long one. */
|
|
51
|
+
export function cutText(s, max) {
|
|
52
|
+
if (typeof s !== "string" || s.length <= max) return s;
|
|
53
|
+
let n = max;
|
|
54
|
+
if (n > 0 && isHigh(s.charCodeAt(n - 1)) && isLow(s.charCodeAt(n))) n--;
|
|
55
|
+
return flat(s.slice(0, n));
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** `s` cut to its last `max` UTF-16 units (never inside a surrogate pair): the text just before a selection keeps the end nearest to it. */
|
|
59
|
+
export function cutTail(s, max) {
|
|
60
|
+
if (typeof s !== "string" || s.length <= max) return s;
|
|
61
|
+
let n = s.length - max;
|
|
62
|
+
if (isLow(s.charCodeAt(n)) && isHigh(s.charCodeAt(n - 1))) n++;
|
|
63
|
+
return flat(s.slice(n));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Every object the page sends that is stored is rebuilt from the fields Onionskin reads, each of a known type and size; anything else is
|
|
67
|
+
// dropped, so the work and the size are bounded by this list, never by what the request carries.
|
|
68
|
+
const int = (v) => (Number.isSafeInteger(v) ? v : undefined);
|
|
69
|
+
const finite = (v) => (typeof v === "number" && Number.isFinite(v) ? v : undefined);
|
|
70
|
+
const str = (v, max) => (typeof v === "string" ? cutText(v, max) : undefined);
|
|
71
|
+
const idOf = (v) => int(v) ?? str(v, 64); // a PowerPoint slide or shape id: a number, or a string from older pages
|
|
72
|
+
const blockPath = (v) => (Array.isArray(v) && v.length > 0 && v.length <= PATH_MAX && v.every((x) => Number.isSafeInteger(x) && x >= 0) ? [...v] : undefined);
|
|
73
|
+
const put = (out, k, v) => {
|
|
74
|
+
if (v !== undefined) out[k] = v;
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Where a comment, a pin or a typed edit sits, as the engine reports it (vendor/engine: docx Sz and the edit records, xlsx un, pptx AV):
|
|
79
|
+
* Word {blockPath, paraId, start, end, protected}, Excel {sheetIndex, sheetName, range}, PowerPoint {slideIndex, slideId, shapeId,
|
|
80
|
+
* shapeName, protected, protectedReason, cell: {row, col}}. Null for anything that is not an object.
|
|
81
|
+
*/
|
|
82
|
+
export function cleanLocation(l) {
|
|
83
|
+
if (!l || typeof l !== "object" || Array.isArray(l)) return null;
|
|
84
|
+
const out = {};
|
|
85
|
+
put(out, "blockPath", blockPath(l.blockPath));
|
|
86
|
+
put(out, "paraId", str(l.paraId, 64));
|
|
87
|
+
put(out, "start", int(l.start));
|
|
88
|
+
put(out, "end", int(l.end));
|
|
89
|
+
put(out, "protected", typeof l.protected === "boolean" ? l.protected : str(l.protected, 200));
|
|
90
|
+
put(out, "protectedReason", str(l.protectedReason, 200));
|
|
91
|
+
put(out, "sheetIndex", int(l.sheetIndex));
|
|
92
|
+
put(out, "sheetName", str(l.sheetName, 256)); // Excel caps a sheet name at 31 characters
|
|
93
|
+
put(out, "range", str(l.range, 64)); // "XFD1048576:XFD1048576" is 21
|
|
94
|
+
put(out, "slideIndex", int(l.slideIndex));
|
|
95
|
+
put(out, "slideId", idOf(l.slideId));
|
|
96
|
+
put(out, "shapeId", idOf(l.shapeId));
|
|
97
|
+
put(out, "shapeName", str(l.shapeName, NAME_MAX));
|
|
98
|
+
if (l.cell && typeof l.cell === "object" && int(l.cell.row) !== undefined && int(l.cell.col) !== undefined) out.cell = { row: l.cell.row, col: l.cell.col };
|
|
99
|
+
// HTML (web/src/html-view.js): the located node's id in one version of the file, and that version's sha256 (an id means nothing in another)
|
|
100
|
+
put(out, "nodeId", int(l.nodeId));
|
|
101
|
+
if (typeof l.version === "string" && /^[0-9a-f]{64}$/.test(l.version)) out.version = l.version;
|
|
102
|
+
return out;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* The page's view at Send (the engine's getViewState): Word {blockPath, textHash, quote, offsetPx, zoom, fraction}, Excel {sheetIndex,
|
|
107
|
+
* sheetName, topLeft, selection}, PowerPoint {slideIndex, zoom}, each with its `format`. Null for anything else.
|
|
108
|
+
*/
|
|
109
|
+
export function cleanView(v) {
|
|
110
|
+
if (!v || typeof v !== "object" || Array.isArray(v) || !["docx", "xlsx", "pptx", "html"].includes(v.format)) return null;
|
|
111
|
+
if (v.format === "html") return cleanHtmlView(v);
|
|
112
|
+
const out = { format: v.format };
|
|
113
|
+
put(out, "blockPath", blockPath(v.blockPath));
|
|
114
|
+
put(out, "textHash", str(v.textHash, 64));
|
|
115
|
+
put(out, "quote", str(v.quote, 200)); // the engine sends 80
|
|
116
|
+
put(out, "offsetPx", finite(v.offsetPx));
|
|
117
|
+
put(out, "zoom", finite(v.zoom) ?? str(v.zoom, 16)); // a number or "fit"
|
|
118
|
+
put(out, "fraction", finite(v.fraction));
|
|
119
|
+
put(out, "sheetIndex", int(v.sheetIndex));
|
|
120
|
+
put(out, "sheetName", str(v.sheetName, 256));
|
|
121
|
+
put(out, "topLeft", str(v.topLeft, 64));
|
|
122
|
+
put(out, "selection", str(v.selection, 64));
|
|
123
|
+
put(out, "slideIndex", int(v.slideIndex));
|
|
124
|
+
return out;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** An HTML view (web/src/html-frame.js getViewState): {version, x, y, fraction, top: {nodeId, quote, offsetPx}}; hidden elements stay in the page. */
|
|
128
|
+
function cleanHtmlView(v) {
|
|
129
|
+
const out = { format: "html" };
|
|
130
|
+
if (typeof v.version === "string" && /^[0-9a-f]{64}$/.test(v.version)) out.version = v.version;
|
|
131
|
+
put(out, "x", finite(v.x));
|
|
132
|
+
put(out, "y", finite(v.y));
|
|
133
|
+
put(out, "fraction", finite(v.fraction));
|
|
134
|
+
const t = v.top;
|
|
135
|
+
if (t && typeof t === "object" && int(t.nodeId) !== undefined) out.top = { nodeId: t.nodeId, quote: str(t.quote, 200) ?? "", offsetPx: finite(t.offsetPx) ?? 0 };
|
|
136
|
+
return out;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The page's anchor of a comment, as stored: the fields the server and the page read (location, quote, prefix, suffix, paraText,
|
|
141
|
+
* format, hitKind), each cut. A quote or paragraph text cut here is marked `cut`, and the anchor is then built from the file as the
|
|
142
|
+
* paragraph or shape it sits in (src/anchors.js), never as a range of the cut text. Null for anything that is not an object.
|
|
143
|
+
*/
|
|
144
|
+
export function cleanPageAnchor(a) {
|
|
145
|
+
if (!a || typeof a !== "object" || Array.isArray(a)) return null;
|
|
146
|
+
const out = {};
|
|
147
|
+
const loc = cleanLocation(a.location);
|
|
148
|
+
if (loc) out.location = loc;
|
|
149
|
+
let cut = false;
|
|
150
|
+
for (const k of ["quote", "paraText"]) {
|
|
151
|
+
if (typeof a[k] !== "string") continue;
|
|
152
|
+
out[k] = cutText(a[k], STRING_MAX);
|
|
153
|
+
if (out[k].length < a[k].length) cut = true;
|
|
154
|
+
}
|
|
155
|
+
put(out, "prefix", typeof a.prefix === "string" ? cutTail(a.prefix, CONTEXT_MAX) : undefined); // the end nearest the selection
|
|
156
|
+
put(out, "suffix", str(a.suffix, CONTEXT_MAX));
|
|
157
|
+
for (const k of ["format", "hitKind"]) put(out, k, str(a[k], 40));
|
|
158
|
+
if (cut || a.cut === true) out.cut = true;
|
|
159
|
+
return out;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** A typed edit the page reported ({kind, before, after, label, location}), as stored for the next batch. */
|
|
163
|
+
export function cleanEdit(e) {
|
|
164
|
+
const side = (v) => (typeof v === "string" ? cutText(v, STRING_MAX) : v == null ? v : typeof v === "number" || typeof v === "boolean" ? cutText(String(v), 100) : null);
|
|
165
|
+
const before = side(e?.before);
|
|
166
|
+
const after = side(e?.after);
|
|
167
|
+
const out = { kind: str(e?.kind, 40) ?? null, before, after, label: str(e?.label, NAME_MAX) ?? null, location: cleanLocation(e?.location) };
|
|
168
|
+
if ((typeof e?.before === "string" && before.length < e.before.length) || (typeof e?.after === "string" && after.length < e.after.length)) out.cut = true;
|
|
169
|
+
return out;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// in UTF-8 bytes, as the file is written (a CJK character is 3)
|
|
173
|
+
export const jsonSize = (v) => {
|
|
174
|
+
try {
|
|
175
|
+
return Buffer.byteLength(JSON.stringify(v) ?? "");
|
|
176
|
+
} catch {
|
|
177
|
+
return Infinity;
|
|
178
|
+
}
|
|
179
|
+
};
|
|
180
|
+
const OPTIONAL = ["values", "types", "formats", "names"];
|
|
181
|
+
/**
|
|
182
|
+
* An anchor built from the file (src/anchors.js buildDurable, src/session.js reanchor), as stored and handed on: every string at
|
|
183
|
+
* STRING_MAX; past ANCHOR_MAX as JSON, an Excel anchor's values cut to VALUE_SHORT characters a cell, then the optional evidence dropped.
|
|
184
|
+
* Hashes, places and the kept text prefix stay. Marked `cut` when anything was cut. Returns the same object when nothing was.
|
|
185
|
+
*/
|
|
186
|
+
export function capDurable(d) {
|
|
187
|
+
if (!d || typeof d !== "object") return d;
|
|
188
|
+
let out = d;
|
|
189
|
+
const own = () => (out === d ? (out = { ...d }) : out);
|
|
190
|
+
for (const [k, v] of Object.entries(d)) if (typeof v === "string" && v.length > STRING_MAX) own()[k] = cutText(v, STRING_MAX);
|
|
191
|
+
if (jsonSize(out) > ANCHOR_MAX) {
|
|
192
|
+
own();
|
|
193
|
+
if (Array.isArray(out.values)) out.values = out.values.map((r) => (Array.isArray(r) ? r.map((x) => (typeof x === "string" ? cutText(x, VALUE_SHORT) : x)) : r));
|
|
194
|
+
if (jsonSize(out) > ANCHOR_MAX) {
|
|
195
|
+
for (const k of OPTIONAL) if (Array.isArray(out[k])) out[k] = [];
|
|
196
|
+
if (out.ctx && typeof out.ctx === "object") out.ctx = { ...out.ctx, rowCells: [] };
|
|
197
|
+
for (const [k, v] of Object.entries(out)) {
|
|
198
|
+
if (typeof v !== "string" || v.length <= 2000 || /hash/i.test(k)) continue;
|
|
199
|
+
// a quote is a range of the text: cut, it would point at a shorter one, so the anchor becomes its whole paragraph or shape
|
|
200
|
+
if (k === "quote") Object.assign(out, { quote: "", start: null, end: null, ...("nth" in out ? { nth: null } : {}) });
|
|
201
|
+
else out[k] = cutText(v, 2000);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
if (d.pinFallback) {
|
|
206
|
+
const p = capDurable(d.pinFallback);
|
|
207
|
+
if (p !== d.pinFallback) own().pinFallback = p;
|
|
208
|
+
}
|
|
209
|
+
if (out !== d) out.cut = true;
|
|
210
|
+
return out;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** A comment as stored, past COMMENT_MAX: the page's anchor is cut to 2,000 characters a field (the anchor built from the file holds the rest). */
|
|
214
|
+
export function capComment(c) {
|
|
215
|
+
if (jsonSize(c) <= COMMENT_MAX) return c;
|
|
216
|
+
const out = { ...c, cut: true };
|
|
217
|
+
if (out.anchor) out.anchor = { ...out.anchor, cut: true, ...Object.fromEntries(["quote", "paraText"].filter((k) => typeof out.anchor[k] === "string").map((k) => [k, cutText(out.anchor[k], 2000)])) };
|
|
218
|
+
if (out.durable) out.durable = capDurable(out.durable);
|
|
219
|
+
return out;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** The size of a request's own data (comments, note, edits, view state) as JSON; Infinity when it cannot be written as JSON. */
|
|
223
|
+
export const requestSize = (parts) => jsonSize(parts);
|
package/src/review.js
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
|
-
import { readUnits } from "./anchors.js";
|
|
2
|
+
import { readUnits, unitsFor } from "./anchors.js";
|
|
3
3
|
import { findAll } from "./ops/runs.js";
|
|
4
4
|
import { untrusted } from "./sanitize.js";
|
|
5
5
|
import { exactHash, normText } from "./text.js";
|
|
6
|
-
import {
|
|
6
|
+
import { parseA1 } from "./ops/common.js";
|
|
7
|
+
import { a1, a1Range, cellAt, mergeAt } from "./xlsx.js";
|
|
8
|
+
import { unknownFormat } from "./constants.js";
|
|
9
|
+
import { htmlRawUnits } from "./html.js";
|
|
7
10
|
|
|
8
11
|
// Idea 21: the agent reviews first. The person taps one of three fixed requests; the batch carries that fixed sentence in
|
|
9
12
|
// `instructions.review` and a read-only outline of the file (`untrusted_document_evidence.units[]`, built here by code). The agent answers
|
|
@@ -50,11 +53,12 @@ const isObj = (v) => v && typeof v === "object" && !Array.isArray(v);
|
|
|
50
53
|
* {units: [{id, label, text, protected}] (at most UNITS_MAX, for the batch), more, index: {id -> where the unit is and its text hash}} (the
|
|
51
54
|
* index stays on the server). A unit an edit cannot address (a DOCX paragraph with no body path, a PPTX shape whose id is shared on its
|
|
52
55
|
* slide) is marked protected: suggestions on it are refused.
|
|
53
|
-
*
|
|
54
|
-
*
|
|
56
|
+
* `reading` (optional) is readUnits of `bytes` that the caller already has (src/anchors.js unitsFor).
|
|
57
|
+
* Ceiling: every paragraph, shape or cell is read and sorted on the server's main thread before the cut to UNITS_MAX; the readers' caps
|
|
58
|
+
* (src/docx.js, src/xlsx.js, src/pptx.js) bound that to a few hundred milliseconds. Upgrade path: build in a worker.
|
|
55
59
|
*/
|
|
56
|
-
export function unitsOf(format, bytes) {
|
|
57
|
-
const raw = rawUnits(format, bytes);
|
|
60
|
+
export function unitsOf(format, bytes, reading = null) {
|
|
61
|
+
const raw = rawUnits(format, bytes, reading);
|
|
58
62
|
const units = [];
|
|
59
63
|
const index = {};
|
|
60
64
|
raw.slice(0, UNITS_MAX).forEach((x, i) => {
|
|
@@ -67,9 +71,9 @@ export function unitsOf(format, bytes) {
|
|
|
67
71
|
}
|
|
68
72
|
|
|
69
73
|
/** Every unit of the file in document order: {label, text (the full text, not cleaned), protected, at (where it is and its text hash)}. */
|
|
70
|
-
function rawUnits(format, bytes) {
|
|
74
|
+
function rawUnits(format, bytes, reading = null) {
|
|
71
75
|
const raw = [];
|
|
72
|
-
const u = readUnits(format, bytes);
|
|
76
|
+
const u = reading || readUnits(format, bytes);
|
|
73
77
|
if (format === "docx") {
|
|
74
78
|
for (const p of u.paragraphs) {
|
|
75
79
|
if (!normText(p.text)) continue;
|
|
@@ -93,11 +97,13 @@ function rawUnits(format, bytes) {
|
|
|
93
97
|
const sd = u.wb.sheetCells(sheet);
|
|
94
98
|
for (const c of cells) {
|
|
95
99
|
const text = c.k === "formula" && c.cached ? `${c.text} (shows ${c.cached})` : c.text;
|
|
96
|
-
const m = sd
|
|
100
|
+
const m = mergeAt(sd, c.r, c.c);
|
|
97
101
|
raw.push({ label: `${sheet.name}!${a1(c.r, c.c)}`, text, protected: false, ...(m ? { merged: a1Range(m) } : {}), at: { sheet: sheet.name, range: a1(c.r, c.c), text_hash: exactHash(c.text) } });
|
|
98
102
|
}
|
|
99
103
|
}
|
|
100
|
-
} else
|
|
104
|
+
} else if (format === "html") {
|
|
105
|
+
for (const x of htmlRawUnits(u.read)) raw.push(x); // src/html.js: elements that hold text and leaf elements, tag names and ids per R7
|
|
106
|
+
} else unknownFormat(format, "rawUnits");
|
|
101
107
|
return raw;
|
|
102
108
|
}
|
|
103
109
|
|
|
@@ -112,8 +118,8 @@ function rawUnits(format, bytes) {
|
|
|
112
118
|
* Every entry, as JSON, stays under OUTLINE_LINE_MAX characters (its text is cut further when quotes or backslashes double in JSON), so
|
|
113
119
|
* the outline file, one entry per line, can be paged with Claude Code's Read tool (which cuts lines at 2,000 characters).
|
|
114
120
|
*/
|
|
115
|
-
export function outlineOf(format, bytes, max = UNITS_MAX) {
|
|
116
|
-
const raw = rawUnits(format, bytes);
|
|
121
|
+
export function outlineOf(format, bytes, max = UNITS_MAX, reading = null) {
|
|
122
|
+
const raw = rawUnits(format, bytes, reading);
|
|
117
123
|
const outline = raw.slice(0, max).map((x, i) => {
|
|
118
124
|
const target = x.protected ? null : targetOf(x.at, format);
|
|
119
125
|
const entry = (n) => {
|
|
@@ -183,9 +189,11 @@ function textNow(units, at) {
|
|
|
183
189
|
const sh = units.deck.slides.find((s) => String(s.id) === at.slide_id)?.shapes.filter((x) => x.id === at.shape_id);
|
|
184
190
|
return sh?.length === 1 && sh[0].hash === at.text_hash ? sh[0].text : null;
|
|
185
191
|
}
|
|
192
|
+
if (units.format !== "xlsx") return unknownFormat(units.format, "textNow");
|
|
186
193
|
const sheet = units.wb.sheets.find((s) => s.name === at.sheet);
|
|
187
194
|
if (!sheet) return null;
|
|
188
|
-
const
|
|
195
|
+
const q = parseA1(String(at.range || ""));
|
|
196
|
+
const c = q && q.r0 === q.r1 && q.c0 === q.c1 ? cellAt(units.wb.sheetCells(sheet), q.r0, q.c0) : null;
|
|
189
197
|
return c && exactHash(c.text) === at.text_hash ? c.text : null;
|
|
190
198
|
}
|
|
191
199
|
|
|
@@ -227,34 +235,40 @@ export function cardFor(item, units, index, { reviewId, format }) {
|
|
|
227
235
|
function placeOf(at, format) {
|
|
228
236
|
if (format === "docx") return { format, ordinal: at.ordinal, para_id: at.para_id, body_path: at.body_path };
|
|
229
237
|
if (format === "pptx") return { format, slide_id: at.slide_id, slide_index: at.slide_index, shape_id: at.shape_id };
|
|
230
|
-
return { format, sheet: at.sheet, range: at.range };
|
|
238
|
+
if (format === "xlsx") return { format, sheet: at.sheet, range: at.range };
|
|
239
|
+
return unknownFormat(format, "review placeOf");
|
|
231
240
|
}
|
|
232
241
|
|
|
233
242
|
/** The op Accept runs: one find-and-replace in one paragraph or shape, with the unit's text hash as `expect`. */
|
|
234
243
|
export function opFor(at, format, id, { find, replace }) {
|
|
235
244
|
if (format === "docx") return { op: "replace_text", id, target: targetOf(at, format), expect: { text_hash: at.text_hash }, find, replace };
|
|
236
245
|
if (format === "pptx") return { op: "replace_text_in_shape", id, target: targetOf(at, format), expect: { text_hash: at.text_hash }, find, replace };
|
|
237
|
-
return null;
|
|
246
|
+
if (format === "xlsx") return null; // a spreadsheet takes notes only (cleanSuggestion)
|
|
247
|
+
return unknownFormat(format, "opFor");
|
|
238
248
|
}
|
|
239
249
|
|
|
240
250
|
/** The explicit op target of a unit (ops.schema.json docx_target, pptx_target, xlsx_range_target). */
|
|
241
251
|
function targetOf(at, format) {
|
|
242
252
|
if (format === "docx") return at.para_id ? { para_id: at.para_id, body_path: at.body_path, text_hash: at.text_hash } : { body_path: at.body_path, text_hash: at.text_hash };
|
|
243
253
|
if (format === "pptx") return { slide_id: at.slide_id, shape_id: at.shape_id };
|
|
244
|
-
return { sheet: at.sheet, range: at.range };
|
|
254
|
+
if (format === "xlsx") return { sheet: at.sheet, range: at.range };
|
|
255
|
+
// HTML: exactly one form, as the op schema takes it (src/ops/schema.js): the element's id when it is unique (it survives elements
|
|
256
|
+
// added before it), else its path; the path is in the outline's `at` either way
|
|
257
|
+
if (format === "html") return at.element_id ? { element_id: at.element_id } : { path: at.path };
|
|
258
|
+
return unknownFormat(format, "targetOf");
|
|
245
259
|
}
|
|
246
260
|
|
|
247
261
|
/**
|
|
248
262
|
* The agent's suggestions -> {cards, dropped: [{index, reason}]}. More than SUGGESTIONS_MAX stores none (refused, never the first 20).
|
|
249
263
|
* `bytes` is the file now; `units` the batch's outline; `index` the server's record.
|
|
250
264
|
*/
|
|
251
|
-
export function cardsFrom(list, { bytes, format, units, index, reviewId }) {
|
|
265
|
+
export function cardsFrom(list, { bytes, sha = null, format, units, index, reviewId }) {
|
|
252
266
|
if (list === undefined || list === null) return { cards: [], dropped: [] };
|
|
253
267
|
if (!Array.isArray(list)) return { cards: [], dropped: [{ index: null, reason: "suggestions must be a list" }] };
|
|
254
268
|
if (list.length > SUGGESTIONS_MAX) return { cards: [], dropped: list.map((_, i) => ({ index: i, reason: `more than ${SUGGESTIONS_MAX} suggestions` })) };
|
|
255
269
|
let now;
|
|
256
270
|
try {
|
|
257
|
-
now = readUnits(format, bytes);
|
|
271
|
+
now = sha ? unitsFor(format, bytes, sha) : readUnits(format, bytes); // `sha`: the bytes' own hash, to reuse the last reading of them
|
|
258
272
|
} catch {
|
|
259
273
|
return { cards: [], dropped: list.map((_, i) => ({ index: i, reason: "the file could not be read" })) };
|
|
260
274
|
}
|
package/src/rewrite-check.js
CHANGED
|
@@ -176,7 +176,7 @@ function expand(s) {
|
|
|
176
176
|
function tokens(s) {
|
|
177
177
|
const out = [];
|
|
178
178
|
// a number keeps a sign written right before it ("-5", "+3", "~240"), but not the dash of a range ("3-5")
|
|
179
|
-
const re = /((?:(?<![\p{L}\p{N}])[-+~<>\u2212\u2248\u00b1])?[$€£¥₹]?\p{N}[\p{N},.:/%]*(?:\s?%)?)|[\p{L}][\p{L}\p{M}'-]
|
|
179
|
+
const re = /((?:(?<![\p{L}\p{N}])[-+~<>\u2212\u2248\u00b1])?[$€£¥₹]?\p{N}[\p{N},.:/%]*(?:\s?%)?)|[\p{L}][\p{L}\p{M}'-]{0,4095}/gu; // a word at most 4,096 characters (a run of 7.9 million marks overflowed the regex stack)
|
|
180
180
|
for (const m of expand(s).matchAll(re)) {
|
|
181
181
|
if (m[1] !== undefined) {
|
|
182
182
|
const kept = m[0].replace(/[.,:/]+$/, "");
|
|
@@ -661,7 +661,7 @@ const factText = (s) =>
|
|
|
661
661
|
.replace(/(\p{N})\s*(?:percent|per cent)\b/giu, "$1%")
|
|
662
662
|
.replace(/(\p{N})\s*([ap])\.?\s?m\.?(?![\p{L}\p{N}])/giu, (_, n, x) => `${n} ${x.toLowerCase()}m`);
|
|
663
663
|
|
|
664
|
-
const APOS_WORD = /\p{L}[\p{L}\p{M}']
|
|
664
|
+
const APOS_WORD = /\p{L}[\p{L}\p{M}']{0,4095}/gu; // bounded, as in tokens() above
|
|
665
665
|
/**
|
|
666
666
|
* Words that differ only by an apostrophe are written alike on both sides, with the apostrophe ("Its" and "It's" both "It's"), but only
|
|
667
667
|
* a word that one side has more often than the other (a word in both is no misspelling of the other's word), and never a pair whose
|
package/src/rewriter.js
CHANGED
|
@@ -71,18 +71,27 @@ export const APPLE_STATES = {
|
|
|
71
71
|
/** The probe's words (scripts/rewriter/apple/main.swift --probe) to the state codes above. */
|
|
72
72
|
export const PROBE_REASONS = { deviceNotEligible: "device_not_eligible", appleIntelligenceNotEnabled: "apple_intelligence_not_enabled", modelNotReady: "model_not_ready" };
|
|
73
73
|
|
|
74
|
+
const shaCache = new Map(); // `${bin}\0${mtimeMs}\0${size}\0${ino}` -> sha256
|
|
75
|
+
const quarantineCache = new Map(); // `${bin}\0${mtimeMs}\0${ctimeMs}\0${size}\0${ino}` -> boolean
|
|
76
|
+
const cacheSet = (m, k, v) => {
|
|
77
|
+
if (m.size >= 32) m.clear();
|
|
78
|
+
m.set(k, v);
|
|
79
|
+
return v;
|
|
80
|
+
};
|
|
81
|
+
|
|
74
82
|
/**
|
|
75
83
|
* Does the file carry the com.apple.quarantine mark? Asked of /usr/bin/xattr, never by running the file. true | false; anything but a clean
|
|
76
84
|
* "No such xattr" answer counts as marked (fail closed).
|
|
77
85
|
*/
|
|
78
86
|
export function isQuarantined(file) {
|
|
79
|
-
const r = spawnSync(XATTR, ["-p", QUARANTINE, file], { encoding: "utf8", timeout: 5000 });
|
|
87
|
+
const r = spawnSync(XATTR, ["-p", QUARANTINE, file], { encoding: "utf8", timeout: 5000, env: { ...process.env, LC_ALL: "C" } });
|
|
80
88
|
if (r.status === 0) return true;
|
|
81
89
|
return !(r.status === 1 && /No such xattr/.test(r.stderr || ""));
|
|
82
90
|
}
|
|
83
91
|
/** Removes the mark from exactly this file. true when the file carries no mark afterwards. */
|
|
84
92
|
export function clearQuarantine(file) {
|
|
85
|
-
spawnSync(XATTR, ["-d", QUARANTINE, file], { encoding: "utf8", timeout: 5000 });
|
|
93
|
+
spawnSync(XATTR, ["-d", QUARANTINE, file], { encoding: "utf8", timeout: 5000, env: { ...process.env, LC_ALL: "C" } });
|
|
94
|
+
for (const k of quarantineCache.keys()) if (k.startsWith(`${file}\0`)) quarantineCache.delete(k);
|
|
86
95
|
return !isQuarantined(file);
|
|
87
96
|
}
|
|
88
97
|
|
|
@@ -112,20 +121,25 @@ export function hostFacts() {
|
|
|
112
121
|
export function appleHelperState(o = {}) {
|
|
113
122
|
const dir = o.dir ?? (env("APPLE_HELPER_DIR", "") || APPLE_HELPER_DIR); // the folder variable is for the browser tests, with a stand-in helper
|
|
114
123
|
const host = o.host ?? hostFacts();
|
|
115
|
-
const marked = o.quarantined ?? isQuarantined;
|
|
116
124
|
const major = Number(String(host.macos ?? "").split(".")[0]);
|
|
117
125
|
if (!Number.isFinite(major) || major < 26) return { ok: false, reason: "macos_too_old" };
|
|
118
126
|
if (!host.appleSilicon) return { ok: false, reason: "device_not_eligible" };
|
|
119
127
|
const bin = path.join(dir, "onionskin-fm");
|
|
120
128
|
let rec;
|
|
129
|
+
let st;
|
|
121
130
|
try {
|
|
122
131
|
rec = JSON.parse(fs.readFileSync(path.join(dir, "onionskin-fm.json"), "utf8"));
|
|
123
|
-
|
|
132
|
+
st = fs.lstatSync(bin);
|
|
133
|
+
if (!st.isFile()) return { ok: false, reason: "helper_missing" };
|
|
124
134
|
} catch {
|
|
125
135
|
return { ok: false, reason: "helper_missing" };
|
|
126
136
|
}
|
|
127
|
-
|
|
128
|
-
|
|
137
|
+
const shaKey = `${bin}\0${st.mtimeMs}\0${st.size}\0${st.ino}`;
|
|
138
|
+
const sha = shaCache.get(shaKey) ?? cacheSet(shaCache, shaKey, fileSha256(bin));
|
|
139
|
+
if (typeof rec?.sha256 !== "string" || sha !== rec.sha256) return { ok: false, reason: "helper_changed" };
|
|
140
|
+
const qKey = `${bin}\0${st.mtimeMs}\0${st.ctimeMs}\0${st.size}\0${st.ino}`;
|
|
141
|
+
const isMarked = o.quarantined ? o.quarantined(bin) : (quarantineCache.get(qKey) ?? cacheSet(quarantineCache, qKey, isQuarantined(bin)));
|
|
142
|
+
if (isMarked) return { ok: false, reason: "quarantined", bin };
|
|
129
143
|
return { ok: true, bin, sha256: rec.sha256 };
|
|
130
144
|
}
|
|
131
145
|
/** The person's words for a reason code, or null. helper_changed reads as helper_missing. */
|
|
@@ -175,7 +189,7 @@ export function workerEnv() {
|
|
|
175
189
|
* Options: idleMs (default 15 minutes), startMs, stuckMs, retryMs.
|
|
176
190
|
*/
|
|
177
191
|
export function createWorker(launch, opts = {}) {
|
|
178
|
-
const o = { idleMs: idleMs(), startMs: START_LIMIT_MS, stuckMs: STUCK_MS, retryMs: RETRY_AFTER_FAILURE_MS, ...opts };
|
|
192
|
+
const o = { idleMs: idleMs(), startMs: START_LIMIT_MS, stuckMs: STUCK_MS, retryMs: RETRY_AFTER_FAILURE_MS, killGraceMs: 500, ...opts };
|
|
179
193
|
let child = null;
|
|
180
194
|
let ready = false;
|
|
181
195
|
let failedAt = 0;
|
|
@@ -197,7 +211,23 @@ export function createWorker(launch, opts = {}) {
|
|
|
197
211
|
try {
|
|
198
212
|
c.stdin.end();
|
|
199
213
|
} catch {}
|
|
200
|
-
c.
|
|
214
|
+
if (c.exitCode === null && c.signalCode === null) {
|
|
215
|
+
try {
|
|
216
|
+
c.kill("SIGTERM");
|
|
217
|
+
} catch {}
|
|
218
|
+
const forceKill = () => {
|
|
219
|
+
try {
|
|
220
|
+
if (c.exitCode === null && c.signalCode === null) c.kill("SIGKILL");
|
|
221
|
+
} catch {}
|
|
222
|
+
};
|
|
223
|
+
const killTimer = setTimeout(forceKill, o.killGraceMs);
|
|
224
|
+
killTimer.unref?.();
|
|
225
|
+
process.once("exit", forceKill);
|
|
226
|
+
c.once("exit", () => {
|
|
227
|
+
clearTimeout(killTimer);
|
|
228
|
+
process.off("exit", forceKill);
|
|
229
|
+
});
|
|
230
|
+
}
|
|
201
231
|
}
|
|
202
232
|
if (busy) {
|
|
203
233
|
busy.settle({ ok: false, code: "worker_exited" });
|
package/src/sanitize.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// Built with new RegExp from escaped strings so no raw separator characters live in this source file.
|
|
2
2
|
// C0 (\n \t \r become spaces first), DEL, C1, bidi controls (LRM/RLM/ALM, embeddings, overrides, isolates), zero-width and BOM.
|
|
3
|
+
import { nfc } from "./nfc.js";
|
|
3
4
|
const STRIP = new RegExp("[\\u0000-\\u001F\\u007F-\\u009F\\u061C\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u2069\\uFEFF]", "g");
|
|
4
5
|
const WS = new RegExp("[\\t\\n\\r\\u2028\\u2029]+", "g");
|
|
5
6
|
// the user's own words keep newlines (\u000A) and tabs; everything else control/bidi is dropped
|
|
@@ -8,21 +9,33 @@ const MAX_UNTRUSTED = 500;
|
|
|
8
9
|
const MARK = "…[truncated]";
|
|
9
10
|
|
|
10
11
|
/** Untrusted document-derived text: strip control and bidi characters, NFC, collapse whitespace, cap at 500 chars including the marker. */
|
|
12
|
+
// Of a longer value only the first READ_PER_KEPT x max characters are cleaned: what is kept is at most `max` characters, and cleaning all
|
|
13
|
+
// of a paragraph of 7.9 million characters, for each of 50 comments quoting it, held the server's thread 2.5 s at Send.
|
|
14
|
+
const READ_PER_KEPT = 16;
|
|
11
15
|
export function untrusted(value, max = MAX_UNTRUSTED) {
|
|
12
16
|
if (value == null) return "";
|
|
13
|
-
let
|
|
17
|
+
let raw = String(value);
|
|
18
|
+
const cut = raw.length > READ_PER_KEPT * max;
|
|
19
|
+
if (cut) raw = raw.slice(0, READ_PER_KEPT * max);
|
|
20
|
+
let s = nfc(raw.replace(WS, " ").replace(STRIP, ""));
|
|
14
21
|
s = s.replace(/ {2,}/g, " ").trim();
|
|
15
22
|
const cps = [...s];
|
|
16
23
|
if (cps.length > max) s = cps.slice(0, max - [...MARK].length).join("") + MARK;
|
|
24
|
+
else if (cut) s = cps.slice(0, Math.max(0, max - [...MARK].length)).join("") + MARK; // cut before it was cleaned: still says so
|
|
17
25
|
return s;
|
|
18
26
|
}
|
|
19
27
|
|
|
20
28
|
/** The user's own words: keep newlines, strip other control and bidi characters, cap generously. */
|
|
29
|
+
// Of a longer value only the first READ_PER_KEPT x max characters are read, as in untrusted(): 50 comments of 7.9 million characters
|
|
30
|
+
// each were normalized whole at every draft save.
|
|
21
31
|
export function userText(value, max = 5000) {
|
|
22
32
|
if (value == null) return "";
|
|
23
|
-
|
|
33
|
+
let raw = String(value);
|
|
34
|
+
const cut = raw.length > READ_PER_KEPT * max;
|
|
35
|
+
if (cut) raw = raw.slice(0, READ_PER_KEPT * max);
|
|
36
|
+
const s = nfc(raw.replace(/\r\n?/g, "\n").replace(USER_STRIP, ""));
|
|
24
37
|
const cps = [...s];
|
|
25
|
-
return cps.length > max ? cps.slice(0, max).join("") + MARK : s;
|
|
38
|
+
return cps.length > max ? cps.slice(0, max).join("") + MARK : cut ? s + MARK : s;
|
|
26
39
|
}
|
|
27
40
|
|
|
28
41
|
const FLAG_KEEP = new Set(["code", "severity"]);
|