onionskin-cli 0.0.3 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +198 -0
- package/INSTALL.md +5 -1
- package/NOTICE +34 -0
- package/README.md +102 -34
- package/docs/contracts/batch.schema.json +27 -4
- package/docs/contracts/ops.schema.json +117 -5
- package/package.json +4 -2
- package/scripts/checker/check.py +24 -5
- package/scripts/checker/requirements.txt +482 -5
- package/scripts/requirements.txt +482 -5
- package/scripts/rewriter/apple/main.swift +1 -1
- package/scripts/rewriter/apple/onionskin-fm +0 -0
- package/scripts/rewriter/apple/onionskin-fm.json +1 -1
- package/skills/onionskin/SKILL.md +12 -9
- package/skills/onionskin/reference/batch.md +5 -3
- package/skills/onionskin/reference/formats.md +5 -1
- package/skills/onionskin/reference/ops.md +48 -4
- package/skills/onionskin/reference/troubleshooting.md +31 -0
- package/src/agent-detect.js +28 -2
- package/src/agent-prompt.js +12 -1
- package/src/agent-runner.js +110 -20
- package/src/anchor-work.js +154 -0
- package/src/anchor-worker.js +38 -0
- package/src/anchors.js +61 -11
- package/src/apply.js +119 -47
- package/src/batch.js +32 -8
- package/src/check-worker.js +8 -5
- package/src/checker.js +238 -33
- package/src/cli.js +248 -42
- package/src/constants.js +34 -1
- package/src/diff.js +3 -1
- package/src/doctor.js +12 -3
- package/src/docx.js +464 -87
- package/src/edit-hash.js +19 -0
- package/src/fsutil.js +14 -6
- package/src/history-worker.js +8 -8
- package/src/history.js +144 -31
- package/src/html-active.js +384 -0
- package/src/html-envelope.js +51 -0
- package/src/html-parser.js +8 -0
- package/src/html-render-worker.js +25 -0
- package/src/html-render.js +939 -0
- package/src/html-utf8.js +65 -0
- package/src/html-write-worker.js +14 -0
- package/src/html-write.js +380 -0
- package/src/html.js +1259 -0
- package/src/local.js +68 -4
- package/src/merge-package.js +1166 -0
- package/src/merge-pptx.js +98 -38
- package/src/merge-proof.js +1019 -0
- package/src/merge-worker.js +60 -0
- package/src/merge-xlsx.js +84 -29
- package/src/merge.js +286 -83
- package/src/nfc.js +46 -0
- package/src/notes.js +122 -0
- package/src/offload.js +419 -1
- package/src/open-worker.js +27 -0
- package/src/ops/common.js +2 -1
- package/src/ops/docx-table.js +5 -3
- package/src/ops/docx.js +58 -29
- package/src/ops/html.js +394 -0
- package/src/ops/pptx.js +82 -23
- package/src/ops/run.js +23 -3
- package/src/ops/runs.js +215 -13
- package/src/ops/schema.js +8 -1
- package/src/ops/xlsx.js +121 -10
- package/src/ops-worker.js +17 -0
- package/src/package.js +43 -4
- package/src/person.js +2 -1
- package/src/pptx.js +323 -23
- package/src/question.js +10 -3
- package/src/quick-plan.js +13 -6
- package/src/quick.js +1 -1
- package/src/redline-check.js +12 -2
- package/src/redline-worker.js +10 -14
- package/src/reply.js +46 -19
- package/src/retain.js +223 -0
- package/src/review.js +32 -18
- package/src/rewrite-check.js +2 -2
- package/src/rewriter.js +38 -8
- package/src/sanitize.js +16 -3
- package/src/server-entry.js +4 -3
- package/src/server.js +175 -62
- package/src/session.js +1151 -210
- package/src/setup.js +20 -2
- package/src/stop-gate.js +34 -0
- package/src/suggest-reply.js +42 -7
- package/src/suggest.js +14 -29
- package/src/text.js +161 -11
- package/src/update.js +517 -0
- package/src/voice-guide.js +15 -9
- package/src/voice-learn.js +15 -4
- package/src/voice-sample-worker.js +7 -6
- package/src/voice-store.js +11 -4
- package/src/voice-worker.js +4 -6
- package/src/voice.js +2 -1
- package/src/worker-host.js +23 -0
- package/src/xlsx.js +378 -97
- package/src/xml-tree.js +6 -3
- package/src/xml.js +82 -34
- package/src/zip-write.js +10 -5
- package/src/zip.js +73 -18
- package/vendor/parse5/LICENSE.txt +32 -0
- package/vendor/parse5/parse5.mjs +7822 -0
- package/vendor/parse5.lock.json +23 -0
- package/web/dist/assets/{index-DS-WST1E.js → index-CjsTf5gj.js} +58 -52
- package/web/dist/assets/index-DnbLtHhG.css +1 -0
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/batch.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
|
-
import { PRODUCT } from "./constants.js";
|
|
4
|
+
import { PRODUCT, unknownFormat } from "./constants.js";
|
|
5
5
|
import { untrusted, userText } from "./sanitize.js";
|
|
6
6
|
import { pptxLocator } from "./pptx.js";
|
|
7
|
+
import { htmlLocator } from "./html.js";
|
|
7
8
|
import { GUIDE_MAX, SECTIONS, SHORT_MAX } from "./voice-guide.js";
|
|
8
9
|
import { xlsxLocator } from "./xlsx.js";
|
|
10
|
+
import { NOTE_MAX } from "./retain.js";
|
|
9
11
|
|
|
10
12
|
const REFERENCE = "The skill's SKILL.md and reference/ files (batch.md, ops.md, troubleshooting.md) hold the full rules and every op.";
|
|
11
13
|
|
|
@@ -32,6 +34,13 @@ const EXAMPLES = [
|
|
|
32
34
|
{ format: "xlsx", description: "Set the cell or range an anchor names; expect.text_hash is the anchor's locator.text_hash.", op: { op: "set_cells", target: { anchor_id: "a1" }, expect: { text_hash: "<locator.text_hash of a1>" }, values: "<new value or =formula>" } },
|
|
33
35
|
{ format: "pptx", description: "Replace text inside the shape an anchor names.", op: { op: "replace_text_in_shape", target: { anchor_id: "a1" }, expect: { text_hash: "<locator.text_hash of a1>" }, find: "<text inside the shape>", replace: "<new text>" } },
|
|
34
36
|
{ format: "docx", description: "Replace text inside the paragraph an anchor names.", op: { op: "replace_text", target: { anchor_id: "a1" }, expect: { text_hash: "<locator.text_hash of a1>" }, find: "<text inside the paragraph>", replace: "<new text>" } },
|
|
37
|
+
{ format: "html", description: "Replace text inside the element an anchor names (inside one text: not across a link or bold text).", op: { op: "replace_text", target: { anchor_id: "a1" }, expect: { text_hash: "<locator.text_hash of a1>" }, find: "<text inside the element>", replace: "<new text>" } },
|
|
38
|
+
{ format: "html", description: "Add markup after the element an anchor names (text, formatting, lists, tables, local images and links only).", op: { op: "insert_html", target: { anchor_id: "a1" }, expect: { text_hash: "<locator.text_hash of a1>" }, where: "after", html: "<p>New paragraph with a <a href=\"https://example.com\">link</a>.</p>" } },
|
|
39
|
+
];
|
|
40
|
+
// HTML (PLAN-html R1): what a write may add, and what the refusal looks like
|
|
41
|
+
const HTML_RULES = [
|
|
42
|
+
"HTML: code that runs in a browser cannot be added or changed (scripts, event handlers such as onclick, frames, forms, stylesheet links, remote images, data- attributes, URLs other than http, https, mailto, tel or relative); script and style elements cannot be edited. Such a write is refused whole with code active_content: \"The agent tried to add code that runs in a browser, so nothing was written.\"",
|
|
43
|
+
"HTML: replace_text works inside one text; a match across an element's edge answers spans_markup: use replace_element with the element's new markup.",
|
|
35
44
|
];
|
|
36
45
|
|
|
37
46
|
/** Static, machine-readable instructions for editing the file (no document text in it). The long rules live in the skill's reference files, not in every batch. */
|
|
@@ -52,6 +61,7 @@ function howToApply(file, batchId = "batch", format = null) {
|
|
|
52
61
|
`Or with apply (same engine as the browser): target ops by anchor id and give each an expect copied from what you read. ${REFERENCE}`,
|
|
53
62
|
`If you edit with other tools, never regenerate the file from scratch and never write the document's own path: edit a COPY, then run \`${PRODUCT.bin} commit ${shellQuote(file)} --from <copy> --lease <ticket>\` (the same checked write as apply; \`done\` flags a file that changed any other way).`,
|
|
54
63
|
"Anchors with protected: true cannot be edited losslessly; apply skips them (see protected_reason).",
|
|
64
|
+
...(format === "html" ? HTML_RULES : []),
|
|
55
65
|
],
|
|
56
66
|
examples: EXAMPLES.filter((e) => !format || e.format === format),
|
|
57
67
|
};
|
|
@@ -66,7 +76,9 @@ export function replyWithOf(anchor, format) {
|
|
|
66
76
|
if (!anchor.locator.text_hash) return "ops";
|
|
67
77
|
if (format === "xlsx") return "set";
|
|
68
78
|
if (format === "pptx") return anchor.unit === "shape" ? "find_replace_or_text" : "ops";
|
|
69
|
-
return "find_replace_or_text";
|
|
79
|
+
if (format === "docx") return "find_replace_or_text";
|
|
80
|
+
if (format === "html") return anchor.unit === "text" ? "find_replace_or_text" : "ops"; // an element that holds text (replace_text, set_text); an image or a container: ops
|
|
81
|
+
return unknownFormat(format, "replyWithOf");
|
|
70
82
|
}
|
|
71
83
|
|
|
72
84
|
/** A comment the person answered: a stored question and an index that is in range of its options. */
|
|
@@ -96,21 +108,25 @@ const MAX_VALUES = { rows: 20, cols: 10 };
|
|
|
96
108
|
const unitOfXlsx = (d) => (d.cell ? (d.r0 === d.r1 && d.c0 === d.c1 ? "cell" : "range") : /^\d+:\d+$/.test(d.range) ? "rows" : /^[A-Za-z]+:[A-Za-z]+$/.test(d.range) ? "columns" : "sheet");
|
|
97
109
|
|
|
98
110
|
/** Format-specific evidence of a durable anchor: locator, quote, unit and context. All strings are document-derived and sanitized by the caller. */
|
|
99
|
-
function evidenceOf(d, extra) {
|
|
111
|
+
function evidenceOf(d, extra, clean = (f) => untrusted(f, 100)) {
|
|
100
112
|
if (d?.format === "xlsx") {
|
|
101
|
-
|
|
113
|
+
// the anchor keeps its top-left cells (src/xlsx.js rangeOf); one stored by an earlier version has its whole text instead
|
|
114
|
+
const grid = Array.isArray(d.values) ? d.values : d.cell ? d.text.split("\n").map((r) => r.split("\t")) : null;
|
|
115
|
+
const values = d.cell ? grid.slice(0, MAX_VALUES.rows).map((r) => r.slice(0, MAX_VALUES.cols).map((v) => untrusted(v, 200))) : undefined;
|
|
102
116
|
const c = d.ctx;
|
|
103
117
|
return {
|
|
104
118
|
unit: unitOfXlsx(d),
|
|
105
119
|
locator: { ...xlsxLocator(d), text_hash: d.cell ? d.rangeHash : null },
|
|
106
120
|
quote: d.text,
|
|
107
121
|
...(values ? { values } : {}),
|
|
108
|
-
...(d.cell && d.types ? { cell_types: d.types, number_formats: d.formats.map((r) => r.map(
|
|
122
|
+
...(d.cell && d.types ? { cell_types: d.types, number_formats: d.formats.map((r) => r.map(clean)) } : {}),
|
|
109
123
|
...(c ? { context: { row_header: untrusted(c.rowHeader, 120), column_header: untrusted(c.colHeader, 120), left: untrusted(c.left, 120), right: untrusted(c.right, 120), above: untrusted(c.up, 120), below: untrusted(c.down, 120) } } : {}),
|
|
110
124
|
};
|
|
111
125
|
}
|
|
112
126
|
if (d?.format === "pptx") return { unit: d.kind, locator: pptxLocator(d), quote: d.quote || d.text, prefix: d.prefix, suffix: d.suffix };
|
|
113
|
-
return
|
|
127
|
+
if (d?.format === "html") return { unit: d.kind, locator: htmlLocator(d), quote: d.quote || d.text, prefix: d.prefix, suffix: d.suffix };
|
|
128
|
+
if (!d || d.format === undefined || d.format === "docx") return null; // Word's evidence is built by the caller (no anchor: unresolved)
|
|
129
|
+
return unknownFormat(d.format, "evidenceOf");
|
|
114
130
|
}
|
|
115
131
|
|
|
116
132
|
/**
|
|
@@ -124,9 +140,16 @@ export function buildBatch({ file, format, baseSha, note, comments, edits, style
|
|
|
124
140
|
const batchId = `b_${crypto.randomBytes(6).toString("hex")}`;
|
|
125
141
|
const anchors = [];
|
|
126
142
|
let n = 0;
|
|
143
|
+
// a number format is sanitized once per batch: 500 anchors on one range repeat the same 200 formats
|
|
144
|
+
const formats = new Map();
|
|
145
|
+
const cleanFormat = (f) => {
|
|
146
|
+
let v = formats.get(f);
|
|
147
|
+
if (v === undefined) formats.set(f, (v = untrusted(f, 100)));
|
|
148
|
+
return v;
|
|
149
|
+
};
|
|
127
150
|
const addAnchor = (d, extra = {}) => {
|
|
128
151
|
const id = `a${++n}`;
|
|
129
|
-
const ev = evidenceOf(d, extra);
|
|
152
|
+
const ev = evidenceOf(d, extra, cleanFormat);
|
|
130
153
|
anchors.push({
|
|
131
154
|
id,
|
|
132
155
|
kind: extra.kind || "paragraph",
|
|
@@ -172,7 +195,8 @@ export function buildBatch({ file, format, baseSha, note, comments, edits, style
|
|
|
172
195
|
format,
|
|
173
196
|
base_sha256: baseSha,
|
|
174
197
|
note: "user_edits are changes the user already made; they are in the file; do not redo or revert them",
|
|
175
|
-
|
|
198
|
+
// the note at NOTE_MAX: a waiting batch's note and the next one together are never cut here (longer is refused at Send, src/session.js)
|
|
199
|
+
instructions: { overall_note: userText(note, NOTE_MAX), ...(style ? { style_note: style } : {}), ...(guide ? { voice_guide: guide } : {}), ...(review ? { review: { kind: review.kind, ask: review.ask } } : {}), comments: instructionComments },
|
|
176
200
|
untrusted_document_evidence: { file_name: untrusted(path.basename(file)), anchors, user_edits: userEdits, ...(questions.length ? { questions } : {}), ...(units ? { units, ...(unitsMore ? { units_more: unitsMore } : {}) } : {}) },
|
|
177
201
|
how_to_apply: howToApply(file, batchId, format),
|
|
178
202
|
agent_rules: AGENT_RULES,
|
package/src/check-worker.js
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
import
|
|
1
|
+
import "./html-parser.js"; // an HTML file is read again at `done` (src/package.js htmlChecks): the parser lives in worker threads only
|
|
2
2
|
import { analyze } from "./analyze.js";
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
/** The `done` analysis (src/offload.js analyzeOffThread), run by src/worker-host.js. One message back: {ok, flags, checks, changes} or {ok: false, error}. */
|
|
5
|
+
export function work(data) {
|
|
6
|
+
try {
|
|
7
|
+
return { ok: true, ...analyze(Buffer.from(data.pre), Buffer.from(data.post), data.format, { deferSoffice: !!data.deferSoffice }) };
|
|
8
|
+
} catch (err) {
|
|
9
|
+
return { ok: false, error: String(err?.message || err) };
|
|
10
|
+
}
|
|
8
11
|
}
|
package/src/checker.js
CHANGED
|
@@ -5,6 +5,7 @@ import path from "node:path";
|
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { PRODUCT, env } from "./constants.js";
|
|
7
7
|
import { stateDir } from "./paths.js";
|
|
8
|
+
import { runInGroup } from "./update.js";
|
|
8
9
|
|
|
9
10
|
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
10
11
|
const CHECK_SCRIPT = path.join(here, "..", "scripts", "checker", "check.py");
|
|
@@ -22,6 +23,64 @@ export function findPython() {
|
|
|
22
23
|
return fs.existsSync(p) ? p : null;
|
|
23
24
|
}
|
|
24
25
|
|
|
26
|
+
const DEEP_KILL_GRACE_MS = 2000;
|
|
27
|
+
const DEEP_MAX_BUFFER = 16 * 1024 * 1024;
|
|
28
|
+
const checkerEnv = (dir) => ({ ...process.env, ONIONSKIN_SOFFICE_PID_FILE: path.join(dir, "soffice.pid") });
|
|
29
|
+
|
|
30
|
+
// check.py clears this file after conversion. A hard kill cannot run its cleanup handler.
|
|
31
|
+
function stopRemainingSoffice(dir) {
|
|
32
|
+
try {
|
|
33
|
+
const record = fs.readFileSync(path.join(dir, "soffice.pid"), "ascii");
|
|
34
|
+
if (!/^\d+\n$/.test(record)) return;
|
|
35
|
+
const pid = Number(record);
|
|
36
|
+
if (Number.isSafeInteger(pid) && pid > 1) process.kill(-pid, "SIGKILL");
|
|
37
|
+
} catch {}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Runs `cmd args...` with SIGTERM at timeoutMs and SIGKILL after graceMs so a process stuck in a native call cannot outlast the timeout.
|
|
41
|
+
const SYNC_RUNNER = `
|
|
42
|
+
const { spawn } = require("node:child_process");
|
|
43
|
+
const fs = require("node:fs");
|
|
44
|
+
const [metaFile, timeoutStr, graceStr, maxBufStr, cmd, ...args] = process.argv.slice(1);
|
|
45
|
+
const timeoutMs = Number(timeoutStr);
|
|
46
|
+
const graceMs = Number(graceStr);
|
|
47
|
+
const maxBuf = Number(maxBufStr);
|
|
48
|
+
let timedOut = false, killTimer = null, t = null, bytes = 0, overflow = false, done = false;
|
|
49
|
+
const write = (o) => { try { fs.writeFileSync(metaFile, JSON.stringify(o)); } catch {} };
|
|
50
|
+
const finish = (o) => {
|
|
51
|
+
if (done) return;
|
|
52
|
+
done = true;
|
|
53
|
+
clearTimeout(t); clearTimeout(killTimer);
|
|
54
|
+
write(o);
|
|
55
|
+
if (timedOut) { child.stdout.destroy(); child.stderr.destroy(); }
|
|
56
|
+
};
|
|
57
|
+
const child = spawn(cmd, args, { env: process.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
58
|
+
const feed = (stream, out) => stream.on("data", (d) => {
|
|
59
|
+
bytes += d.length;
|
|
60
|
+
if (bytes > maxBuf) {
|
|
61
|
+
if (!overflow) { overflow = true; try { child.kill("SIGKILL"); } catch {} }
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
out.write(d);
|
|
65
|
+
});
|
|
66
|
+
feed(child.stdout, process.stdout);
|
|
67
|
+
feed(child.stderr, process.stderr);
|
|
68
|
+
if (timeoutMs > 0) {
|
|
69
|
+
t = setTimeout(() => {
|
|
70
|
+
timedOut = true;
|
|
71
|
+
if (child.exitCode !== null || child.signalCode !== null) return finish({ timedOut });
|
|
72
|
+
try { child.kill("SIGTERM"); } catch {}
|
|
73
|
+
killTimer = setTimeout(() => { try { child.kill("SIGKILL"); } catch {} }, graceMs);
|
|
74
|
+
killTimer.unref?.();
|
|
75
|
+
}, timeoutMs);
|
|
76
|
+
}
|
|
77
|
+
child.once("error", (err) => finish({ timedOut, error: { code: err.code, message: err.message } }));
|
|
78
|
+
child.once("exit", (status, signal) => {
|
|
79
|
+
if (timedOut) finish({ timedOut, status, signal, error: overflow ? { code: "ENOBUFS", message: "maxBuffer exceeded" } : null });
|
|
80
|
+
});
|
|
81
|
+
child.once("close", (status, signal) => finish({ timedOut, status, signal, error: overflow ? { code: "ENOBUFS", message: "maxBuffer exceeded" } : null }));
|
|
82
|
+
`;
|
|
83
|
+
|
|
25
84
|
/**
|
|
26
85
|
* Runs scripts/checker/check.py on the two package versions. Never returns "clean" for a check that did not run:
|
|
27
86
|
* {ran:false, reason} when Python or its libraries are missing, and a `deep_check_failed` flag when it ran but broke.
|
|
@@ -35,26 +94,43 @@ export function runDeepCheck(preBytes, postBytes, format, { deferSoffice = false
|
|
|
35
94
|
try {
|
|
36
95
|
const pre = path.join(dir, `pre.${format}`);
|
|
37
96
|
const post = path.join(dir, `post.${format}`);
|
|
97
|
+
const metaFile = path.join(dir, "meta.json");
|
|
98
|
+
const timeoutMs = CHECK_TIMEOUT_MS();
|
|
38
99
|
fs.writeFileSync(pre, preBytes);
|
|
39
100
|
fs.writeFileSync(post, postBytes);
|
|
40
|
-
const r = spawnSync(
|
|
41
|
-
|
|
42
|
-
|
|
101
|
+
const r = spawnSync(
|
|
102
|
+
process.execPath,
|
|
103
|
+
["-e", SYNC_RUNNER, metaFile, String(timeoutMs), String(DEEP_KILL_GRACE_MS), String(DEEP_MAX_BUFFER), py, "-I", CHECK_SCRIPT, pre, post, format, ...(deferSoffice ? ["--no-soffice"] : [])],
|
|
104
|
+
{ encoding: "utf8", detached: true, timeout: timeoutMs > 0 ? timeoutMs + DEEP_KILL_GRACE_MS + 5000 : 0, maxBuffer: DEEP_MAX_BUFFER + 65536, env: checkerEnv(dir), killSignal: "SIGKILL" },
|
|
105
|
+
);
|
|
106
|
+
let meta = null;
|
|
107
|
+
try {
|
|
108
|
+
meta = JSON.parse(fs.readFileSync(metaFile, "utf8"));
|
|
109
|
+
} catch {}
|
|
110
|
+
if (meta?.timedOut || r.error?.code === "ETIMEDOUT") {
|
|
111
|
+
// Backstop for a stuck runner or descendants that still hold its pipes.
|
|
112
|
+
if (r.pid > 1) try { process.kill(-r.pid, "SIGKILL"); } catch {}
|
|
113
|
+
return { ran: false, code: "timed_out", flags: [{ code: "deep_check_failed", severity: "warn", detail: `timed out after ${timeoutMs / 1000}s` }], reason: "the deeper check timed out" };
|
|
114
|
+
}
|
|
115
|
+
const err = meta?.error || r.error;
|
|
116
|
+
if (err) return { ran: false, code: "failed", flags: [{ code: "deep_check_failed", severity: "warn", detail: err.message }], reason: "the deeper check could not start" };
|
|
117
|
+
const status = meta ? meta.status : r.status;
|
|
43
118
|
let j;
|
|
44
119
|
try {
|
|
45
120
|
j = JSON.parse(r.stdout);
|
|
46
121
|
} catch {
|
|
47
122
|
j = null;
|
|
48
123
|
}
|
|
49
|
-
if (
|
|
50
|
-
if (
|
|
51
|
-
const why = (j?.error || r.stderr || `exit ${
|
|
124
|
+
if (status === 3) return { ran: false, flags: [], code: "libraries_missing", reason: `${py} lacks lxml/defusedxml: run \`${PRODUCT.bin} setup-checker\`` };
|
|
125
|
+
if (status !== 0 || !j || !Array.isArray(j.flags)) {
|
|
126
|
+
const why = (j?.error || r.stderr || `exit ${status}`).toString().slice(0, 300);
|
|
52
127
|
return { ran: false, code: "failed", flags: [{ code: "deep_check_failed", severity: "warn", detail: why }], reason: "the deeper check failed to run" };
|
|
53
128
|
}
|
|
54
129
|
const skipped = j.skipped || [];
|
|
55
130
|
const pending = skipped.some((x) => x.check === "soffice-convert" && x.pending);
|
|
56
131
|
return { ran: true, flags: j.flags, checksRun: j.ran, skipped: pending ? skipped.filter((x) => !x.pending) : skipped, sofficePending: pending, python: py };
|
|
57
132
|
} finally {
|
|
133
|
+
stopRemainingSoffice(dir);
|
|
58
134
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
59
135
|
}
|
|
60
136
|
}
|
|
@@ -71,6 +147,36 @@ const lateNext = () => {
|
|
|
71
147
|
if (start) start();
|
|
72
148
|
};
|
|
73
149
|
|
|
150
|
+
const KILL_GRACE_MS = 5000;
|
|
151
|
+
|
|
152
|
+
/** Sends SIGTERM to a live child and schedules an unref'd SIGKILL fallback so a hung child cannot stall the queue or leak temp dirs. */
|
|
153
|
+
function gracefulKill(getChild) {
|
|
154
|
+
let killTimer = null;
|
|
155
|
+
const terminate = () => {
|
|
156
|
+
const child = getChild();
|
|
157
|
+
if (!child || child.exitCode !== null || child.signalCode !== null) return;
|
|
158
|
+
try {
|
|
159
|
+
child.kill("SIGTERM");
|
|
160
|
+
} catch {}
|
|
161
|
+
if (!killTimer) {
|
|
162
|
+
killTimer = setTimeout(() => {
|
|
163
|
+
const c = getChild();
|
|
164
|
+
if (c && c.exitCode === null && c.signalCode === null) {
|
|
165
|
+
try {
|
|
166
|
+
c.kill("SIGKILL");
|
|
167
|
+
} catch {}
|
|
168
|
+
}
|
|
169
|
+
}, KILL_GRACE_MS);
|
|
170
|
+
killTimer.unref?.();
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
const clear = () => {
|
|
174
|
+
clearTimeout(killTimer);
|
|
175
|
+
killTimer = null;
|
|
176
|
+
};
|
|
177
|
+
return { terminate, clear };
|
|
178
|
+
}
|
|
179
|
+
|
|
74
180
|
/**
|
|
75
181
|
* The LibreOffice conversion check for a version `done` already accepted, run in the background (check.py --soffice-only). Never blocks:
|
|
76
182
|
* `onDone({ran, flags, skipped, reason})` is called exactly once, also when the check fails, times out, is cancelled or never got its turn
|
|
@@ -85,17 +191,27 @@ export function startLateSoffice(preBytes, postBytes, format, onDone) {
|
|
|
85
191
|
let child = null;
|
|
86
192
|
let timer = null;
|
|
87
193
|
let dir = null;
|
|
194
|
+
const killer = gracefulKill(() => child);
|
|
195
|
+
const cleanupDir = () => {
|
|
196
|
+
if (!dir) return;
|
|
197
|
+
const d = dir;
|
|
198
|
+
dir = null;
|
|
199
|
+
stopRemainingSoffice(d);
|
|
200
|
+
fs.rmSync(d, { recursive: true, force: true });
|
|
201
|
+
};
|
|
88
202
|
const finish = (r) => {
|
|
89
203
|
if (settled) return;
|
|
90
204
|
settled = true;
|
|
91
205
|
clearTimeout(timer);
|
|
92
|
-
if (dir) fs.rmSync(dir, { recursive: true, force: true });
|
|
93
206
|
// the next one starts only when this one's process is really gone (a cancelled or timed-out check takes a moment to die)
|
|
94
207
|
if (started && child?.pid && child.exitCode === null && child.signalCode === null) child.once("close", lateNext);
|
|
95
|
-
else if (started) lateNext();
|
|
96
208
|
else {
|
|
97
|
-
|
|
98
|
-
if (
|
|
209
|
+
cleanupDir();
|
|
210
|
+
if (started) lateNext();
|
|
211
|
+
else {
|
|
212
|
+
const at = lateQueue.indexOf(start);
|
|
213
|
+
if (at >= 0) lateQueue.splice(at, 1);
|
|
214
|
+
}
|
|
99
215
|
}
|
|
100
216
|
try {
|
|
101
217
|
onDone(r);
|
|
@@ -103,7 +219,7 @@ export function startLateSoffice(preBytes, postBytes, format, onDone) {
|
|
|
103
219
|
};
|
|
104
220
|
const cancel = (reason = "the check was cancelled") => {
|
|
105
221
|
if (settled) return;
|
|
106
|
-
|
|
222
|
+
killer.terminate();
|
|
107
223
|
finish({ ran: false, flags: [], skipped: [], reason });
|
|
108
224
|
};
|
|
109
225
|
function start() {
|
|
@@ -115,15 +231,20 @@ export function startLateSoffice(preBytes, postBytes, format, onDone) {
|
|
|
115
231
|
const post = path.join(dir, `post.${format}`);
|
|
116
232
|
fs.writeFileSync(pre, preBytes);
|
|
117
233
|
fs.writeFileSync(post, postBytes);
|
|
118
|
-
child = spawn(py, ["-I", CHECK_SCRIPT, pre, post, format, "--soffice-only"], { env:
|
|
234
|
+
child = spawn(py, ["-I", CHECK_SCRIPT, pre, post, format, "--soffice-only"], { env: checkerEnv(dir), stdio: ["ignore", "pipe", "pipe"] });
|
|
119
235
|
} catch (err) {
|
|
120
236
|
return finish({ ran: false, flags: [], skipped: [], reason: `could not start: ${err.message}` });
|
|
121
237
|
}
|
|
122
238
|
let out = "";
|
|
123
239
|
child.stdout.on("data", (d) => (out = (out + d).slice(0, 16 * 1024 * 1024)));
|
|
124
240
|
child.stderr.resume();
|
|
125
|
-
child.once("error", (err) =>
|
|
241
|
+
child.once("error", (err) => {
|
|
242
|
+
killer.clear();
|
|
243
|
+
finish({ ran: false, flags: [], skipped: [], reason: `could not start: ${err.message}` });
|
|
244
|
+
});
|
|
126
245
|
child.once("close", (code) => {
|
|
246
|
+
killer.clear();
|
|
247
|
+
cleanupDir();
|
|
127
248
|
let j = null;
|
|
128
249
|
try {
|
|
129
250
|
j = JSON.parse(out);
|
|
@@ -132,7 +253,7 @@ export function startLateSoffice(preBytes, postBytes, format, onDone) {
|
|
|
132
253
|
else finish({ ran: false, flags: [], skipped: [], reason: (j?.error || `exit ${code}`).toString().slice(0, 200) });
|
|
133
254
|
});
|
|
134
255
|
timer = setTimeout(() => {
|
|
135
|
-
|
|
256
|
+
killer.terminate();
|
|
136
257
|
finish({ ran: false, flags: [], skipped: [], reason: `timed out after ${CHECK_TIMEOUT_MS() / 1000}s` });
|
|
137
258
|
}, CHECK_TIMEOUT_MS());
|
|
138
259
|
}
|
|
@@ -144,20 +265,37 @@ export function startLateSoffice(preBytes, postBytes, format, onDone) {
|
|
|
144
265
|
}
|
|
145
266
|
|
|
146
267
|
/** `onionskin setup-checker`: the one place that uses the network (pip). Creates <state dir>/venv. */
|
|
147
|
-
export function setupChecker(log = (s) => process.stderr.write(s + "\n")) {
|
|
268
|
+
export async function setupChecker(log = (s) => process.stderr.write(s + "\n")) {
|
|
148
269
|
const base = process.env[`${PRODUCT.envPrefix}_BASE_PYTHON`] || "python3";
|
|
149
270
|
fs.mkdirSync(stateDir(), { recursive: true, mode: 0o700 });
|
|
271
|
+
const deadline = Date.now() + SETUP_TIMEOUT_MS();
|
|
272
|
+
const runStep = async (cmd, args) => {
|
|
273
|
+
const remaining = deadline - Date.now();
|
|
274
|
+
let expired = remaining <= 0;
|
|
275
|
+
// The installer may itself receive SIGKILL shortly after TERM. End this step's group before that happens.
|
|
276
|
+
const status = expired ? 1 : await runInGroup(cmd, args, { timeout: remaining, signalGraceMs: 0, onTimeout: () => { expired = true; } });
|
|
277
|
+
if (expired) throw new Error("Deeper check setup timed out.");
|
|
278
|
+
if (status === 129 || status === 130 || status === 143) throw new Error("Deeper check setup was stopped.");
|
|
279
|
+
return status;
|
|
280
|
+
};
|
|
281
|
+
const imports = ["-c", "import lxml, docx, openpyxl, pptx, defusedxml"];
|
|
282
|
+
if (env("SETUP_IF_NEEDED") === "1") {
|
|
283
|
+
if (fs.existsSync(venvPython()) && await runStep(venvPython(), imports) === 0) return venvPython();
|
|
284
|
+
if (!env("BASE_PYTHON") && process.platform === "darwin") {
|
|
285
|
+
const guard = '[ "$(command -v python3)" != /usr/bin/python3 ] || xcode-select -p >/dev/null 2>&1';
|
|
286
|
+
if (await runStep("/bin/sh", ["-c", guard]) !== 0) throw new Error("Python needs the macOS developer tools, which are not installed.");
|
|
287
|
+
}
|
|
288
|
+
}
|
|
150
289
|
const steps = [
|
|
151
290
|
[base, ["-m", "venv", venvDir()]],
|
|
152
|
-
[venvPython(), ["-m", "pip", "install", "--disable-pip-version-check", "--quiet", "-r", CHECKER_REQUIREMENTS]],
|
|
291
|
+
[venvPython(), ["-m", "pip", "install", "--disable-pip-version-check", "--quiet", "--require-hashes", "--only-binary=:all:", "-r", CHECKER_REQUIREMENTS]],
|
|
153
292
|
];
|
|
154
293
|
for (const [cmd, args] of steps) {
|
|
155
294
|
log(`$ ${path.basename(cmd)} ${args.slice(0, 3).join(" ")} ...`);
|
|
156
|
-
const
|
|
157
|
-
if (
|
|
295
|
+
const status = await runStep(cmd, args);
|
|
296
|
+
if (status !== 0) throw new Error(`${cmd} ${args.slice(0, 2).join(" ")} failed (exit ${status})`);
|
|
158
297
|
}
|
|
159
|
-
|
|
160
|
-
if (probe.status !== 0) throw new Error(`the venv was created but the libraries do not import: ${probe.stderr}`);
|
|
298
|
+
if (await runStep(venvPython(), imports) !== 0) throw new Error("the venv was created but the libraries do not import");
|
|
161
299
|
return venvPython();
|
|
162
300
|
}
|
|
163
301
|
|
|
@@ -165,7 +303,8 @@ export function setupChecker(log = (s) => process.stderr.write(s + "\n")) {
|
|
|
165
303
|
// The server must keep answering while pip runs (about a minute), so this variant spawns instead of spawnSync. One job at a time for the
|
|
166
304
|
// whole server: a second start while one runs gets the same promise. The job's outcome is kept until the next start so the page can poll it.
|
|
167
305
|
const SETUP_TIMEOUT_MS = () => Number(env("SETUP_TIMEOUT_MS", 10 * 60_000));
|
|
168
|
-
export const NETWORK_WORDS = /could not resolve|name or service not known|nodename nor servname|temporary failure in name resolution|failed to establish a new connection|connection (?:error|refused|reset|timed out)|network is unreachable|
|
|
306
|
+
export const NETWORK_WORDS = /could not (?:resolve|fetch url)|name or service not known|nodename nor servname|temporary failure in name resolution|failed to establish a new connection|newconnectionerror|connection (?:error|refused|reset|timed out)|network is unreachable|max retries exceeded|read timed out|ssl|proxy/i;
|
|
307
|
+
export const UNSUPPORTED_PYTHON_WORDS = /requires-python|ignored the following versions/i;
|
|
169
308
|
let job = null; // {promise} while running; {outcome: {reason, detail}} after a failure; null when idle or after a success
|
|
170
309
|
|
|
171
310
|
/** One command, async. Resolves {status, stderr (last 4000 bytes), error}; never rejects, never inherits the server's stdio. */
|
|
@@ -192,6 +331,36 @@ export function run(cmd, args, timeoutMs) {
|
|
|
192
331
|
});
|
|
193
332
|
}
|
|
194
333
|
|
|
334
|
+
/** Returns e.g. "Python 3.9.6" or "" if the interpreter cannot be queried. */
|
|
335
|
+
function pythonVersion(py) {
|
|
336
|
+
return new Promise((resolve) => {
|
|
337
|
+
let out = "";
|
|
338
|
+
let settled = false;
|
|
339
|
+
const done = (v) => {
|
|
340
|
+
if (settled) return;
|
|
341
|
+
settled = true;
|
|
342
|
+
clearTimeout(timer);
|
|
343
|
+
resolve(v);
|
|
344
|
+
};
|
|
345
|
+
let child;
|
|
346
|
+
try {
|
|
347
|
+
child = spawn(py, ["--version"], { stdio: ["ignore", "pipe", "pipe"], env: process.env });
|
|
348
|
+
} catch {
|
|
349
|
+
return resolve("");
|
|
350
|
+
}
|
|
351
|
+
const timer = setTimeout(() => {
|
|
352
|
+
try {
|
|
353
|
+
child.kill("SIGKILL");
|
|
354
|
+
} catch {}
|
|
355
|
+
done("");
|
|
356
|
+
}, 5000);
|
|
357
|
+
child.stdout.on("data", (d) => (out += d));
|
|
358
|
+
child.stderr.on("data", (d) => (out += d));
|
|
359
|
+
child.once("error", () => done(""));
|
|
360
|
+
child.once("close", (status) => done(status === 0 ? out.trim().slice(0, 80) : ""));
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
|
|
195
364
|
/** Do the venv's libraries import? (The one probe both setup and the status use: a venv whose pip said ok but whose libraries do not import is not ready.) */
|
|
196
365
|
async function librariesImport(py) {
|
|
197
366
|
return (await run(py, ["-c", "import lxml, docx, openpyxl, pptx, defusedxml"], 30_000)).status === 0;
|
|
@@ -210,10 +379,18 @@ async function setupSteps() {
|
|
|
210
379
|
const missing = venv.error?.code === "ENOENT";
|
|
211
380
|
throw Object.assign(new Error(`${base} -m venv failed (${venv.error?.message || `exit ${venv.status}`}) ${venv.stderr}`.trim()), { reason: missing ? "no_python" : "failed" });
|
|
212
381
|
}
|
|
213
|
-
const pip = await run(venvPython(), ["-m", "pip", "install", "--disable-pip-version-check", "--quiet", "-r", CHECKER_REQUIREMENTS], SETUP_TIMEOUT_MS());
|
|
382
|
+
const pip = await run(venvPython(), ["-m", "pip", "install", "--disable-pip-version-check", "--quiet", "--require-hashes", "--only-binary=:all:", "-r", CHECKER_REQUIREMENTS], SETUP_TIMEOUT_MS());
|
|
214
383
|
if (pip.status !== 0) {
|
|
215
384
|
const text = `pip install failed (${pip.error?.message || `exit ${pip.status}`}) ${pip.stderr}`.trim();
|
|
216
|
-
|
|
385
|
+
if (NETWORK_WORDS.test(text)) {
|
|
386
|
+
throw Object.assign(new Error(text), { reason: "no_network" });
|
|
387
|
+
}
|
|
388
|
+
if (UNSUPPORTED_PYTHON_WORDS.test(text)) {
|
|
389
|
+
const ver = await pythonVersion(venvPython());
|
|
390
|
+
const detail = ver ? `this Python version or platform (${ver}) is not supported: ${text}` : `this Python version or platform is not supported: ${text}`;
|
|
391
|
+
throw Object.assign(new Error(detail), { reason: "unsupported_python", pythonVersion: ver || undefined });
|
|
392
|
+
}
|
|
393
|
+
throw Object.assign(new Error(text), { reason: "failed" });
|
|
217
394
|
}
|
|
218
395
|
if (!(await librariesImport(venvPython()))) throw Object.assign(new Error("the venv was created but the libraries do not import"), { reason: "failed" });
|
|
219
396
|
// runs use findPython(): an ONIONSKIN_PYTHON that points elsewhere would keep the checker off after a "successful" setup
|
|
@@ -224,7 +401,7 @@ async function setupSteps() {
|
|
|
224
401
|
|
|
225
402
|
/**
|
|
226
403
|
* Starts the setup (or joins the one that is running) and returns its promise. It resolves to the venv's python, or rejects with an error that
|
|
227
|
-
* carries `reason`: no_python | no_network | failed. The caller does not need to await it; setupCheckerStatus() reports it.
|
|
404
|
+
* carries `reason`: no_python | no_network | unsupported_python | failed. The caller does not need to await it; setupCheckerStatus() reports it.
|
|
228
405
|
*/
|
|
229
406
|
export function startSetupChecker() {
|
|
230
407
|
if (job?.promise) return job.promise;
|
|
@@ -232,7 +409,13 @@ export function startSetupChecker() {
|
|
|
232
409
|
mine.promise = setupSteps().then(
|
|
233
410
|
(py) => ((job = null), py),
|
|
234
411
|
(err) => {
|
|
235
|
-
job = {
|
|
412
|
+
job = {
|
|
413
|
+
outcome: {
|
|
414
|
+
reason: err.reason || "failed",
|
|
415
|
+
detail: String(err.message).slice(0, 2000),
|
|
416
|
+
...(err.pythonVersion ? { pythonVersion: err.pythonVersion } : {}),
|
|
417
|
+
},
|
|
418
|
+
};
|
|
236
419
|
throw err;
|
|
237
420
|
},
|
|
238
421
|
);
|
|
@@ -264,23 +447,45 @@ export function deepRedline(preBytes, postBytes, outBytes) {
|
|
|
264
447
|
const py = findPython();
|
|
265
448
|
if (!py) return Promise.resolve({ ran: false, reason: "not_set_up" });
|
|
266
449
|
return new Promise((resolve) => {
|
|
267
|
-
|
|
450
|
+
let dir = fs.mkdtempSync(path.join(os.tmpdir(), "onionskin-redline-"));
|
|
268
451
|
const files = ["pre", "post", "out"].map((n) => path.join(dir, `${n}.docx`));
|
|
269
|
-
[preBytes, postBytes, outBytes].forEach((b, k) => fs.writeFileSync(files[k], b));
|
|
270
452
|
let out = "";
|
|
271
453
|
let settled = false;
|
|
454
|
+
let child = null;
|
|
455
|
+
let timer = null;
|
|
456
|
+
const killer = gracefulKill(() => child);
|
|
457
|
+
const cleanupDir = () => {
|
|
458
|
+
if (!dir) return;
|
|
459
|
+
const d = dir;
|
|
460
|
+
dir = null;
|
|
461
|
+
stopRemainingSoffice(d);
|
|
462
|
+
fs.rmSync(d, { recursive: true, force: true });
|
|
463
|
+
};
|
|
272
464
|
const finish = (r) => {
|
|
273
465
|
if (settled) return;
|
|
274
466
|
settled = true;
|
|
275
467
|
clearTimeout(timer);
|
|
276
|
-
|
|
468
|
+
if (!child?.pid || child.exitCode !== null || child.signalCode !== null) cleanupDir();
|
|
277
469
|
resolve(r);
|
|
278
470
|
};
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
471
|
+
try {
|
|
472
|
+
[preBytes, postBytes, outBytes].forEach((b, k) => fs.writeFileSync(files[k], b));
|
|
473
|
+
child = spawn(py, ["-I", CHECK_SCRIPT, "--redline", ...files], { env: checkerEnv(dir), stdio: ["ignore", "pipe", "ignore"] });
|
|
474
|
+
} catch {
|
|
475
|
+
return finish({ ran: true, ok: false, why: "the deeper check could not start" });
|
|
476
|
+
}
|
|
477
|
+
timer = setTimeout(() => {
|
|
478
|
+
killer.terminate();
|
|
479
|
+
finish({ ran: true, ok: false, why: "the deeper check timed out" });
|
|
480
|
+
}, CHECK_TIMEOUT_MS());
|
|
481
|
+
child.stdout.on("data", (c) => (out += c).length > 1e6 && killer.terminate());
|
|
482
|
+
child.once("error", () => {
|
|
483
|
+
killer.clear();
|
|
484
|
+
finish({ ran: true, ok: false, why: "the deeper check could not start" });
|
|
485
|
+
});
|
|
486
|
+
child.once("close", (code) => {
|
|
487
|
+
killer.clear();
|
|
488
|
+
cleanupDir();
|
|
284
489
|
if (code === 3) return finish({ ran: false, reason: "libraries_missing" });
|
|
285
490
|
let j = null;
|
|
286
491
|
try {
|