onionskin-cli 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +98 -0
- package/README.md +4 -4
- package/package.json +1 -1
- package/scripts/checker/check.py +24 -5
- package/scripts/checker/requirements.txt +482 -5
- package/scripts/requirements.txt +482 -5
- package/scripts/rewriter/apple/main.swift +1 -1
- package/scripts/rewriter/apple/onionskin-fm +0 -0
- package/scripts/rewriter/apple/onionskin-fm.json +1 -1
- package/skills/onionskin/SKILL.md +4 -4
- package/skills/onionskin/reference/ops.md +1 -1
- package/src/agent-runner.js +94 -15
- package/src/checker.js +238 -33
- package/src/cli.js +173 -33
- package/src/doctor.js +12 -3
- package/src/docx.js +130 -63
- package/src/fsutil.js +13 -5
- package/src/history.js +11 -2
- package/src/html-render.js +74 -15
- package/src/html-utf8.js +1 -1
- package/src/html.js +3 -1
- package/src/merge-package.js +82 -26
- package/src/offload.js +25 -6
- package/src/ops/docx-table.js +5 -3
- package/src/ops/pptx.js +8 -2
- package/src/ops/xlsx.js +114 -8
- package/src/package.js +17 -2
- package/src/pptx.js +244 -8
- package/src/question.js +10 -3
- package/src/quick.js +1 -1
- package/src/redline-check.js +12 -2
- package/src/rewriter.js +38 -8
- package/src/server-entry.js +4 -3
- package/src/server.js +97 -28
- package/src/session.js +13 -2
- package/src/setup.js +20 -2
- package/src/update.js +248 -19
- package/src/voice-guide.js +12 -7
- package/src/voice-learn.js +11 -1
- package/src/voice-store.js +11 -4
- package/src/voice.js +2 -1
- package/src/xlsx.js +25 -5
- package/src/zip-write.js +8 -4
- package/src/zip.js +64 -13
- package/web/dist/assets/{index-DytKkMdi.js → index-CjsTf5gj.js} +108 -106
- package/web/dist/assets/index-DnbLtHhG.css +1 -0
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-DkNx0LSH.css +0 -1
package/src/agent-runner.js
CHANGED
|
@@ -18,9 +18,10 @@ import { execFile as nodeExecFile, spawn as nodeSpawn } from "node:child_process
|
|
|
18
18
|
import crypto from "node:crypto";
|
|
19
19
|
import fs from "node:fs";
|
|
20
20
|
import path from "node:path";
|
|
21
|
+
import { StringDecoder } from "node:string_decoder";
|
|
21
22
|
import { fileURLToPath } from "node:url";
|
|
22
23
|
import { env as cfg } from "./constants.js";
|
|
23
|
-
import { stateDir, sub, writeStateFile } from "./paths.js";
|
|
24
|
+
import { stateDir, sub, writeFully, writeStateFile } from "./paths.js";
|
|
24
25
|
import * as settings from "./settings.js";
|
|
25
26
|
import { DEFAULT_MODEL, POLL_TIMEOUT, agentsDisabled, bgHolder, buildArgv, cleanEnv, findClaude, findCodex, isBgHolder, preflight } from "./agent-detect.js";
|
|
26
27
|
import { backgroundPrompt, readSkill, refDir, resumePrompt } from "./agent-prompt.js";
|
|
@@ -55,12 +56,31 @@ const fallbackEvents = {
|
|
|
55
56
|
const loadedEvents = await import("./agent-events.js").catch(() => fallbackEvents);
|
|
56
57
|
|
|
57
58
|
const HOUR = 60 * 60 * 1000;
|
|
59
|
+
const TRANSCRIPT_MAX = 1024 * 1024;
|
|
58
60
|
export const LIMITS = { maxAgents: 3, startsPerHour: 5, graceMs: 20_000, idleMs: 20 * 60 * 1000, killMs: 4000, closeGraceMs: 10_000, activity: 50 };
|
|
59
61
|
const backoffMs = (n) => 1000 * 2 ** Math.min(Math.max(n, 1), 5); // 2, 4, 8, 16, 32 s
|
|
60
62
|
|
|
61
63
|
const agentsDir = () => sub("agents");
|
|
62
64
|
const recordFile = (key) => path.join(agentsDir(), `${key}.json`);
|
|
63
65
|
|
|
66
|
+
// On overflow keep a bounded tail, starting on a line boundary when newer events are available.
|
|
67
|
+
function appendTranscript(fd, chunk) {
|
|
68
|
+
const size = fs.fstatSync(fd).size;
|
|
69
|
+
if (size + chunk.length > TRANSCRIPT_MAX) {
|
|
70
|
+
if (chunk.length >= TRANSCRIPT_MAX) chunk = chunk.subarray(-TRANSCRIPT_MAX);
|
|
71
|
+
else {
|
|
72
|
+
const keep = Math.min(TRANSCRIPT_MAX / 2, TRANSCRIPT_MAX - chunk.length);
|
|
73
|
+
const tail = Buffer.alloc(keep);
|
|
74
|
+
const read = fs.readSync(fd, tail, 0, keep, size - keep);
|
|
75
|
+
chunk = Buffer.concat([tail.subarray(0, read), chunk]);
|
|
76
|
+
}
|
|
77
|
+
const newline = chunk.indexOf(10);
|
|
78
|
+
if (newline >= 0 && newline < chunk.length - 1) chunk = chunk.subarray(newline + 1);
|
|
79
|
+
fs.ftruncateSync(fd, 0);
|
|
80
|
+
}
|
|
81
|
+
writeFully(fd, chunk);
|
|
82
|
+
}
|
|
83
|
+
|
|
64
84
|
/**
|
|
65
85
|
* deps: {spawn, execFile, kill(pid, sig), now, setTimeout, env, events, tabs(key) -> number of open tabs, open(file) -> opens a .command
|
|
66
86
|
* file in Terminal, limits}. Everything that starts, signals or waits is injectable, so tests run with no process and a fake clock.
|
|
@@ -81,7 +101,7 @@ export function agentRunner(sessions, deps = {}) {
|
|
|
81
101
|
const entry = (key) => {
|
|
82
102
|
let e = entries.get(key);
|
|
83
103
|
if (!e) {
|
|
84
|
-
e = { key, holder: bgHolder(key), child: null, state: "none", failure: null, activity: [], starts: [], crashes: 0, nextStartAt: 0, strandedAt: 0, idleSince: 0, waiting: false, external: false, sessionId: null, resumable: false, lastBatch: null, resumeNext: false };
|
|
104
|
+
e = { key, holder: bgHolder(key), child: null, gen: 0, state: "none", failure: null, activity: [], starts: [], crashes: 0, nextStartAt: 0, strandedAt: 0, idleSince: 0, waiting: false, external: false, sessionId: null, resumable: false, lastBatch: null, resumeNext: false };
|
|
85
105
|
entries.set(key, e);
|
|
86
106
|
}
|
|
87
107
|
return e;
|
|
@@ -171,6 +191,7 @@ export function agentRunner(sessions, deps = {}) {
|
|
|
171
191
|
}
|
|
172
192
|
|
|
173
193
|
async function launch(e, { counted, resume = false }) {
|
|
194
|
+
const gen = e.gen;
|
|
174
195
|
const s = sessionOf(e.key);
|
|
175
196
|
if (!s || s.closed) {
|
|
176
197
|
e.state = "none";
|
|
@@ -199,6 +220,7 @@ export function agentRunner(sessions, deps = {}) {
|
|
|
199
220
|
const binDir = ensureBin();
|
|
200
221
|
const childEnv = cleanEnv({ binDir, stateDir: fs.realpathSync(stateDir()), holder: e.holder, workDir, env });
|
|
201
222
|
const pre = await preflight(bin, { env: childEnv, execFile });
|
|
223
|
+
if (e.gen !== gen || e.state === "handover") return;
|
|
202
224
|
if (pre) return fail(e, pre.code, pre.update ? { update: true } : {});
|
|
203
225
|
if (!sessionOf(e.key) || sessionOf(e.key).closed || stoppingAll) {
|
|
204
226
|
e.state = "none";
|
|
@@ -217,7 +239,7 @@ export function agentRunner(sessions, deps = {}) {
|
|
|
217
239
|
} catch (err) {
|
|
218
240
|
fail(e, "crashed");
|
|
219
241
|
} finally {
|
|
220
|
-
e.launching = false;
|
|
242
|
+
if (e.gen === gen) e.launching = false;
|
|
221
243
|
}
|
|
222
244
|
}
|
|
223
245
|
|
|
@@ -266,34 +288,83 @@ export function agentRunner(sessions, deps = {}) {
|
|
|
266
288
|
});
|
|
267
289
|
let transcript = null;
|
|
268
290
|
try {
|
|
269
|
-
transcript = fs.openSync(path.join(workDir, "transcript.jsonl"), "a", 0o600);
|
|
291
|
+
transcript = fs.openSync(path.join(workDir, "transcript.jsonl"), "a+", 0o600);
|
|
292
|
+
appendTranscript(transcript, Buffer.alloc(0));
|
|
270
293
|
} catch {}
|
|
271
|
-
|
|
272
|
-
child.stdout?.on("data", (d) => {
|
|
294
|
+
const closeTranscript = () => {
|
|
273
295
|
if (transcript !== null) {
|
|
274
296
|
try {
|
|
275
|
-
fs.
|
|
297
|
+
fs.closeSync(transcript);
|
|
276
298
|
} catch {}
|
|
299
|
+
transcript = null;
|
|
277
300
|
}
|
|
278
|
-
|
|
301
|
+
};
|
|
302
|
+
const decoder = new StringDecoder("utf8");
|
|
303
|
+
let buf = "";
|
|
304
|
+
const flushLines = () => {
|
|
279
305
|
let i;
|
|
280
306
|
while ((i = buf.indexOf("\n")) >= 0) {
|
|
281
307
|
const line = buf.slice(0, i);
|
|
282
308
|
buf = buf.slice(i + 1);
|
|
283
309
|
onLine(e, line);
|
|
284
310
|
}
|
|
285
|
-
}
|
|
286
|
-
child.
|
|
287
|
-
child.on("error", () => {});
|
|
288
|
-
child.on("exit", (code, signal) => {
|
|
311
|
+
};
|
|
312
|
+
child.stdout?.on("data", (d) => {
|
|
289
313
|
if (transcript !== null) {
|
|
290
314
|
try {
|
|
291
|
-
|
|
315
|
+
appendTranscript(transcript, d);
|
|
292
316
|
} catch {}
|
|
293
317
|
}
|
|
318
|
+
buf += decoder.write(d);
|
|
319
|
+
flushLines();
|
|
320
|
+
});
|
|
321
|
+
child.stderr?.on("data", () => {}); // drained; Claude Code writes nothing there that the page needs
|
|
322
|
+
let finished = false;
|
|
323
|
+
let exitTimer = null;
|
|
324
|
+
const finish = (code, signal) => {
|
|
325
|
+
if (finished) return;
|
|
326
|
+
finished = true;
|
|
327
|
+
if (exitTimer) {
|
|
328
|
+
clearTimeout(exitTimer);
|
|
329
|
+
exitTimer = null;
|
|
330
|
+
}
|
|
331
|
+
closeTranscript();
|
|
332
|
+
buf += decoder.end();
|
|
333
|
+
flushLines();
|
|
294
334
|
if (buf) onLine(e, buf);
|
|
335
|
+
buf = "";
|
|
295
336
|
if (e.child === child) onExit(e, code, signal);
|
|
337
|
+
};
|
|
338
|
+
child.on("error", () => {
|
|
339
|
+
if (finished) return;
|
|
340
|
+
finished = true;
|
|
341
|
+
if (exitTimer) {
|
|
342
|
+
clearTimeout(exitTimer);
|
|
343
|
+
exitTimer = null;
|
|
344
|
+
}
|
|
345
|
+
closeTranscript();
|
|
346
|
+
if (e.child === child) {
|
|
347
|
+
e.child = null;
|
|
348
|
+
fs.rmSync(recordFile(e.key), { force: true });
|
|
349
|
+
fail(e, "crashed");
|
|
350
|
+
startWaiting();
|
|
351
|
+
}
|
|
296
352
|
});
|
|
353
|
+
child.on("exit", (code, signal) => {
|
|
354
|
+
if (finished || exitTimer) return;
|
|
355
|
+
exitTimer = setTimeout(() => {
|
|
356
|
+
exitTimer = null;
|
|
357
|
+
try {
|
|
358
|
+
child.stdout?.destroy();
|
|
359
|
+
} catch {}
|
|
360
|
+
try {
|
|
361
|
+
child.stderr?.destroy();
|
|
362
|
+
} catch {}
|
|
363
|
+
finish(code, signal);
|
|
364
|
+
}, 500);
|
|
365
|
+
exitTimer.unref();
|
|
366
|
+
});
|
|
367
|
+
child.on("close", (code, signal) => finish(code, signal));
|
|
297
368
|
e.state = "starting";
|
|
298
369
|
notify(e.key);
|
|
299
370
|
}
|
|
@@ -399,6 +470,8 @@ export function agentRunner(sessions, deps = {}) {
|
|
|
399
470
|
async function stop(key, why = "stop") {
|
|
400
471
|
const e = entries.get(key);
|
|
401
472
|
if (!e) return false;
|
|
473
|
+
e.gen++;
|
|
474
|
+
e.launching = false;
|
|
402
475
|
e.waiting = false;
|
|
403
476
|
e.nextStartAt = 0;
|
|
404
477
|
e.strandedAt = 0;
|
|
@@ -436,7 +509,7 @@ export function agentRunner(sessions, deps = {}) {
|
|
|
436
509
|
async function stopAll() {
|
|
437
510
|
stoppingAll = true;
|
|
438
511
|
const live = [...entries.values()].filter((e) => e.child);
|
|
439
|
-
const exits = live.map((e) => new Promise((r) => (e.child.once("
|
|
512
|
+
const exits = live.map((e) => new Promise((r) => (e.child.once("close", r), e.child.once("error", r), later(r, L.killMs + 500)?.unref?.())));
|
|
440
513
|
await Promise.all(live.map((e) => stop(e.key, "shutdown")));
|
|
441
514
|
await Promise.all(exits);
|
|
442
515
|
}
|
|
@@ -463,6 +536,10 @@ export function agentRunner(sessions, deps = {}) {
|
|
|
463
536
|
});
|
|
464
537
|
sessions.on("closed", (s) => {
|
|
465
538
|
const e = entries.get(s.key);
|
|
539
|
+
if (e) {
|
|
540
|
+
e.gen++;
|
|
541
|
+
e.launching = false;
|
|
542
|
+
}
|
|
466
543
|
if (e?.child) {
|
|
467
544
|
// its poll was answered "closed", so it ends by itself (and writes its last result); the group is stopped only if it has not after 10 s.
|
|
468
545
|
// The cleanup runs after the process is gone, so nothing writes into the folder again.
|
|
@@ -566,11 +643,13 @@ export function agentRunner(sessions, deps = {}) {
|
|
|
566
643
|
/** Open in Terminal: the background agent stops and the same conversation opens in Terminal with the same limits. */
|
|
567
644
|
async function handover(s) {
|
|
568
645
|
const e = entry(s.key);
|
|
646
|
+
e.gen++;
|
|
647
|
+
e.launching = false;
|
|
569
648
|
const bin = findClaude(env);
|
|
570
649
|
if (!bin || agentsDisabled(env)) return fail(e, "not_installed");
|
|
571
650
|
if (e.state === "failed" && e.failure?.code === "signed_out") return signIn(e, bin);
|
|
572
651
|
// the background process is gone before Terminal opens the same conversation (two processes never share one), at most 5 s
|
|
573
|
-
const gone = e.child ? new Promise((r) => (e.child.once("
|
|
652
|
+
const gone = e.child ? new Promise((r) => (e.child.once("close", r), e.child.once("error", r), later(r, L.killMs + 1000)?.unref?.())) : null;
|
|
574
653
|
await stop(s.key, "handover");
|
|
575
654
|
if (gone) await gone;
|
|
576
655
|
e.state = "handover";
|
package/src/checker.js
CHANGED
|
@@ -5,6 +5,7 @@ import path from "node:path";
|
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { PRODUCT, env } from "./constants.js";
|
|
7
7
|
import { stateDir } from "./paths.js";
|
|
8
|
+
import { runInGroup } from "./update.js";
|
|
8
9
|
|
|
9
10
|
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
10
11
|
const CHECK_SCRIPT = path.join(here, "..", "scripts", "checker", "check.py");
|
|
@@ -22,6 +23,64 @@ export function findPython() {
|
|
|
22
23
|
return fs.existsSync(p) ? p : null;
|
|
23
24
|
}
|
|
24
25
|
|
|
26
|
+
const DEEP_KILL_GRACE_MS = 2000;
|
|
27
|
+
const DEEP_MAX_BUFFER = 16 * 1024 * 1024;
|
|
28
|
+
const checkerEnv = (dir) => ({ ...process.env, ONIONSKIN_SOFFICE_PID_FILE: path.join(dir, "soffice.pid") });
|
|
29
|
+
|
|
30
|
+
// check.py clears this file after conversion. A hard kill cannot run its cleanup handler.
|
|
31
|
+
function stopRemainingSoffice(dir) {
|
|
32
|
+
try {
|
|
33
|
+
const record = fs.readFileSync(path.join(dir, "soffice.pid"), "ascii");
|
|
34
|
+
if (!/^\d+\n$/.test(record)) return;
|
|
35
|
+
const pid = Number(record);
|
|
36
|
+
if (Number.isSafeInteger(pid) && pid > 1) process.kill(-pid, "SIGKILL");
|
|
37
|
+
} catch {}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Runs `cmd args...` with SIGTERM at timeoutMs and SIGKILL after graceMs so a process stuck in a native call cannot outlast the timeout.
|
|
41
|
+
const SYNC_RUNNER = `
|
|
42
|
+
const { spawn } = require("node:child_process");
|
|
43
|
+
const fs = require("node:fs");
|
|
44
|
+
const [metaFile, timeoutStr, graceStr, maxBufStr, cmd, ...args] = process.argv.slice(1);
|
|
45
|
+
const timeoutMs = Number(timeoutStr);
|
|
46
|
+
const graceMs = Number(graceStr);
|
|
47
|
+
const maxBuf = Number(maxBufStr);
|
|
48
|
+
let timedOut = false, killTimer = null, t = null, bytes = 0, overflow = false, done = false;
|
|
49
|
+
const write = (o) => { try { fs.writeFileSync(metaFile, JSON.stringify(o)); } catch {} };
|
|
50
|
+
const finish = (o) => {
|
|
51
|
+
if (done) return;
|
|
52
|
+
done = true;
|
|
53
|
+
clearTimeout(t); clearTimeout(killTimer);
|
|
54
|
+
write(o);
|
|
55
|
+
if (timedOut) { child.stdout.destroy(); child.stderr.destroy(); }
|
|
56
|
+
};
|
|
57
|
+
const child = spawn(cmd, args, { env: process.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
58
|
+
const feed = (stream, out) => stream.on("data", (d) => {
|
|
59
|
+
bytes += d.length;
|
|
60
|
+
if (bytes > maxBuf) {
|
|
61
|
+
if (!overflow) { overflow = true; try { child.kill("SIGKILL"); } catch {} }
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
out.write(d);
|
|
65
|
+
});
|
|
66
|
+
feed(child.stdout, process.stdout);
|
|
67
|
+
feed(child.stderr, process.stderr);
|
|
68
|
+
if (timeoutMs > 0) {
|
|
69
|
+
t = setTimeout(() => {
|
|
70
|
+
timedOut = true;
|
|
71
|
+
if (child.exitCode !== null || child.signalCode !== null) return finish({ timedOut });
|
|
72
|
+
try { child.kill("SIGTERM"); } catch {}
|
|
73
|
+
killTimer = setTimeout(() => { try { child.kill("SIGKILL"); } catch {} }, graceMs);
|
|
74
|
+
killTimer.unref?.();
|
|
75
|
+
}, timeoutMs);
|
|
76
|
+
}
|
|
77
|
+
child.once("error", (err) => finish({ timedOut, error: { code: err.code, message: err.message } }));
|
|
78
|
+
child.once("exit", (status, signal) => {
|
|
79
|
+
if (timedOut) finish({ timedOut, status, signal, error: overflow ? { code: "ENOBUFS", message: "maxBuffer exceeded" } : null });
|
|
80
|
+
});
|
|
81
|
+
child.once("close", (status, signal) => finish({ timedOut, status, signal, error: overflow ? { code: "ENOBUFS", message: "maxBuffer exceeded" } : null }));
|
|
82
|
+
`;
|
|
83
|
+
|
|
25
84
|
/**
|
|
26
85
|
* Runs scripts/checker/check.py on the two package versions. Never returns "clean" for a check that did not run:
|
|
27
86
|
* {ran:false, reason} when Python or its libraries are missing, and a `deep_check_failed` flag when it ran but broke.
|
|
@@ -35,26 +94,43 @@ export function runDeepCheck(preBytes, postBytes, format, { deferSoffice = false
|
|
|
35
94
|
try {
|
|
36
95
|
const pre = path.join(dir, `pre.${format}`);
|
|
37
96
|
const post = path.join(dir, `post.${format}`);
|
|
97
|
+
const metaFile = path.join(dir, "meta.json");
|
|
98
|
+
const timeoutMs = CHECK_TIMEOUT_MS();
|
|
38
99
|
fs.writeFileSync(pre, preBytes);
|
|
39
100
|
fs.writeFileSync(post, postBytes);
|
|
40
|
-
const r = spawnSync(
|
|
41
|
-
|
|
42
|
-
|
|
101
|
+
const r = spawnSync(
|
|
102
|
+
process.execPath,
|
|
103
|
+
["-e", SYNC_RUNNER, metaFile, String(timeoutMs), String(DEEP_KILL_GRACE_MS), String(DEEP_MAX_BUFFER), py, "-I", CHECK_SCRIPT, pre, post, format, ...(deferSoffice ? ["--no-soffice"] : [])],
|
|
104
|
+
{ encoding: "utf8", detached: true, timeout: timeoutMs > 0 ? timeoutMs + DEEP_KILL_GRACE_MS + 5000 : 0, maxBuffer: DEEP_MAX_BUFFER + 65536, env: checkerEnv(dir), killSignal: "SIGKILL" },
|
|
105
|
+
);
|
|
106
|
+
let meta = null;
|
|
107
|
+
try {
|
|
108
|
+
meta = JSON.parse(fs.readFileSync(metaFile, "utf8"));
|
|
109
|
+
} catch {}
|
|
110
|
+
if (meta?.timedOut || r.error?.code === "ETIMEDOUT") {
|
|
111
|
+
// Backstop for a stuck runner or descendants that still hold its pipes.
|
|
112
|
+
if (r.pid > 1) try { process.kill(-r.pid, "SIGKILL"); } catch {}
|
|
113
|
+
return { ran: false, code: "timed_out", flags: [{ code: "deep_check_failed", severity: "warn", detail: `timed out after ${timeoutMs / 1000}s` }], reason: "the deeper check timed out" };
|
|
114
|
+
}
|
|
115
|
+
const err = meta?.error || r.error;
|
|
116
|
+
if (err) return { ran: false, code: "failed", flags: [{ code: "deep_check_failed", severity: "warn", detail: err.message }], reason: "the deeper check could not start" };
|
|
117
|
+
const status = meta ? meta.status : r.status;
|
|
43
118
|
let j;
|
|
44
119
|
try {
|
|
45
120
|
j = JSON.parse(r.stdout);
|
|
46
121
|
} catch {
|
|
47
122
|
j = null;
|
|
48
123
|
}
|
|
49
|
-
if (
|
|
50
|
-
if (
|
|
51
|
-
const why = (j?.error || r.stderr || `exit ${
|
|
124
|
+
if (status === 3) return { ran: false, flags: [], code: "libraries_missing", reason: `${py} lacks lxml/defusedxml: run \`${PRODUCT.bin} setup-checker\`` };
|
|
125
|
+
if (status !== 0 || !j || !Array.isArray(j.flags)) {
|
|
126
|
+
const why = (j?.error || r.stderr || `exit ${status}`).toString().slice(0, 300);
|
|
52
127
|
return { ran: false, code: "failed", flags: [{ code: "deep_check_failed", severity: "warn", detail: why }], reason: "the deeper check failed to run" };
|
|
53
128
|
}
|
|
54
129
|
const skipped = j.skipped || [];
|
|
55
130
|
const pending = skipped.some((x) => x.check === "soffice-convert" && x.pending);
|
|
56
131
|
return { ran: true, flags: j.flags, checksRun: j.ran, skipped: pending ? skipped.filter((x) => !x.pending) : skipped, sofficePending: pending, python: py };
|
|
57
132
|
} finally {
|
|
133
|
+
stopRemainingSoffice(dir);
|
|
58
134
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
59
135
|
}
|
|
60
136
|
}
|
|
@@ -71,6 +147,36 @@ const lateNext = () => {
|
|
|
71
147
|
if (start) start();
|
|
72
148
|
};
|
|
73
149
|
|
|
150
|
+
const KILL_GRACE_MS = 5000;
|
|
151
|
+
|
|
152
|
+
/** Sends SIGTERM to a live child and schedules an unref'd SIGKILL fallback so a hung child cannot stall the queue or leak temp dirs. */
|
|
153
|
+
function gracefulKill(getChild) {
|
|
154
|
+
let killTimer = null;
|
|
155
|
+
const terminate = () => {
|
|
156
|
+
const child = getChild();
|
|
157
|
+
if (!child || child.exitCode !== null || child.signalCode !== null) return;
|
|
158
|
+
try {
|
|
159
|
+
child.kill("SIGTERM");
|
|
160
|
+
} catch {}
|
|
161
|
+
if (!killTimer) {
|
|
162
|
+
killTimer = setTimeout(() => {
|
|
163
|
+
const c = getChild();
|
|
164
|
+
if (c && c.exitCode === null && c.signalCode === null) {
|
|
165
|
+
try {
|
|
166
|
+
c.kill("SIGKILL");
|
|
167
|
+
} catch {}
|
|
168
|
+
}
|
|
169
|
+
}, KILL_GRACE_MS);
|
|
170
|
+
killTimer.unref?.();
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
const clear = () => {
|
|
174
|
+
clearTimeout(killTimer);
|
|
175
|
+
killTimer = null;
|
|
176
|
+
};
|
|
177
|
+
return { terminate, clear };
|
|
178
|
+
}
|
|
179
|
+
|
|
74
180
|
/**
|
|
75
181
|
* The LibreOffice conversion check for a version `done` already accepted, run in the background (check.py --soffice-only). Never blocks:
|
|
76
182
|
* `onDone({ran, flags, skipped, reason})` is called exactly once, also when the check fails, times out, is cancelled or never got its turn
|
|
@@ -85,17 +191,27 @@ export function startLateSoffice(preBytes, postBytes, format, onDone) {
|
|
|
85
191
|
let child = null;
|
|
86
192
|
let timer = null;
|
|
87
193
|
let dir = null;
|
|
194
|
+
const killer = gracefulKill(() => child);
|
|
195
|
+
const cleanupDir = () => {
|
|
196
|
+
if (!dir) return;
|
|
197
|
+
const d = dir;
|
|
198
|
+
dir = null;
|
|
199
|
+
stopRemainingSoffice(d);
|
|
200
|
+
fs.rmSync(d, { recursive: true, force: true });
|
|
201
|
+
};
|
|
88
202
|
const finish = (r) => {
|
|
89
203
|
if (settled) return;
|
|
90
204
|
settled = true;
|
|
91
205
|
clearTimeout(timer);
|
|
92
|
-
if (dir) fs.rmSync(dir, { recursive: true, force: true });
|
|
93
206
|
// the next one starts only when this one's process is really gone (a cancelled or timed-out check takes a moment to die)
|
|
94
207
|
if (started && child?.pid && child.exitCode === null && child.signalCode === null) child.once("close", lateNext);
|
|
95
|
-
else if (started) lateNext();
|
|
96
208
|
else {
|
|
97
|
-
|
|
98
|
-
if (
|
|
209
|
+
cleanupDir();
|
|
210
|
+
if (started) lateNext();
|
|
211
|
+
else {
|
|
212
|
+
const at = lateQueue.indexOf(start);
|
|
213
|
+
if (at >= 0) lateQueue.splice(at, 1);
|
|
214
|
+
}
|
|
99
215
|
}
|
|
100
216
|
try {
|
|
101
217
|
onDone(r);
|
|
@@ -103,7 +219,7 @@ export function startLateSoffice(preBytes, postBytes, format, onDone) {
|
|
|
103
219
|
};
|
|
104
220
|
const cancel = (reason = "the check was cancelled") => {
|
|
105
221
|
if (settled) return;
|
|
106
|
-
|
|
222
|
+
killer.terminate();
|
|
107
223
|
finish({ ran: false, flags: [], skipped: [], reason });
|
|
108
224
|
};
|
|
109
225
|
function start() {
|
|
@@ -115,15 +231,20 @@ export function startLateSoffice(preBytes, postBytes, format, onDone) {
|
|
|
115
231
|
const post = path.join(dir, `post.${format}`);
|
|
116
232
|
fs.writeFileSync(pre, preBytes);
|
|
117
233
|
fs.writeFileSync(post, postBytes);
|
|
118
|
-
child = spawn(py, ["-I", CHECK_SCRIPT, pre, post, format, "--soffice-only"], { env:
|
|
234
|
+
child = spawn(py, ["-I", CHECK_SCRIPT, pre, post, format, "--soffice-only"], { env: checkerEnv(dir), stdio: ["ignore", "pipe", "pipe"] });
|
|
119
235
|
} catch (err) {
|
|
120
236
|
return finish({ ran: false, flags: [], skipped: [], reason: `could not start: ${err.message}` });
|
|
121
237
|
}
|
|
122
238
|
let out = "";
|
|
123
239
|
child.stdout.on("data", (d) => (out = (out + d).slice(0, 16 * 1024 * 1024)));
|
|
124
240
|
child.stderr.resume();
|
|
125
|
-
child.once("error", (err) =>
|
|
241
|
+
child.once("error", (err) => {
|
|
242
|
+
killer.clear();
|
|
243
|
+
finish({ ran: false, flags: [], skipped: [], reason: `could not start: ${err.message}` });
|
|
244
|
+
});
|
|
126
245
|
child.once("close", (code) => {
|
|
246
|
+
killer.clear();
|
|
247
|
+
cleanupDir();
|
|
127
248
|
let j = null;
|
|
128
249
|
try {
|
|
129
250
|
j = JSON.parse(out);
|
|
@@ -132,7 +253,7 @@ export function startLateSoffice(preBytes, postBytes, format, onDone) {
|
|
|
132
253
|
else finish({ ran: false, flags: [], skipped: [], reason: (j?.error || `exit ${code}`).toString().slice(0, 200) });
|
|
133
254
|
});
|
|
134
255
|
timer = setTimeout(() => {
|
|
135
|
-
|
|
256
|
+
killer.terminate();
|
|
136
257
|
finish({ ran: false, flags: [], skipped: [], reason: `timed out after ${CHECK_TIMEOUT_MS() / 1000}s` });
|
|
137
258
|
}, CHECK_TIMEOUT_MS());
|
|
138
259
|
}
|
|
@@ -144,20 +265,37 @@ export function startLateSoffice(preBytes, postBytes, format, onDone) {
|
|
|
144
265
|
}
|
|
145
266
|
|
|
146
267
|
/** `onionskin setup-checker`: the one place that uses the network (pip). Creates <state dir>/venv. */
|
|
147
|
-
export function setupChecker(log = (s) => process.stderr.write(s + "\n")) {
|
|
268
|
+
export async function setupChecker(log = (s) => process.stderr.write(s + "\n")) {
|
|
148
269
|
const base = process.env[`${PRODUCT.envPrefix}_BASE_PYTHON`] || "python3";
|
|
149
270
|
fs.mkdirSync(stateDir(), { recursive: true, mode: 0o700 });
|
|
271
|
+
const deadline = Date.now() + SETUP_TIMEOUT_MS();
|
|
272
|
+
const runStep = async (cmd, args) => {
|
|
273
|
+
const remaining = deadline - Date.now();
|
|
274
|
+
let expired = remaining <= 0;
|
|
275
|
+
// The installer may itself receive SIGKILL shortly after TERM. End this step's group before that happens.
|
|
276
|
+
const status = expired ? 1 : await runInGroup(cmd, args, { timeout: remaining, signalGraceMs: 0, onTimeout: () => { expired = true; } });
|
|
277
|
+
if (expired) throw new Error("Deeper check setup timed out.");
|
|
278
|
+
if (status === 129 || status === 130 || status === 143) throw new Error("Deeper check setup was stopped.");
|
|
279
|
+
return status;
|
|
280
|
+
};
|
|
281
|
+
const imports = ["-c", "import lxml, docx, openpyxl, pptx, defusedxml"];
|
|
282
|
+
if (env("SETUP_IF_NEEDED") === "1") {
|
|
283
|
+
if (fs.existsSync(venvPython()) && await runStep(venvPython(), imports) === 0) return venvPython();
|
|
284
|
+
if (!env("BASE_PYTHON") && process.platform === "darwin") {
|
|
285
|
+
const guard = '[ "$(command -v python3)" != /usr/bin/python3 ] || xcode-select -p >/dev/null 2>&1';
|
|
286
|
+
if (await runStep("/bin/sh", ["-c", guard]) !== 0) throw new Error("Python needs the macOS developer tools, which are not installed.");
|
|
287
|
+
}
|
|
288
|
+
}
|
|
150
289
|
const steps = [
|
|
151
290
|
[base, ["-m", "venv", venvDir()]],
|
|
152
|
-
[venvPython(), ["-m", "pip", "install", "--disable-pip-version-check", "--quiet", "-r", CHECKER_REQUIREMENTS]],
|
|
291
|
+
[venvPython(), ["-m", "pip", "install", "--disable-pip-version-check", "--quiet", "--require-hashes", "--only-binary=:all:", "-r", CHECKER_REQUIREMENTS]],
|
|
153
292
|
];
|
|
154
293
|
for (const [cmd, args] of steps) {
|
|
155
294
|
log(`$ ${path.basename(cmd)} ${args.slice(0, 3).join(" ")} ...`);
|
|
156
|
-
const
|
|
157
|
-
if (
|
|
295
|
+
const status = await runStep(cmd, args);
|
|
296
|
+
if (status !== 0) throw new Error(`${cmd} ${args.slice(0, 2).join(" ")} failed (exit ${status})`);
|
|
158
297
|
}
|
|
159
|
-
|
|
160
|
-
if (probe.status !== 0) throw new Error(`the venv was created but the libraries do not import: ${probe.stderr}`);
|
|
298
|
+
if (await runStep(venvPython(), imports) !== 0) throw new Error("the venv was created but the libraries do not import");
|
|
161
299
|
return venvPython();
|
|
162
300
|
}
|
|
163
301
|
|
|
@@ -165,7 +303,8 @@ export function setupChecker(log = (s) => process.stderr.write(s + "\n")) {
|
|
|
165
303
|
// The server must keep answering while pip runs (about a minute), so this variant spawns instead of spawnSync. One job at a time for the
|
|
166
304
|
// whole server: a second start while one runs gets the same promise. The job's outcome is kept until the next start so the page can poll it.
|
|
167
305
|
const SETUP_TIMEOUT_MS = () => Number(env("SETUP_TIMEOUT_MS", 10 * 60_000));
|
|
168
|
-
export const NETWORK_WORDS = /could not resolve|name or service not known|nodename nor servname|temporary failure in name resolution|failed to establish a new connection|connection (?:error|refused|reset|timed out)|network is unreachable|
|
|
306
|
+
export const NETWORK_WORDS = /could not (?:resolve|fetch url)|name or service not known|nodename nor servname|temporary failure in name resolution|failed to establish a new connection|newconnectionerror|connection (?:error|refused|reset|timed out)|network is unreachable|max retries exceeded|read timed out|ssl|proxy/i;
|
|
307
|
+
export const UNSUPPORTED_PYTHON_WORDS = /requires-python|ignored the following versions/i;
|
|
169
308
|
let job = null; // {promise} while running; {outcome: {reason, detail}} after a failure; null when idle or after a success
|
|
170
309
|
|
|
171
310
|
/** One command, async. Resolves {status, stderr (last 4000 bytes), error}; never rejects, never inherits the server's stdio. */
|
|
@@ -192,6 +331,36 @@ export function run(cmd, args, timeoutMs) {
|
|
|
192
331
|
});
|
|
193
332
|
}
|
|
194
333
|
|
|
334
|
+
/** Returns e.g. "Python 3.9.6" or "" if the interpreter cannot be queried. */
|
|
335
|
+
function pythonVersion(py) {
|
|
336
|
+
return new Promise((resolve) => {
|
|
337
|
+
let out = "";
|
|
338
|
+
let settled = false;
|
|
339
|
+
const done = (v) => {
|
|
340
|
+
if (settled) return;
|
|
341
|
+
settled = true;
|
|
342
|
+
clearTimeout(timer);
|
|
343
|
+
resolve(v);
|
|
344
|
+
};
|
|
345
|
+
let child;
|
|
346
|
+
try {
|
|
347
|
+
child = spawn(py, ["--version"], { stdio: ["ignore", "pipe", "pipe"], env: process.env });
|
|
348
|
+
} catch {
|
|
349
|
+
return resolve("");
|
|
350
|
+
}
|
|
351
|
+
const timer = setTimeout(() => {
|
|
352
|
+
try {
|
|
353
|
+
child.kill("SIGKILL");
|
|
354
|
+
} catch {}
|
|
355
|
+
done("");
|
|
356
|
+
}, 5000);
|
|
357
|
+
child.stdout.on("data", (d) => (out += d));
|
|
358
|
+
child.stderr.on("data", (d) => (out += d));
|
|
359
|
+
child.once("error", () => done(""));
|
|
360
|
+
child.once("close", (status) => done(status === 0 ? out.trim().slice(0, 80) : ""));
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
|
|
195
364
|
/** Do the venv's libraries import? (The one probe both setup and the status use: a venv whose pip said ok but whose libraries do not import is not ready.) */
|
|
196
365
|
async function librariesImport(py) {
|
|
197
366
|
return (await run(py, ["-c", "import lxml, docx, openpyxl, pptx, defusedxml"], 30_000)).status === 0;
|
|
@@ -210,10 +379,18 @@ async function setupSteps() {
|
|
|
210
379
|
const missing = venv.error?.code === "ENOENT";
|
|
211
380
|
throw Object.assign(new Error(`${base} -m venv failed (${venv.error?.message || `exit ${venv.status}`}) ${venv.stderr}`.trim()), { reason: missing ? "no_python" : "failed" });
|
|
212
381
|
}
|
|
213
|
-
const pip = await run(venvPython(), ["-m", "pip", "install", "--disable-pip-version-check", "--quiet", "-r", CHECKER_REQUIREMENTS], SETUP_TIMEOUT_MS());
|
|
382
|
+
const pip = await run(venvPython(), ["-m", "pip", "install", "--disable-pip-version-check", "--quiet", "--require-hashes", "--only-binary=:all:", "-r", CHECKER_REQUIREMENTS], SETUP_TIMEOUT_MS());
|
|
214
383
|
if (pip.status !== 0) {
|
|
215
384
|
const text = `pip install failed (${pip.error?.message || `exit ${pip.status}`}) ${pip.stderr}`.trim();
|
|
216
|
-
|
|
385
|
+
if (NETWORK_WORDS.test(text)) {
|
|
386
|
+
throw Object.assign(new Error(text), { reason: "no_network" });
|
|
387
|
+
}
|
|
388
|
+
if (UNSUPPORTED_PYTHON_WORDS.test(text)) {
|
|
389
|
+
const ver = await pythonVersion(venvPython());
|
|
390
|
+
const detail = ver ? `this Python version or platform (${ver}) is not supported: ${text}` : `this Python version or platform is not supported: ${text}`;
|
|
391
|
+
throw Object.assign(new Error(detail), { reason: "unsupported_python", pythonVersion: ver || undefined });
|
|
392
|
+
}
|
|
393
|
+
throw Object.assign(new Error(text), { reason: "failed" });
|
|
217
394
|
}
|
|
218
395
|
if (!(await librariesImport(venvPython()))) throw Object.assign(new Error("the venv was created but the libraries do not import"), { reason: "failed" });
|
|
219
396
|
// runs use findPython(): an ONIONSKIN_PYTHON that points elsewhere would keep the checker off after a "successful" setup
|
|
@@ -224,7 +401,7 @@ async function setupSteps() {
|
|
|
224
401
|
|
|
225
402
|
/**
|
|
226
403
|
* Starts the setup (or joins the one that is running) and returns its promise. It resolves to the venv's python, or rejects with an error that
|
|
227
|
-
* carries `reason`: no_python | no_network | failed. The caller does not need to await it; setupCheckerStatus() reports it.
|
|
404
|
+
* carries `reason`: no_python | no_network | unsupported_python | failed. The caller does not need to await it; setupCheckerStatus() reports it.
|
|
228
405
|
*/
|
|
229
406
|
export function startSetupChecker() {
|
|
230
407
|
if (job?.promise) return job.promise;
|
|
@@ -232,7 +409,13 @@ export function startSetupChecker() {
|
|
|
232
409
|
mine.promise = setupSteps().then(
|
|
233
410
|
(py) => ((job = null), py),
|
|
234
411
|
(err) => {
|
|
235
|
-
job = {
|
|
412
|
+
job = {
|
|
413
|
+
outcome: {
|
|
414
|
+
reason: err.reason || "failed",
|
|
415
|
+
detail: String(err.message).slice(0, 2000),
|
|
416
|
+
...(err.pythonVersion ? { pythonVersion: err.pythonVersion } : {}),
|
|
417
|
+
},
|
|
418
|
+
};
|
|
236
419
|
throw err;
|
|
237
420
|
},
|
|
238
421
|
);
|
|
@@ -264,23 +447,45 @@ export function deepRedline(preBytes, postBytes, outBytes) {
|
|
|
264
447
|
const py = findPython();
|
|
265
448
|
if (!py) return Promise.resolve({ ran: false, reason: "not_set_up" });
|
|
266
449
|
return new Promise((resolve) => {
|
|
267
|
-
|
|
450
|
+
let dir = fs.mkdtempSync(path.join(os.tmpdir(), "onionskin-redline-"));
|
|
268
451
|
const files = ["pre", "post", "out"].map((n) => path.join(dir, `${n}.docx`));
|
|
269
|
-
[preBytes, postBytes, outBytes].forEach((b, k) => fs.writeFileSync(files[k], b));
|
|
270
452
|
let out = "";
|
|
271
453
|
let settled = false;
|
|
454
|
+
let child = null;
|
|
455
|
+
let timer = null;
|
|
456
|
+
const killer = gracefulKill(() => child);
|
|
457
|
+
const cleanupDir = () => {
|
|
458
|
+
if (!dir) return;
|
|
459
|
+
const d = dir;
|
|
460
|
+
dir = null;
|
|
461
|
+
stopRemainingSoffice(d);
|
|
462
|
+
fs.rmSync(d, { recursive: true, force: true });
|
|
463
|
+
};
|
|
272
464
|
const finish = (r) => {
|
|
273
465
|
if (settled) return;
|
|
274
466
|
settled = true;
|
|
275
467
|
clearTimeout(timer);
|
|
276
|
-
|
|
468
|
+
if (!child?.pid || child.exitCode !== null || child.signalCode !== null) cleanupDir();
|
|
277
469
|
resolve(r);
|
|
278
470
|
};
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
471
|
+
try {
|
|
472
|
+
[preBytes, postBytes, outBytes].forEach((b, k) => fs.writeFileSync(files[k], b));
|
|
473
|
+
child = spawn(py, ["-I", CHECK_SCRIPT, "--redline", ...files], { env: checkerEnv(dir), stdio: ["ignore", "pipe", "ignore"] });
|
|
474
|
+
} catch {
|
|
475
|
+
return finish({ ran: true, ok: false, why: "the deeper check could not start" });
|
|
476
|
+
}
|
|
477
|
+
timer = setTimeout(() => {
|
|
478
|
+
killer.terminate();
|
|
479
|
+
finish({ ran: true, ok: false, why: "the deeper check timed out" });
|
|
480
|
+
}, CHECK_TIMEOUT_MS());
|
|
481
|
+
child.stdout.on("data", (c) => (out += c).length > 1e6 && killer.terminate());
|
|
482
|
+
child.once("error", () => {
|
|
483
|
+
killer.clear();
|
|
484
|
+
finish({ ran: true, ok: false, why: "the deeper check could not start" });
|
|
485
|
+
});
|
|
486
|
+
child.once("close", (code) => {
|
|
487
|
+
killer.clear();
|
|
488
|
+
cleanupDir();
|
|
284
489
|
if (code === 3) return finish({ ran: false, reason: "libraries_missing" });
|
|
285
490
|
let j = null;
|
|
286
491
|
try {
|