onionskin-cli 0.0.3 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CHANGELOG.md +198 -0
  2. package/INSTALL.md +5 -1
  3. package/NOTICE +34 -0
  4. package/README.md +102 -34
  5. package/docs/contracts/batch.schema.json +27 -4
  6. package/docs/contracts/ops.schema.json +117 -5
  7. package/package.json +4 -2
  8. package/scripts/checker/check.py +24 -5
  9. package/scripts/checker/requirements.txt +482 -5
  10. package/scripts/requirements.txt +482 -5
  11. package/scripts/rewriter/apple/main.swift +1 -1
  12. package/scripts/rewriter/apple/onionskin-fm +0 -0
  13. package/scripts/rewriter/apple/onionskin-fm.json +1 -1
  14. package/skills/onionskin/SKILL.md +12 -9
  15. package/skills/onionskin/reference/batch.md +5 -3
  16. package/skills/onionskin/reference/formats.md +5 -1
  17. package/skills/onionskin/reference/ops.md +48 -4
  18. package/skills/onionskin/reference/troubleshooting.md +31 -0
  19. package/src/agent-detect.js +28 -2
  20. package/src/agent-prompt.js +12 -1
  21. package/src/agent-runner.js +110 -20
  22. package/src/anchor-work.js +154 -0
  23. package/src/anchor-worker.js +38 -0
  24. package/src/anchors.js +61 -11
  25. package/src/apply.js +119 -47
  26. package/src/batch.js +32 -8
  27. package/src/check-worker.js +8 -5
  28. package/src/checker.js +238 -33
  29. package/src/cli.js +248 -42
  30. package/src/constants.js +34 -1
  31. package/src/diff.js +3 -1
  32. package/src/doctor.js +12 -3
  33. package/src/docx.js +464 -87
  34. package/src/edit-hash.js +19 -0
  35. package/src/fsutil.js +14 -6
  36. package/src/history-worker.js +8 -8
  37. package/src/history.js +144 -31
  38. package/src/html-active.js +384 -0
  39. package/src/html-envelope.js +51 -0
  40. package/src/html-parser.js +8 -0
  41. package/src/html-render-worker.js +25 -0
  42. package/src/html-render.js +939 -0
  43. package/src/html-utf8.js +65 -0
  44. package/src/html-write-worker.js +14 -0
  45. package/src/html-write.js +380 -0
  46. package/src/html.js +1259 -0
  47. package/src/local.js +68 -4
  48. package/src/merge-package.js +1166 -0
  49. package/src/merge-pptx.js +98 -38
  50. package/src/merge-proof.js +1019 -0
  51. package/src/merge-worker.js +60 -0
  52. package/src/merge-xlsx.js +84 -29
  53. package/src/merge.js +286 -83
  54. package/src/nfc.js +46 -0
  55. package/src/notes.js +122 -0
  56. package/src/offload.js +419 -1
  57. package/src/open-worker.js +27 -0
  58. package/src/ops/common.js +2 -1
  59. package/src/ops/docx-table.js +5 -3
  60. package/src/ops/docx.js +58 -29
  61. package/src/ops/html.js +394 -0
  62. package/src/ops/pptx.js +82 -23
  63. package/src/ops/run.js +23 -3
  64. package/src/ops/runs.js +215 -13
  65. package/src/ops/schema.js +8 -1
  66. package/src/ops/xlsx.js +121 -10
  67. package/src/ops-worker.js +17 -0
  68. package/src/package.js +43 -4
  69. package/src/person.js +2 -1
  70. package/src/pptx.js +323 -23
  71. package/src/question.js +10 -3
  72. package/src/quick-plan.js +13 -6
  73. package/src/quick.js +1 -1
  74. package/src/redline-check.js +12 -2
  75. package/src/redline-worker.js +10 -14
  76. package/src/reply.js +46 -19
  77. package/src/retain.js +223 -0
  78. package/src/review.js +32 -18
  79. package/src/rewrite-check.js +2 -2
  80. package/src/rewriter.js +38 -8
  81. package/src/sanitize.js +16 -3
  82. package/src/server-entry.js +4 -3
  83. package/src/server.js +175 -62
  84. package/src/session.js +1151 -210
  85. package/src/setup.js +20 -2
  86. package/src/stop-gate.js +34 -0
  87. package/src/suggest-reply.js +42 -7
  88. package/src/suggest.js +14 -29
  89. package/src/text.js +161 -11
  90. package/src/update.js +517 -0
  91. package/src/voice-guide.js +15 -9
  92. package/src/voice-learn.js +15 -4
  93. package/src/voice-sample-worker.js +7 -6
  94. package/src/voice-store.js +11 -4
  95. package/src/voice-worker.js +4 -6
  96. package/src/voice.js +2 -1
  97. package/src/worker-host.js +23 -0
  98. package/src/xlsx.js +378 -97
  99. package/src/xml-tree.js +6 -3
  100. package/src/xml.js +82 -34
  101. package/src/zip-write.js +10 -5
  102. package/src/zip.js +73 -18
  103. package/vendor/parse5/LICENSE.txt +32 -0
  104. package/vendor/parse5/parse5.mjs +7822 -0
  105. package/vendor/parse5.lock.json +23 -0
  106. package/web/dist/assets/{index-DS-WST1E.js → index-CjsTf5gj.js} +58 -52
  107. package/web/dist/assets/index-DnbLtHhG.css +1 -0
  108. package/web/dist/index.html +2 -2
  109. package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/update.js ADDED
@@ -0,0 +1,517 @@
1
+ // `onionskin update [--check] [--yes] [--force]`: brings this copy to the latest release, the way this copy was installed.
2
+ // The network is used only when a person (or their agent) runs this command: `--check` reads one small file, `version.txt`, from the
3
+ // release folder; `update` on an installer copy then runs the release's get.sh exactly as the README one-liner does (get.sh keeps the
4
+ // SHA-256 check). No prompts, no sudo; snapshots and the state folder are never deleted here.
5
+ //
6
+ // Everything this command needs after get.sh starts is imported at the top: install.sh deletes the app folder this process runs from.
7
+ import { spawn, spawnSync } from "node:child_process";
8
+ import fs from "node:fs";
9
+ import os from "node:os";
10
+ import path from "node:path";
11
+ import { PRODUCT } from "./constants.js";
12
+
13
+ const BIN = PRODUCT.bin;
14
+ export const DEFAULT_BASE_URL = "https://github.com/abe238/onionskin/releases/latest/download";
15
+ export const VERSION_FILE = "version.txt";
16
+ export const NPM_COMMANDS = ["npm install -g onionskin-cli", `${BIN} setup --force`]; // README "### Stay up to date", word for word
17
+ const CURL = ["-fsSL", "--connect-timeout", "20", "--max-time", "30", "--proto-redir", "=https"]; // get.sh's own flags, a shorter time limit
18
+
19
+ /**
20
+ * get.sh's rule for its URLs, the same rule (scripts/get.sh url_ok; test/get-script.test.js checks both on one table): https://, file:///,
21
+ * or http:// to exactly 127.0.0.1 with an optional port, for local tests. No white space anywhere, and no "@" or backslash before the
22
+ * first "/" after the scheme: a user name hides the real host (http://127.0.0.1:80@evil.example/ is evil.example). Read as text, the way
23
+ * the shell rule reads it, not through a URL parser that may read the host differently than curl does.
24
+ */
25
+ export function baseUrlOk(u) {
26
+ const s = String(u ?? "");
27
+ if (/\s/.test(s)) return false;
28
+ if (s.startsWith("file:///")) return true;
29
+ const m = /^(https?):\/\/([^/]*)/.exec(s);
30
+ if (!m || /[@\\]/.test(m[2])) return false;
31
+ return m[1] === "https" || /^127\.0\.0\.1(?::\d+)?$/.test(m[2]);
32
+ }
33
+
34
+ /** curl's --proto for a URL baseUrlOk accepted: only the scheme of the URL itself (redirects stay https only, CURL). */
35
+ export const curlProto = (u) => {
36
+ const s = String(u ?? "");
37
+ if (s.startsWith("https://")) return ["--proto", "=https"];
38
+ if (s.startsWith("file://")) return ["--proto", "=file"];
39
+ if (/^http:\/\/127\.0\.0\.1(?::\d+)?(?:\/|$)/.test(s)) return ["--proto", "=http"];
40
+ return ["--proto", "=https"];
41
+ };
42
+
43
+ const VERSION_RE = /^(\d{1,6})\.(\d{1,6})\.(\d{1,6})(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/;
44
+ const MAX_VERSION = 40; // characters of a whole version
45
+ const MAX_DIGITS = 20; // digits of one pre-release number
46
+
47
+ /** `s` as a version's parts, or null: at most MAX_VERSION characters, and no pre-release number longer than MAX_DIGITS digits. */
48
+ function versionParts(s) {
49
+ const v = String(s || "");
50
+ const m = v.length <= MAX_VERSION ? VERSION_RE.exec(v) : null;
51
+ if (!m || (m[4] && m[4].split(".").some((x) => /^\d+$/.test(x) && x.length > MAX_DIGITS))) return null;
52
+ return m;
53
+ }
54
+
55
+ /** The first line of version.txt, or null when it is not a version (the file is untrusted: install.sh's character set, 40 characters at most). */
56
+ export function parseVersionText(text) {
57
+ const line = String(text || "").split(/\r?\n/)[0].trim();
58
+ return /^[0-9A-Za-z.+-]+$/.test(line) && versionParts(line) ? line : null;
59
+ }
60
+
61
+ /** Two strings of digits compared as numbers of any size (no Number: past 2^53 it rounds): -1, 0 or 1. */
62
+ function compareDigits(x, y) {
63
+ x = x.replace(/^0+(?=\d)/, "");
64
+ y = y.replace(/^0+(?=\d)/, "");
65
+ if (x.length !== y.length) return x.length < y.length ? -1 : 1;
66
+ return x === y ? 0 : x < y ? -1 : 1;
67
+ }
68
+
69
+ /** -1, 0 or 1 as `a` is older than, the same as or newer than `b` (numbers compared as numbers: 0.0.10 is newer than 0.0.9); null when either is not a version. */
70
+ export function compareVersions(a, b) {
71
+ const pa = versionParts(a);
72
+ const pb = versionParts(b);
73
+ if (!pa || !pb) return null;
74
+ for (let i = 1; i <= 3; i++) if (Number(pa[i]) !== Number(pb[i])) return Number(pa[i]) < Number(pb[i]) ? -1 : 1;
75
+ if (pa[4] === pb[4]) return 0;
76
+ if (!pa[4]) return 1; // 1.0.0 is newer than 1.0.0-rc.1
77
+ if (!pb[4]) return -1;
78
+ const xa = pa[4].split(".");
79
+ const xb = pb[4].split(".");
80
+ for (let i = 0; i < Math.max(xa.length, xb.length); i++) {
81
+ if (xa[i] === undefined) return -1;
82
+ if (xb[i] === undefined) return 1;
83
+ if (xa[i] === xb[i]) continue;
84
+ const na = /^\d+$/.test(xa[i]);
85
+ const nb = /^\d+$/.test(xb[i]);
86
+ if (na && nb) {
87
+ const c = compareDigits(xa[i], xb[i]);
88
+ if (c) return c;
89
+ continue;
90
+ }
91
+ if (na !== nb) return na ? -1 : 1;
92
+ return xa[i] < xb[i] ? -1 : 1;
93
+ }
94
+ return 0;
95
+ }
96
+
97
+ const real = (p) => {
98
+ try {
99
+ return fs.realpathSync(p);
100
+ } catch {
101
+ return path.resolve(p);
102
+ }
103
+ };
104
+
105
+ /**
106
+ * How the copy at `appRoot` was installed: "source" (a .git file or folder beside package.json; a worktree has a .git file), "installer"
107
+ * (~/.onionskin/app/<version>/, what install.sh makes), "npm" (…/node_modules/onionskin-cli) or "unknown" (an unzipped folder run directly).
108
+ */
109
+ export function detectRoute({ appRoot, home }) {
110
+ const root = real(appRoot);
111
+ if (fs.existsSync(path.join(root, ".git")) && fs.existsSync(path.join(root, "package.json"))) return { route: "source", root };
112
+ if (path.dirname(root) === path.join(real(home), PRODUCT.stateDirName, "app")) return { route: "installer", root };
113
+ if (path.basename(root) === "onionskin-cli" && path.basename(path.dirname(root)) === "node_modules") return { route: "npm", root };
114
+ return { route: "unknown", root };
115
+ }
116
+
117
+ /**
118
+ * Reads <base>/version.txt with curl (one request; curl follows the release redirect, https only). Returns
119
+ * {version} | {missing: true, url} (the release has no version.txt, as releases up to 0.0.3) | {bad: true, url} | {unreachable: detail, url}.
120
+ */
121
+ export function fetchLatest(base, { run = spawnSync } = {}) {
122
+ const url = `${String(base).replace(/\/+$/, "")}/${VERSION_FILE}`;
123
+ const r = run("curl", [...CURL, ...curlProto(url), url], { encoding: "utf8", timeout: 40_000, stdio: ["ignore", "pipe", "pipe"] });
124
+ if (r.error) return { unreachable: r.error.code === "ENOENT" ? "curl was not found" : r.error.message, url };
125
+ // the server answered 404 (curl says "returned error: 404", exit 22, or 56 over HTTP/2 as GitHub's redirect serves it); 37: no such file:// path
126
+ if (/returned error: 404\b/.test(r.stderr || "")) return { missing: true, url };
127
+ if (r.status === 37) return { missing: true, url };
128
+ if (r.status !== 0) return { unreachable: String(r.stderr || "").trim().split("\n").pop() || `curl exit ${r.status}`, url };
129
+ const version = parseVersionText(r.stdout);
130
+ return version ? { version, url } : { bad: true, url };
131
+ }
132
+
133
+ /** What a running server holds that an update would cut off (from GET /cli/open-work): plain phrases, never file names. */
134
+ export function workAtRisk(w) {
135
+ const n = (k) => (Number.isInteger(w?.[k]) && w[k] > 0 ? w[k] : 0);
136
+ const out = [];
137
+ const plural = (c, one, many) => `${c} ${c === 1 ? one : many}`;
138
+ if (n("open_sessions")) out.push(plural(n("open_sessions"), "file open in a browser tab now or in the last 20 minutes (edits typed there stay in the tab until Send)", "files open in a browser tab now or in the last 20 minutes (edits typed there stay in the tab until Send)"));
139
+ if (n("unsent_edits")) out.push(plural(n("unsent_edits"), "unsent edit", "unsent edits"));
140
+ if (n("draft_comments")) out.push(plural(n("draft_comments"), "draft comment", "draft comments"));
141
+ if (n("batches")) out.push(plural(n("batches"), "batch that was sent and is not finished", "batches that were sent and are not finished"));
142
+ return out;
143
+ }
144
+
145
+ export function parseUpdateArgs(args) {
146
+ const a = { check: false, yes: false, force: false };
147
+ for (const x of args) {
148
+ if (x === "--check") a.check = true;
149
+ else if (x === "--yes") a.yes = true;
150
+ else if (x === "--force") a.force = true;
151
+ else if (x.startsWith("-")) throw new Error(`unknown option ${x}`);
152
+ else throw new Error(`unexpected argument ${x}`);
153
+ }
154
+ return a;
155
+ }
156
+
157
+ /**
158
+ * @param {string[]} args the arguments after `update`
159
+ * @param {object} d version (running), appRoot, home, env, print(line), and the parts that act:
160
+ * fetchLatest(base), server: {healthy({reportMismatch: true}) -> boolean | "incompatible", openWork() -> {status, body}}, stop(), runGetSh(base) -> exit status,
161
+ * launcherVersion(route, root) -> string|null (the installed command's --version), run(cmd, args) -> exit status (npm --yes)
162
+ * @returns {Promise<number>} the exit code
163
+ */
164
+ export async function updateCommand(args, d) {
165
+ const a = parseUpdateArgs(args);
166
+ const say = d.print;
167
+ const home = d.home || os.homedir();
168
+ const env = d.env || process.env;
169
+ const shown = (p) => {
170
+ let t = String(p);
171
+ for (const h of [real(home), home]) t = t.split(h).join("~");
172
+ return t;
173
+ };
174
+ const base = env[`${PRODUCT.envPrefix}_BASE_URL`] || DEFAULT_BASE_URL;
175
+ if (!baseUrlOk(base)) {
176
+ say(`Refusing the release URL ${base} (only https:// is accepted). Nothing was changed.`);
177
+ return 1;
178
+ }
179
+ const { route, root } = detectRoute({ appRoot: d.appRoot, home });
180
+ const latest = (d.fetchLatest || fetchLatest)(base);
181
+ const cmp = latest.version ? compareVersions(d.version, latest.version) : null;
182
+
183
+ say(`This copy: ${PRODUCT.name} ${d.version}`);
184
+ if (latest.version) say(`Latest release: ${latest.version}`);
185
+ else if (latest.missing) say(`Latest release: unknown (the release has no ${VERSION_FILE}, as releases up to 0.0.3 do not)`);
186
+ else if (latest.bad) say(`Latest release: unknown (${latest.url} does not hold a version)`);
187
+ else say(`Latest release: unknown (could not reach ${latest.url}: ${latest.unreachable})`);
188
+
189
+ if (a.check) {
190
+ if (latest.unreachable) {
191
+ say("Could not check for an update. Nothing was changed.");
192
+ return 1;
193
+ }
194
+ if (!latest.version) say("Could not tell whether an update exists. Nothing was changed.");
195
+ else if (cmp < 0) say(`An update is available: run ${BIN} update`);
196
+ else if (cmp === 0) say("Up to date.");
197
+ else say("This copy is newer than the latest release. Nothing to update.");
198
+ return 0;
199
+ }
200
+
201
+ if (route === "source") {
202
+ say(`This is a source checkout (${shown(root)}). Update it with git: git pull, npm install, npm run build:web, then ${BIN} setup --force for the skill folders. Nothing was changed.`);
203
+ return 0;
204
+ }
205
+ if (route === "unknown") {
206
+ say(`This copy runs from ${shown(root)}, which is not the installer's folder (~/.onionskin/app), an npm install or a source checkout, so ${BIN} update cannot tell how to update it. Nothing was changed. To update, install again as the README says under Install.`);
207
+ return 1;
208
+ }
209
+ if (latest.unreachable) {
210
+ say("Not updated: could not reach the release. Nothing was changed.");
211
+ return 1;
212
+ }
213
+ if (cmp === 0) {
214
+ say(`Already up to date: ${PRODUCT.name} ${d.version} is the latest release. Nothing was changed.`);
215
+ return 0;
216
+ }
217
+ if (cmp > 0) {
218
+ say(`This copy (${d.version}) is newer than the latest release (${latest.version}). Nothing was changed.`);
219
+ return 0;
220
+ }
221
+ if (route === "npm" && !a.yes) {
222
+ say("This copy was installed with npm. To update it, run these two commands:");
223
+ for (const c of NPM_COMMANDS) say(` ${c}`);
224
+ say(`Or run ${BIN} update --yes to have them run for you.`);
225
+ return 0;
226
+ }
227
+ // without the release's version this cannot tell an update from a reinstall or a downgrade, so it installs nothing
228
+ if (!latest.version) {
229
+ say(`Not updated: the release does not say its version, so ${BIN} update cannot tell whether it is newer than this copy. Nothing was changed. To install it anyway, run the install command from the README again.`);
230
+ return 1;
231
+ }
232
+
233
+ // An update restarts the server: a tab open now cannot reconnect (the new server listens on a new port). Refuse while work is open.
234
+ let stopped = false;
235
+ const health = await d.server.healthy({ reportMismatch: true });
236
+ if (health) {
237
+ if (health === "incompatible" && !a.force) {
238
+ say(`A running Onionskin server from another version may hold unsent work: run ${BIN} stop, or ${BIN} update --force.`);
239
+ return 1;
240
+ }
241
+ const w = health === true ? await d.server.openWork() : null;
242
+ const sessions = Number.isInteger(w?.body?.sessions) ? w.body.sessions : null;
243
+ // a server that does not count its open tabs (open_sessions) cannot vouch for edits typed in a tab: like one without the route
244
+ const risk = w?.status === 200 && Number.isInteger(w.body?.open_sessions) ? workAtRisk(w.body) : null;
245
+ const reason = risk ? (risk.length ? `open work that the update would cut off: ${risk.join(", ")}` : null) : sessions === 0 ? null : "a server from an older version that cannot say whether its open sessions hold unsent work";
246
+ if (reason && !a.force) {
247
+ say(`Not updated: ${PRODUCT.name} is running with ${reason}. The update restarts ${PRODUCT.name}, and a browser tab that is open now cannot reconnect to it, so anything not sent from it would be lost. Send or finish that work, then press Close session in the file's page (or close its tab and wait 20 minutes), and run ${BIN} update again. To update anyway, run ${BIN} update --force. Nothing was changed.`);
248
+ return 1;
249
+ }
250
+ try {
251
+ await d.stop();
252
+ } catch (err) {
253
+ say(`Not updated: could not stop the running ${PRODUCT.name} server (${err?.message || err}). Nothing was changed.`);
254
+ say(`AGENT: the update did not run because the running ${PRODUCT.name} server could not be stopped. Tell the person nothing was changed and suggest running \`${BIN} stop\` or \`${BIN} doctor\`.`);
255
+ return 1;
256
+ }
257
+ stopped = true;
258
+ say(`Stopped the ${PRODUCT.name} server for the update${reason ? ` (--force: ${reason})` : ""}.`);
259
+ }
260
+
261
+ if (route === "npm") {
262
+ const limit = 10 * 60_000;
263
+ const deadline = Date.now() + limit;
264
+ const run = d.run || runInGroup;
265
+ for (const c of NPM_COMMANDS) {
266
+ say(`$ ${c}`);
267
+ const [cmd, ...rest] = c.split(" ");
268
+ const remaining = deadline - Date.now();
269
+ let timedOut = remaining <= 0;
270
+ const opts = { env, timeout: remaining, totalTimeout: limit, onTimeout: () => { timedOut = true; } };
271
+ const args = cmd === BIN ? [path.join(root, "src", "cli.js"), ...rest] : rest;
272
+ const status = timedOut ? 1 : await run(cmd === BIN ? process.execPath : cmd, args, opts);
273
+ if (timedOut) {
274
+ say("The npm update timed out after 10 minutes.");
275
+ return 1;
276
+ }
277
+ if (status !== 0) {
278
+ say(`Not updated: \`${c}\` failed (exit ${status}). Run the two commands above yourself to see why.`);
279
+ return 1;
280
+ }
281
+ }
282
+ const now = d.launcherVersion(route, root);
283
+ if (now !== latest.version) {
284
+ say(`Not updated as expected: npm finished, but this copy now says ${now || "nothing"}, not ${latest.version} (npm may not have that version yet). Run ${BIN} doctor.`);
285
+ return 1;
286
+ }
287
+ say(`UPDATED: ${PRODUCT.name} ${d.version} -> ${now} (npm).`);
288
+ if (stopped) say(`The server was stopped; it starts again on the next ${BIN} command. Open your file again to keep working.`);
289
+ return 0;
290
+ }
291
+
292
+ // the installer copy: get.sh from the same release folder, piped into sh as the README does
293
+ say(`Running the installer from ${base}/get.sh`);
294
+ let skillSetupFailed = false;
295
+ let outBuf = "";
296
+ let timedOutMs = 0;
297
+ const scanLine = (line) => {
298
+ if (line.includes("SKILL_SETUP_FAILED")) skillSetupFailed = true;
299
+ };
300
+ let noChecker = true;
301
+ try {
302
+ noChecker = !fs.lstatSync(path.join(home, PRODUCT.stateDirName, "venv", "pyvenv.cfg")).isFile();
303
+ } catch {
304
+ // An inaccessible or missing configuration means there is no environment to update.
305
+ }
306
+ const rawRes = await d.runGetSh(base, {
307
+ noChecker,
308
+ onOutput: (chunk) => {
309
+ outBuf += String(chunk);
310
+ let nl;
311
+ while ((nl = outBuf.indexOf("\n")) !== -1) {
312
+ scanLine(outBuf.slice(0, nl));
313
+ outBuf = outBuf.slice(nl + 1);
314
+ }
315
+ },
316
+ onTimeout: (ms) => {
317
+ timedOutMs = ms;
318
+ },
319
+ });
320
+ if (outBuf) scanLine(outBuf);
321
+ const status = typeof rawRes === "object" && rawRes !== null ? rawRes.status : rawRes;
322
+ if (typeof rawRes === "object" && rawRes !== null) {
323
+ if (rawRes.skillSetupFailed) skillSetupFailed = true;
324
+ if (rawRes.timedOut) timedOutMs = rawRes.timeoutMs || 360_000;
325
+ }
326
+ const now = d.launcherVersion(route, root);
327
+ if (status !== 0) {
328
+ if (timedOutMs) {
329
+ const secs = Math.max(1, Math.round(timedOutMs / 1000));
330
+ const dur = secs >= 60 && secs % 60 === 0 ? `${secs / 60} ${secs === 60 ? "minute" : "minutes"}` : `${secs}s`;
331
+ say(`Not updated: the installer timed out after ${dur}. The command now runs ${PRODUCT.name} ${now || "(version unknown)"}.`);
332
+ } else {
333
+ say(`Not updated: the installer stopped (exit ${status}; its lines above say why). The command now runs ${PRODUCT.name} ${now || "(version unknown)"}.`);
334
+ }
335
+ return 1;
336
+ }
337
+ if (now !== latest.version) {
338
+ say(`Not updated as expected: the installer finished, but ~/.local/bin/${BIN} --version says ${now || "nothing"}, not ${latest.version}. Run ${BIN} doctor.`);
339
+ return 1;
340
+ }
341
+ say("");
342
+ if (skillSetupFailed) {
343
+ say(`UPDATED (with warning): ${PRODUCT.name} ${d.version} -> ${now}, but skill setup failed. Run \`${BIN} setup\` to finish installing the agent skill. Snapshots and settings in ~/.onionskin were kept.`);
344
+ if (stopped) say(`The server was stopped; it starts again on the next ${BIN} command. Open your file again to keep working.`);
345
+ say(`AGENT: this was an update to ${now}, but skill setup failed (SKILL_SETUP_FAILED). Tell the person ${PRODUCT.name} is updated to ${now}, and that skill setup did not finish so they should run \`${BIN} setup\`.`);
346
+ } else {
347
+ say(`UPDATED: ${PRODUCT.name} ${d.version} -> ${now}. Snapshots and settings in ~/.onionskin were kept.`);
348
+ if (stopped) say(`The server was stopped; it starts again on the next ${BIN} command. Open your file again to keep working.`);
349
+ say(`AGENT: this was an update. Tell the person this one sentence, and not the installer's sentence above: ${PRODUCT.name} is updated to ${now}.`);
350
+ }
351
+ return 0;
352
+ }
353
+
354
+ const SIG_CODES = { SIGHUP: 129, SIGINT: 130, SIGTERM: 143 };
355
+
356
+ /** Runs a command in its own process group; forwards signals and kills the entire process group on timeout or parent exit. */
357
+ export function runInGroup(cmd, args, { env = process.env, timeout, totalTimeout = timeout, graceMs = 500, signalGraceMs = 10_000, onOutput, onTimeout }) {
358
+ return new Promise((resolve) => {
359
+ let settled = false;
360
+ let timedOut = false;
361
+ let signaled = null;
362
+ let killTimer = null;
363
+ let exitTimer = null;
364
+ let stdoutBuf = "";
365
+ let stderrBuf = "";
366
+
367
+ let child;
368
+ try {
369
+ child = spawn(cmd, args, {
370
+ detached: true,
371
+ stdio: onOutput ? ["ignore", "pipe", "pipe"] : ["ignore", "inherit", "inherit"],
372
+ env,
373
+ });
374
+ } catch {
375
+ return resolve(1);
376
+ }
377
+
378
+ const killGroup = (sig) => {
379
+ if (!child.pid) return;
380
+ try {
381
+ process.kill(-child.pid, sig);
382
+ } catch {}
383
+ };
384
+
385
+ const emitLine = (stream, line, nl) => {
386
+ if (onOutput) onOutput(line + nl);
387
+ const out = line.includes("SKILL_SETUP_FAILED")
388
+ ? ` Warning: skill setup did not finish; run '${BIN} setup' to finish installing the skill.`
389
+ : line;
390
+ stream.write(out + nl);
391
+ };
392
+
393
+ const feed = (stream, which, chunk) => {
394
+ let buf = (which === "out" ? stdoutBuf : stderrBuf) + chunk.toString("utf8");
395
+ let idx;
396
+ while ((idx = buf.indexOf("\n")) !== -1) {
397
+ emitLine(stream, buf.slice(0, idx), "\n");
398
+ buf = buf.slice(idx + 1);
399
+ }
400
+ if (which === "out") stdoutBuf = buf;
401
+ else stderrBuf = buf;
402
+ };
403
+
404
+ const flushStreams = () => {
405
+ if (stdoutBuf) {
406
+ const s = stdoutBuf;
407
+ stdoutBuf = "";
408
+ emitLine(process.stdout, s, "");
409
+ }
410
+ if (stderrBuf) {
411
+ const s = stderrBuf;
412
+ stderrBuf = "";
413
+ emitLine(process.stderr, s, "");
414
+ }
415
+ };
416
+
417
+ const onParentExit = () => {
418
+ killGroup("SIGTERM");
419
+ killGroup("SIGKILL");
420
+ };
421
+
422
+ const onSignal = (sig) => {
423
+ if (settled || signaled) return;
424
+ signaled = sig;
425
+ killGroup(sig);
426
+ clearTimeout(killTimer);
427
+ const hardStop = () => {
428
+ killGroup("SIGKILL");
429
+ try { child.stdout?.destroy(); } catch {}
430
+ try { child.stderr?.destroy(); } catch {}
431
+ finish(SIG_CODES[sig] || 1);
432
+ };
433
+ if (signalGraceMs === 0) return hardStop();
434
+ killTimer = setTimeout(hardStop, signalGraceMs);
435
+ killTimer.unref?.();
436
+ };
437
+
438
+ const sigHandlers = ["SIGINT", "SIGTERM", "SIGHUP"].map((sig) => {
439
+ const fn = () => onSignal(sig);
440
+ process.on(sig, fn);
441
+ return [sig, fn];
442
+ });
443
+ process.on("exit", onParentExit);
444
+
445
+ const cleanupListeners = () => {
446
+ for (const [sig, fn] of sigHandlers) process.off(sig, fn);
447
+ process.off("exit", onParentExit);
448
+ };
449
+
450
+ const finish = (code) => {
451
+ if (settled) return;
452
+ settled = true;
453
+ clearTimeout(timer);
454
+ clearTimeout(killTimer);
455
+ clearTimeout(exitTimer);
456
+ cleanupListeners();
457
+ // If the leader exited on timeout/signal while a descendant ignored SIGTERM/SIGINT, reap the group.
458
+ if (timedOut || signaled) killGroup("SIGKILL");
459
+ flushStreams();
460
+ if (timedOut) return resolve(1);
461
+ if (signaled) return resolve(SIG_CODES[signaled] || 1);
462
+ resolve(code ?? 1);
463
+ };
464
+
465
+ child.stdout?.on("data", (buf) => feed(process.stdout, "out", buf));
466
+ child.stderr?.on("data", (buf) => feed(process.stderr, "err", buf));
467
+ child.once("error", () => finish(1));
468
+ child.once("exit", (code) => {
469
+ if (settled) return;
470
+ exitTimer = setTimeout(() => {
471
+ try { child.stdout?.destroy(); } catch {}
472
+ try { child.stderr?.destroy(); } catch {}
473
+ finish(code);
474
+ }, graceMs);
475
+ exitTimer.unref?.();
476
+ });
477
+ child.once("close", (code) => finish(code));
478
+
479
+ const timer = setTimeout(() => {
480
+ timedOut = true;
481
+ if (onTimeout) onTimeout(totalTimeout);
482
+ killGroup("SIGTERM");
483
+ killTimer = setTimeout(() => {
484
+ killGroup("SIGKILL");
485
+ try { child.stdout?.destroy(); } catch {}
486
+ try { child.stderr?.destroy(); } catch {}
487
+ finish(1);
488
+ }, graceMs);
489
+ killTimer.unref?.();
490
+ }, timeout);
491
+ timer.unref?.();
492
+ });
493
+ }
494
+
495
+ /** The README one-liner, run for `base` (already checked by baseUrlOk): downloads get.sh to a 0600 temp file, then executes it with /bin/sh. */
496
+ export async function runGetShDefault(base, env = process.env, { timeout = 360_000, graceMs = 500, onOutput = () => {}, onTimeout, noChecker = false } = {}) {
497
+ const url = String(base).replace(/\/+$/, "");
498
+ const tmpDir = fs.mkdtempSync(path.join(env.TMPDIR || os.tmpdir(), "onionskin-get-"));
499
+ const tmpFile = path.join(tmpDir, "get.sh");
500
+ const deadline = Date.now() + timeout;
501
+ try {
502
+ const fd = fs.openSync(tmpFile, "wx", 0o600);
503
+ fs.closeSync(fd);
504
+ const dl = spawnSync(
505
+ "curl",
506
+ ["-fsSL", "--connect-timeout", "20", "--max-time", "60", ...curlProto(url), "--proto-redir", "=https", "-o", tmpFile, `${url}/get.sh`],
507
+ { stdio: ["ignore", "inherit", "inherit"], env, timeout: Math.min(70_000, timeout) }
508
+ );
509
+ if (dl.error || dl.status !== 0) return dl.status ?? 1;
510
+ fs.chmodSync(tmpFile, 0o600);
511
+ const remaining = Math.max(1, deadline - Date.now());
512
+ const args = noChecker ? [tmpFile, "--no-checker"] : [tmpFile];
513
+ return await runInGroup("/bin/sh", args, { env: { ...env, ONIONSKIN_BASE_URL: url }, timeout: remaining, totalTimeout: timeout, graceMs, onOutput, onTimeout });
514
+ } finally {
515
+ fs.rmSync(tmpDir, { recursive: true, force: true });
516
+ }
517
+ }
@@ -7,6 +7,7 @@
7
7
  // A draft is agent text: untrusted until the person approves it word for word. The check makes sure that what reaches the page is plain
8
8
  // text in the fixed layout, with nothing hidden, no link and no markup, so that every word the person approves is a word they could see.
9
9
  // Ceiling: it cannot make a persuasive sentence harmless. That is what the person's review is for.
10
+ import { nfc } from "./nfc.js";
10
11
 
11
12
  /** The twelve sections, in order. `title` is the exact `## ` heading; `max` the cap of its body, in characters (code points). */
12
13
  export const SECTIONS = [
@@ -70,19 +71,24 @@ const BLANK_RUN_MAX = 2;
70
71
  const TLD = /^\p{L}{2,}$/u;
71
72
  const NAME_CHAR = /[\p{L}\p{N}-]/u;
72
73
  const PORT = /:\d{1,5}$/u;
74
+ const LOOKALIKE_DOTS = /[\u3002\uFF61\uFF0E\u2024]/gu;
75
+ const INVISIBLE = /[\u00AD\u034F\u061C\u115F\u1160\u17B4\u17B5\u180E\u200B-\u200F\u202A-\u202E\u2060-\u2064\u2066-\u2069\u2800\u3164\uFEFF\uFFA0]/gu;
76
+ /** NFKC, invisible characters stripped, and look-alike dots folded to ASCII "." for link checks. */
77
+ export const foldLinkText = (s) => nfc(String(s ?? "").replace(INVISIBLE, ""), "NFKC").replace(INVISIBLE, "").replace(LOOKALIKE_DOTS, ".");
78
+
73
79
  /**
74
- * A domain with a path, at least as strict as the examples' rule (LINK in src/voice.js, a backtracking pattern that is fine on 400
75
- * characters but not on 32 KB): any part of a word before one of its slashes that ends in a name character, a dot and two or more letters,
76
- * an optional port after it ("evil.example/x", "//evil.example/login", "x/evil.example/", "evil.example:8080/x"). "4.5/5", "and/or" and
77
- * "e.g. 3/4" pass. One pass over each word.
80
+ * A domain with a path (0.0.5 rule), linear-time on 32 KB. Expects `foldLinkText(s)`: any part of a word before one of its slashes
81
+ * that ends in a name character, a dot and two or more letters, with an optional port after it ("evil.example/x",
82
+ * "//evil.example/login", "x/evil.example/", "evil.example:8080/x"). Bare domains without a slash are intentionally not matched.
83
+ * "4.5/5", "and/or", "e.g. 3/4", "main.py:42", "Socket.IO", and "Finished.No" pass. One pass over each word.
78
84
  */
79
- function domainPath(s) {
85
+ export function domainPath(s) {
80
86
  for (const word of s.split(/\s+/u)) {
81
87
  const parts = word.split("/");
82
88
  for (let i = 0; i < parts.length - 1; i++) {
83
89
  const name = parts[i].replace(PORT, "");
84
90
  const dot = name.lastIndexOf(".");
85
- if (dot > 0 && NAME_CHAR.test(name[dot - 1]) && TLD.test(name.slice(dot + 1))) return true;
91
+ if (dot > 0 && NAME_CHAR.test([...name.slice(0, dot)].at(-1)) && TLD.test(name.slice(dot + 1))) return true;
86
92
  }
87
93
  }
88
94
  return false;
@@ -101,7 +107,7 @@ function splitSections(lines, { all }) {
101
107
  let titleSeen = false;
102
108
  for (const line of lines) {
103
109
  const t = line.trimEnd();
104
- if (/^\s*#/u.test(t.normalize("NFKC"))) { // a fullwidth number sign looks like # on the page
110
+ if (/^\s*#/u.test(nfc(t, "NFKC"))) { // a fullwidth number sign looks like # on the page
105
111
  if (!cur && !titleSeen && t === DRAFT_TITLE) {
106
112
  titleSeen = true;
107
113
  continue;
@@ -138,9 +144,9 @@ export function cleanVoiceDraft(value) {
138
144
  if (typeof value !== "string") return refuse("bad_structure", "the draft must be text");
139
145
  // size first: a huge body is refused before any regex or normalization runs over it
140
146
  if (Buffer.byteLength(value, "utf8") > DRAFT_MAX_BYTES) return refuse("too_long", `the draft is over ${DRAFT_MAX_BYTES} bytes`);
141
- const text = value.replace(/\r\n?/gu, "\n").normalize("NFC").replace(VS, "");
147
+ const text = nfc(value.replace(/\r\n?/gu, "\n")).replace(VS, "");
142
148
  if (HIDDEN.test(text)) return refuse("hidden_characters", "the draft holds a hidden or control character (tabs included)");
143
- const nfkc = text.normalize("NFKC");
149
+ const nfkc = foldLinkText(text);
144
150
  if (MARKUP.some((re) => re.test(nfkc)) || domainPath(nfkc)) return refuse("link_or_markup", "the draft holds a link, an address, @, HTML or Markdown markup, or a code fence");
145
151
  const lines = text.split("\n");
146
152
  // padding: a run of blank lines could push a planted line far below the part of a box the person reads first
@@ -4,7 +4,6 @@
4
4
  // What reaches the store from a Send: the five booleans per edit pair and the untrusted examples (from the worker, over three stored
5
5
  // versions), and the trait ids that fixed English phrases in the person's own comments point at. Never a comment's text, a file name or a
6
6
  // path. Everything here fails quietly: learning never stops a Send, and a step that fails learns nothing.
7
- import { Worker } from "node:worker_threads";
8
7
  import { buildVoiceBatch, printedSize } from "./batch.js";
9
8
  import { env } from "./constants.js";
10
9
  import { checkRegularOwned, readDoc, sha256 } from "./fsutil.js";
@@ -17,6 +16,7 @@ import { parseGuide } from "./voice-guide.js";
17
16
  import { budget, pickRequests, pickRewrites, voiceBatchMax, voiceShares } from "./voice-samples.js";
18
17
  import { VoiceError, appendRecords, discardDraft, forget, readDraft, readEvidence, readGuide, setAnswer, setLearning, writeGuide } from "./voice-store.js";
19
18
  import { commentHits, suggestions } from "./voice.js";
19
+ import { startWorker } from "./offload.js";
20
20
 
21
21
  const WORKER_TIMEOUT_MS = () => Number(env("VOICE_TIMEOUT_MS", 60_000));
22
22
  const TRAIT_IDS = TRAITS.map((t) => t.id);
@@ -65,6 +65,7 @@ export function voiceJob(s, { last = null, diskSha = null, writtenSha = null, se
65
65
 
66
66
  const running = new Set(); // live workers, ended by stopVoice
67
67
  let epoch = 0; // moved on by Forget and the learning switch: a job that started before either stores nothing
68
+ let activeSampleToken = null; // concurrency cap of 1 active voice-sample-worker to avoid >450MB OOM spikes
68
69
 
69
70
  /** Runs src/voice-worker.js over three blob paths. Resolves {pairs, examples}, or null when it fails, crashes or times out (then it is ended). */
70
71
  function analyze(versions) {
@@ -81,7 +82,7 @@ function analyze(versions) {
81
82
  resolve(r);
82
83
  };
83
84
  try {
84
- worker = new Worker(new URL("./voice-worker.js", import.meta.url), { workerData: versions });
85
+ worker = startWorker("voice-worker.js", { workerData: versions });
85
86
  } catch {
86
87
  return finish(null);
87
88
  }
@@ -106,6 +107,7 @@ export async function learnFromSend(job) {
106
107
 
107
108
  /** Ends every running learning worker (the server is stopping). */
108
109
  export function stopVoice() {
110
+ activeSampleToken = null;
109
111
  for (const w of [...running]) w.terminate();
110
112
  running.clear();
111
113
  }
@@ -221,8 +223,11 @@ function agentPairs(s) {
221
223
  return runs.slice(-AGENT_PAIRS_MAX).map((u) => ({ before: s.snapshots.blob(u.before), after: s.snapshots.blob(u.after) }));
222
224
  }
223
225
 
224
- /** Runs src/voice-sample-worker.js. Resolves {sources} or {error: code}; a crash or a timeout (60 s) is "unreadable" and the worker is ended. */
226
+ /** Runs src/voice-sample-worker.js. Resolves {sources} or {error: code}; at most 1 active worker at a time. */
225
227
  function sampleInWorker(job) {
228
+ if (activeSampleToken !== null) return Promise.resolve({ error: "busy", busy: true });
229
+ const token = {};
230
+ activeSampleToken = token;
226
231
  return new Promise((resolve) => {
227
232
  let worker;
228
233
  let settled = false;
@@ -230,13 +235,14 @@ function sampleInWorker(job) {
230
235
  const finish = (r) => {
231
236
  if (settled) return;
232
237
  settled = true;
238
+ if (activeSampleToken === token) activeSampleToken = null;
233
239
  clearTimeout(timer);
234
240
  running.delete(worker);
235
241
  worker?.terminate();
236
242
  resolve(r);
237
243
  };
238
244
  try {
239
- worker = new Worker(new URL("./voice-sample-worker.js", import.meta.url), { workerData: job });
245
+ worker = startWorker("voice-sample-worker.js", { workerData: job });
240
246
  } catch {
241
247
  return finish({ error: "unreadable" });
242
248
  }
@@ -270,11 +276,15 @@ export async function buildVoiceRequest(sessions, key, body) {
270
276
  if (s.closed) throw httpError(409, "session closed", "closed");
271
277
  if (s.pending || s.delivered) throw httpError(409, "Wait for the agent to finish, then try again.", "busy");
272
278
  if (body.open_file && s.format === "xlsx") throw new VoiceError("Excel files are not used as writing samples.");
279
+ if (body.open_file && s.format === "html") throw new VoiceError("HTML pages are not used as writing samples."); // PLAN-html D9
273
280
  const examples = body.edits ? readEvidence().examples : [];
274
281
  if (!body.open_file && !files.length && !body.texts.length && !examples.length) throw new VoiceError(NO_SAMPLE);
275
282
  const saved = readGuide();
276
283
  if (body.improve && !saved.text) throw new VoiceError("There is no saved guide to improve.");
277
284
 
285
+ if ((body.open_file || files.length || body.texts.length) && activeSampleToken !== null) {
286
+ throw httpError(429, "Another voice guide is being built right now. Wait a moment, then try again.", "busy");
287
+ }
278
288
  let open = null;
279
289
  if (body.open_file) {
280
290
  let bytes;
@@ -297,6 +307,7 @@ export async function buildVoiceRequest(sessions, key, body) {
297
307
  let sampled = { sources: [] };
298
308
  if (sources) {
299
309
  sampled = await sampleInWorker({ open, files, texts: body.texts, shares: shares.source });
310
+ if (sampled.busy) throw httpError(429, "Another voice guide is being built right now. Wait a moment, then try again.", "busy");
300
311
  if (sampled.error) throw new VoiceError(UNREADABLE);
301
312
  }
302
313
  const evidence = {