onionskin-cli 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +98 -0
  2. package/README.md +4 -4
  3. package/package.json +1 -1
  4. package/scripts/checker/check.py +24 -5
  5. package/scripts/checker/requirements.txt +482 -5
  6. package/scripts/requirements.txt +482 -5
  7. package/scripts/rewriter/apple/main.swift +1 -1
  8. package/scripts/rewriter/apple/onionskin-fm +0 -0
  9. package/scripts/rewriter/apple/onionskin-fm.json +1 -1
  10. package/skills/onionskin/SKILL.md +4 -4
  11. package/skills/onionskin/reference/ops.md +1 -1
  12. package/src/agent-runner.js +94 -15
  13. package/src/checker.js +238 -33
  14. package/src/cli.js +173 -33
  15. package/src/doctor.js +12 -3
  16. package/src/docx.js +130 -63
  17. package/src/fsutil.js +13 -5
  18. package/src/history.js +11 -2
  19. package/src/html-render.js +74 -15
  20. package/src/html-utf8.js +1 -1
  21. package/src/html.js +3 -1
  22. package/src/merge-package.js +82 -26
  23. package/src/offload.js +25 -6
  24. package/src/ops/docx-table.js +5 -3
  25. package/src/ops/pptx.js +8 -2
  26. package/src/ops/xlsx.js +114 -8
  27. package/src/package.js +17 -2
  28. package/src/pptx.js +244 -8
  29. package/src/question.js +10 -3
  30. package/src/quick.js +1 -1
  31. package/src/redline-check.js +12 -2
  32. package/src/rewriter.js +38 -8
  33. package/src/server-entry.js +4 -3
  34. package/src/server.js +97 -28
  35. package/src/session.js +13 -2
  36. package/src/setup.js +20 -2
  37. package/src/update.js +248 -19
  38. package/src/voice-guide.js +12 -7
  39. package/src/voice-learn.js +11 -1
  40. package/src/voice-store.js +11 -4
  41. package/src/voice.js +2 -1
  42. package/src/xlsx.js +25 -5
  43. package/src/zip-write.js +8 -4
  44. package/src/zip.js +64 -13
  45. package/web/dist/assets/{index-DytKkMdi.js → index-CjsTf5gj.js} +108 -106
  46. package/web/dist/assets/index-DnbLtHhG.css +1 -0
  47. package/web/dist/index.html +2 -2
  48. package/web/dist/assets/index-DkNx0LSH.css +0 -1
package/src/update.js CHANGED
@@ -4,7 +4,7 @@
4
4
  // SHA-256 check). No prompts, no sudo; snapshots and the state folder are never deleted here.
5
5
  //
6
6
  // Everything this command needs after get.sh starts is imported at the top: install.sh deletes the app folder this process runs from.
7
- import { spawnSync } from "node:child_process";
7
+ import { spawn, spawnSync } from "node:child_process";
8
8
  import fs from "node:fs";
9
9
  import os from "node:os";
10
10
  import path from "node:path";
@@ -31,8 +31,14 @@ export function baseUrlOk(u) {
31
31
  return m[1] === "https" || /^127\.0\.0\.1(?::\d+)?$/.test(m[2]);
32
32
  }
33
33
 
34
- /** curl's --proto for a URL baseUrlOk accepted: plain http only for 127.0.0.1 (redirects stay https only, CURL). */
35
- export const curlProto = (u) => ["--proto", String(u).startsWith("http://") ? "=https,file,http" : "=https,file"];
34
+ /** curl's --proto for a URL baseUrlOk accepted: only the scheme of the URL itself (redirects stay https only, CURL). */
35
+ export const curlProto = (u) => {
36
+ const s = String(u ?? "");
37
+ if (s.startsWith("https://")) return ["--proto", "=https"];
38
+ if (s.startsWith("file://")) return ["--proto", "=file"];
39
+ if (/^http:\/\/127\.0\.0\.1(?::\d+)?(?:\/|$)/.test(s)) return ["--proto", "=http"];
40
+ return ["--proto", "=https"];
41
+ };
36
42
 
37
43
  const VERSION_RE = /^(\d{1,6})\.(\d{1,6})\.(\d{1,6})(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/;
38
44
  const MAX_VERSION = 40; // characters of a whole version
@@ -151,7 +157,7 @@ export function parseUpdateArgs(args) {
151
157
  /**
152
158
  * @param {string[]} args the arguments after `update`
153
159
  * @param {object} d version (running), appRoot, home, env, print(line), and the parts that act:
154
- * fetchLatest(base), server: {healthy(), openWork() -> {status, body}}, stop(), runGetSh(base) -> exit status,
160
+ * fetchLatest(base), server: {healthy({reportMismatch: true}) -> boolean | "incompatible", openWork() -> {status, body}}, stop(), runGetSh(base) -> exit status,
155
161
  * launcherVersion(route, root) -> string|null (the installed command's --version), run(cmd, args) -> exit status (npm --yes)
156
162
  * @returns {Promise<number>} the exit code
157
163
  */
@@ -226,26 +232,48 @@ export async function updateCommand(args, d) {
226
232
 
227
233
  // An update restarts the server: a tab open now cannot reconnect (the new server listens on a new port). Refuse while work is open.
228
234
  let stopped = false;
229
- if (await d.server.healthy()) {
230
- const w = await d.server.openWork();
231
- const sessions = Number.isInteger(w.body?.sessions) ? w.body.sessions : null;
235
+ const health = await d.server.healthy({ reportMismatch: true });
236
+ if (health) {
237
+ if (health === "incompatible" && !a.force) {
238
+ say(`A running Onionskin server from another version may hold unsent work: run ${BIN} stop, or ${BIN} update --force.`);
239
+ return 1;
240
+ }
241
+ const w = health === true ? await d.server.openWork() : null;
242
+ const sessions = Number.isInteger(w?.body?.sessions) ? w.body.sessions : null;
232
243
  // a server that does not count its open tabs (open_sessions) cannot vouch for edits typed in a tab: like one without the route
233
- const risk = w.status === 200 && Number.isInteger(w.body?.open_sessions) ? workAtRisk(w.body) : null;
244
+ const risk = w?.status === 200 && Number.isInteger(w.body?.open_sessions) ? workAtRisk(w.body) : null;
234
245
  const reason = risk ? (risk.length ? `open work that the update would cut off: ${risk.join(", ")}` : null) : sessions === 0 ? null : "a server from an older version that cannot say whether its open sessions hold unsent work";
235
246
  if (reason && !a.force) {
236
247
  say(`Not updated: ${PRODUCT.name} is running with ${reason}. The update restarts ${PRODUCT.name}, and a browser tab that is open now cannot reconnect to it, so anything not sent from it would be lost. Send or finish that work, then press Close session in the file's page (or close its tab and wait 20 minutes), and run ${BIN} update again. To update anyway, run ${BIN} update --force. Nothing was changed.`);
237
248
  return 1;
238
249
  }
239
- await d.stop();
250
+ try {
251
+ await d.stop();
252
+ } catch (err) {
253
+ say(`Not updated: could not stop the running ${PRODUCT.name} server (${err?.message || err}). Nothing was changed.`);
254
+ say(`AGENT: the update did not run because the running ${PRODUCT.name} server could not be stopped. Tell the person nothing was changed and suggest running \`${BIN} stop\` or \`${BIN} doctor\`.`);
255
+ return 1;
256
+ }
240
257
  stopped = true;
241
258
  say(`Stopped the ${PRODUCT.name} server for the update${reason ? ` (--force: ${reason})` : ""}.`);
242
259
  }
243
260
 
244
261
  if (route === "npm") {
262
+ const limit = 10 * 60_000;
263
+ const deadline = Date.now() + limit;
264
+ const run = d.run || runInGroup;
245
265
  for (const c of NPM_COMMANDS) {
246
266
  say(`$ ${c}`);
247
267
  const [cmd, ...rest] = c.split(" ");
248
- const status = cmd === BIN ? d.run(process.execPath, [path.join(root, "src", "cli.js"), ...rest]) : d.run(cmd, rest);
268
+ const remaining = deadline - Date.now();
269
+ let timedOut = remaining <= 0;
270
+ const opts = { env, timeout: remaining, totalTimeout: limit, onTimeout: () => { timedOut = true; } };
271
+ const args = cmd === BIN ? [path.join(root, "src", "cli.js"), ...rest] : rest;
272
+ const status = timedOut ? 1 : await run(cmd === BIN ? process.execPath : cmd, args, opts);
273
+ if (timedOut) {
274
+ say("The npm update timed out after 10 minutes.");
275
+ return 1;
276
+ }
249
277
  if (status !== 0) {
250
278
  say(`Not updated: \`${c}\` failed (exit ${status}). Run the two commands above yourself to see why.`);
251
279
  return 1;
@@ -263,10 +291,47 @@ export async function updateCommand(args, d) {
263
291
 
264
292
  // the installer copy: get.sh from the same release folder, piped into sh as the README does
265
293
  say(`Running the installer from ${base}/get.sh`);
266
- const status = d.runGetSh(base);
294
+ let skillSetupFailed = false;
295
+ let outBuf = "";
296
+ let timedOutMs = 0;
297
+ const scanLine = (line) => {
298
+ if (line.includes("SKILL_SETUP_FAILED")) skillSetupFailed = true;
299
+ };
300
+ let noChecker = true;
301
+ try {
302
+ noChecker = !fs.lstatSync(path.join(home, PRODUCT.stateDirName, "venv", "pyvenv.cfg")).isFile();
303
+ } catch {
304
+ // An inaccessible or missing configuration means there is no environment to update.
305
+ }
306
+ const rawRes = await d.runGetSh(base, {
307
+ noChecker,
308
+ onOutput: (chunk) => {
309
+ outBuf += String(chunk);
310
+ let nl;
311
+ while ((nl = outBuf.indexOf("\n")) !== -1) {
312
+ scanLine(outBuf.slice(0, nl));
313
+ outBuf = outBuf.slice(nl + 1);
314
+ }
315
+ },
316
+ onTimeout: (ms) => {
317
+ timedOutMs = ms;
318
+ },
319
+ });
320
+ if (outBuf) scanLine(outBuf);
321
+ const status = typeof rawRes === "object" && rawRes !== null ? rawRes.status : rawRes;
322
+ if (typeof rawRes === "object" && rawRes !== null) {
323
+ if (rawRes.skillSetupFailed) skillSetupFailed = true;
324
+ if (rawRes.timedOut) timedOutMs = rawRes.timeoutMs || 360_000;
325
+ }
267
326
  const now = d.launcherVersion(route, root);
268
327
  if (status !== 0) {
269
- say(`Not updated: the installer stopped (exit ${status}; its lines above say why). The command now runs ${PRODUCT.name} ${now || "(version unknown)"}.`);
328
+ if (timedOutMs) {
329
+ const secs = Math.max(1, Math.round(timedOutMs / 1000));
330
+ const dur = secs >= 60 && secs % 60 === 0 ? `${secs / 60} ${secs === 60 ? "minute" : "minutes"}` : `${secs}s`;
331
+ say(`Not updated: the installer timed out after ${dur}. The command now runs ${PRODUCT.name} ${now || "(version unknown)"}.`);
332
+ } else {
333
+ say(`Not updated: the installer stopped (exit ${status}; its lines above say why). The command now runs ${PRODUCT.name} ${now || "(version unknown)"}.`);
334
+ }
270
335
  return 1;
271
336
  }
272
337
  if (now !== latest.version) {
@@ -274,15 +339,179 @@ export async function updateCommand(args, d) {
274
339
  return 1;
275
340
  }
276
341
  say("");
277
- say(`UPDATED: ${PRODUCT.name} ${d.version} -> ${now}. Snapshots and settings in ~/.onionskin were kept.`);
278
- if (stopped) say(`The server was stopped; it starts again on the next ${BIN} command. Open your file again to keep working.`);
279
- say(`AGENT: this was an update. Tell the person this one sentence, and not the installer's sentence above: ${PRODUCT.name} is updated to ${now}.`);
342
+ if (skillSetupFailed) {
343
+ say(`UPDATED (with warning): ${PRODUCT.name} ${d.version} -> ${now}, but skill setup failed. Run \`${BIN} setup\` to finish installing the agent skill. Snapshots and settings in ~/.onionskin were kept.`);
344
+ if (stopped) say(`The server was stopped; it starts again on the next ${BIN} command. Open your file again to keep working.`);
345
+ say(`AGENT: this was an update to ${now}, but skill setup failed (SKILL_SETUP_FAILED). Tell the person ${PRODUCT.name} is updated to ${now}, and that skill setup did not finish so they should run \`${BIN} setup\`.`);
346
+ } else {
347
+ say(`UPDATED: ${PRODUCT.name} ${d.version} -> ${now}. Snapshots and settings in ~/.onionskin were kept.`);
348
+ if (stopped) say(`The server was stopped; it starts again on the next ${BIN} command. Open your file again to keep working.`);
349
+ say(`AGENT: this was an update. Tell the person this one sentence, and not the installer's sentence above: ${PRODUCT.name} is updated to ${now}.`);
350
+ }
280
351
  return 0;
281
352
  }
282
353
 
283
- /** The README one-liner, run for `base` (already checked by baseUrlOk): stdin closed, output to this terminal. pipefail, so a failed download is not a success. */
284
- export function runGetShDefault(base, env = process.env) {
354
+ const SIG_CODES = { SIGHUP: 129, SIGINT: 130, SIGTERM: 143 };
355
+
356
+ /** Runs a command in its own process group; forwards signals and kills the entire process group on timeout or parent exit. */
357
+ export function runInGroup(cmd, args, { env = process.env, timeout, totalTimeout = timeout, graceMs = 500, signalGraceMs = 10_000, onOutput, onTimeout }) {
358
+ return new Promise((resolve) => {
359
+ let settled = false;
360
+ let timedOut = false;
361
+ let signaled = null;
362
+ let killTimer = null;
363
+ let exitTimer = null;
364
+ let stdoutBuf = "";
365
+ let stderrBuf = "";
366
+
367
+ let child;
368
+ try {
369
+ child = spawn(cmd, args, {
370
+ detached: true,
371
+ stdio: onOutput ? ["ignore", "pipe", "pipe"] : ["ignore", "inherit", "inherit"],
372
+ env,
373
+ });
374
+ } catch {
375
+ return resolve(1);
376
+ }
377
+
378
+ const killGroup = (sig) => {
379
+ if (!child.pid) return;
380
+ try {
381
+ process.kill(-child.pid, sig);
382
+ } catch {}
383
+ };
384
+
385
+ const emitLine = (stream, line, nl) => {
386
+ if (onOutput) onOutput(line + nl);
387
+ const out = line.includes("SKILL_SETUP_FAILED")
388
+ ? ` Warning: skill setup did not finish; run '${BIN} setup' to finish installing the skill.`
389
+ : line;
390
+ stream.write(out + nl);
391
+ };
392
+
393
+ const feed = (stream, which, chunk) => {
394
+ let buf = (which === "out" ? stdoutBuf : stderrBuf) + chunk.toString("utf8");
395
+ let idx;
396
+ while ((idx = buf.indexOf("\n")) !== -1) {
397
+ emitLine(stream, buf.slice(0, idx), "\n");
398
+ buf = buf.slice(idx + 1);
399
+ }
400
+ if (which === "out") stdoutBuf = buf;
401
+ else stderrBuf = buf;
402
+ };
403
+
404
+ const flushStreams = () => {
405
+ if (stdoutBuf) {
406
+ const s = stdoutBuf;
407
+ stdoutBuf = "";
408
+ emitLine(process.stdout, s, "");
409
+ }
410
+ if (stderrBuf) {
411
+ const s = stderrBuf;
412
+ stderrBuf = "";
413
+ emitLine(process.stderr, s, "");
414
+ }
415
+ };
416
+
417
+ const onParentExit = () => {
418
+ killGroup("SIGTERM");
419
+ killGroup("SIGKILL");
420
+ };
421
+
422
+ const onSignal = (sig) => {
423
+ if (settled || signaled) return;
424
+ signaled = sig;
425
+ killGroup(sig);
426
+ clearTimeout(killTimer);
427
+ const hardStop = () => {
428
+ killGroup("SIGKILL");
429
+ try { child.stdout?.destroy(); } catch {}
430
+ try { child.stderr?.destroy(); } catch {}
431
+ finish(SIG_CODES[sig] || 1);
432
+ };
433
+ if (signalGraceMs === 0) return hardStop();
434
+ killTimer = setTimeout(hardStop, signalGraceMs);
435
+ killTimer.unref?.();
436
+ };
437
+
438
+ const sigHandlers = ["SIGINT", "SIGTERM", "SIGHUP"].map((sig) => {
439
+ const fn = () => onSignal(sig);
440
+ process.on(sig, fn);
441
+ return [sig, fn];
442
+ });
443
+ process.on("exit", onParentExit);
444
+
445
+ const cleanupListeners = () => {
446
+ for (const [sig, fn] of sigHandlers) process.off(sig, fn);
447
+ process.off("exit", onParentExit);
448
+ };
449
+
450
+ const finish = (code) => {
451
+ if (settled) return;
452
+ settled = true;
453
+ clearTimeout(timer);
454
+ clearTimeout(killTimer);
455
+ clearTimeout(exitTimer);
456
+ cleanupListeners();
457
+ // If the leader exited on timeout/signal while a descendant ignored SIGTERM/SIGINT, reap the group.
458
+ if (timedOut || signaled) killGroup("SIGKILL");
459
+ flushStreams();
460
+ if (timedOut) return resolve(1);
461
+ if (signaled) return resolve(SIG_CODES[signaled] || 1);
462
+ resolve(code ?? 1);
463
+ };
464
+
465
+ child.stdout?.on("data", (buf) => feed(process.stdout, "out", buf));
466
+ child.stderr?.on("data", (buf) => feed(process.stderr, "err", buf));
467
+ child.once("error", () => finish(1));
468
+ child.once("exit", (code) => {
469
+ if (settled) return;
470
+ exitTimer = setTimeout(() => {
471
+ try { child.stdout?.destroy(); } catch {}
472
+ try { child.stderr?.destroy(); } catch {}
473
+ finish(code);
474
+ }, graceMs);
475
+ exitTimer.unref?.();
476
+ });
477
+ child.once("close", (code) => finish(code));
478
+
479
+ const timer = setTimeout(() => {
480
+ timedOut = true;
481
+ if (onTimeout) onTimeout(totalTimeout);
482
+ killGroup("SIGTERM");
483
+ killTimer = setTimeout(() => {
484
+ killGroup("SIGKILL");
485
+ try { child.stdout?.destroy(); } catch {}
486
+ try { child.stderr?.destroy(); } catch {}
487
+ finish(1);
488
+ }, graceMs);
489
+ killTimer.unref?.();
490
+ }, timeout);
491
+ timer.unref?.();
492
+ });
493
+ }
494
+
495
+ /** The README one-liner, run for `base` (already checked by baseUrlOk): downloads get.sh to a 0600 temp file, then executes it with /bin/sh. */
496
+ export async function runGetShDefault(base, env = process.env, { timeout = 360_000, graceMs = 500, onOutput = () => {}, onTimeout, noChecker = false } = {}) {
285
497
  const url = String(base).replace(/\/+$/, "");
286
- const r = spawnSync("/bin/sh", ["-c", 'set -o pipefail; curl -fsSL "$2" "$3" --proto-redir =https "$1/get.sh" | ONIONSKIN_BASE_URL="$1" sh', "sh", url, ...curlProto(url)], { stdio: ["ignore", "inherit", "inherit"], env });
287
- return r.status ?? 1;
498
+ const tmpDir = fs.mkdtempSync(path.join(env.TMPDIR || os.tmpdir(), "onionskin-get-"));
499
+ const tmpFile = path.join(tmpDir, "get.sh");
500
+ const deadline = Date.now() + timeout;
501
+ try {
502
+ const fd = fs.openSync(tmpFile, "wx", 0o600);
503
+ fs.closeSync(fd);
504
+ const dl = spawnSync(
505
+ "curl",
506
+ ["-fsSL", "--connect-timeout", "20", "--max-time", "60", ...curlProto(url), "--proto-redir", "=https", "-o", tmpFile, `${url}/get.sh`],
507
+ { stdio: ["ignore", "inherit", "inherit"], env, timeout: Math.min(70_000, timeout) }
508
+ );
509
+ if (dl.error || dl.status !== 0) return dl.status ?? 1;
510
+ fs.chmodSync(tmpFile, 0o600);
511
+ const remaining = Math.max(1, deadline - Date.now());
512
+ const args = noChecker ? [tmpFile, "--no-checker"] : [tmpFile];
513
+ return await runInGroup("/bin/sh", args, { env: { ...env, ONIONSKIN_BASE_URL: url }, timeout: remaining, totalTimeout: timeout, graceMs, onOutput, onTimeout });
514
+ } finally {
515
+ fs.rmSync(tmpDir, { recursive: true, force: true });
516
+ }
288
517
  }
@@ -71,19 +71,24 @@ const BLANK_RUN_MAX = 2;
71
71
  const TLD = /^\p{L}{2,}$/u;
72
72
  const NAME_CHAR = /[\p{L}\p{N}-]/u;
73
73
  const PORT = /:\d{1,5}$/u;
74
+ const LOOKALIKE_DOTS = /[\u3002\uFF61\uFF0E\u2024]/gu;
75
+ const INVISIBLE = /[\u00AD\u034F\u061C\u115F\u1160\u17B4\u17B5\u180E\u200B-\u200F\u202A-\u202E\u2060-\u2064\u2066-\u2069\u2800\u3164\uFEFF\uFFA0]/gu;
76
+ /** NFKC, invisible characters stripped, and look-alike dots folded to ASCII "." for link checks. */
77
+ export const foldLinkText = (s) => nfc(String(s ?? "").replace(INVISIBLE, ""), "NFKC").replace(INVISIBLE, "").replace(LOOKALIKE_DOTS, ".");
78
+
74
79
  /**
75
- * A domain with a path, at least as strict as the examples' rule (LINK in src/voice.js, a backtracking pattern that is fine on 400
76
- * characters but not on 32 KB): any part of a word before one of its slashes that ends in a name character, a dot and two or more letters,
77
- * an optional port after it ("evil.example/x", "//evil.example/login", "x/evil.example/", "evil.example:8080/x"). "4.5/5", "and/or" and
78
- * "e.g. 3/4" pass. One pass over each word.
80
+ * A domain with a path (0.0.5 rule), linear-time on 32 KB. Expects `foldLinkText(s)`: any part of a word before one of its slashes
81
+ * that ends in a name character, a dot and two or more letters, with an optional port after it ("evil.example/x",
82
+ * "//evil.example/login", "x/evil.example/", "evil.example:8080/x"). Bare domains without a slash are intentionally not matched.
83
+ * "4.5/5", "and/or", "e.g. 3/4", "main.py:42", "Socket.IO", and "Finished.No" pass. One pass over each word.
79
84
  */
80
- function domainPath(s) {
85
+ export function domainPath(s) {
81
86
  for (const word of s.split(/\s+/u)) {
82
87
  const parts = word.split("/");
83
88
  for (let i = 0; i < parts.length - 1; i++) {
84
89
  const name = parts[i].replace(PORT, "");
85
90
  const dot = name.lastIndexOf(".");
86
- if (dot > 0 && NAME_CHAR.test(name[dot - 1]) && TLD.test(name.slice(dot + 1))) return true;
91
+ if (dot > 0 && NAME_CHAR.test([...name.slice(0, dot)].at(-1)) && TLD.test(name.slice(dot + 1))) return true;
87
92
  }
88
93
  }
89
94
  return false;
@@ -141,7 +146,7 @@ export function cleanVoiceDraft(value) {
141
146
  if (Buffer.byteLength(value, "utf8") > DRAFT_MAX_BYTES) return refuse("too_long", `the draft is over ${DRAFT_MAX_BYTES} bytes`);
142
147
  const text = nfc(value.replace(/\r\n?/gu, "\n")).replace(VS, "");
143
148
  if (HIDDEN.test(text)) return refuse("hidden_characters", "the draft holds a hidden or control character (tabs included)");
144
- const nfkc = nfc(text, "NFKC");
149
+ const nfkc = foldLinkText(text);
145
150
  if (MARKUP.some((re) => re.test(nfkc)) || domainPath(nfkc)) return refuse("link_or_markup", "the draft holds a link, an address, @, HTML or Markdown markup, or a code fence");
146
151
  const lines = text.split("\n");
147
152
  // padding: a run of blank lines could push a planted line far below the part of a box the person reads first
@@ -65,6 +65,7 @@ export function voiceJob(s, { last = null, diskSha = null, writtenSha = null, se
65
65
 
66
66
  const running = new Set(); // live workers, ended by stopVoice
67
67
  let epoch = 0; // moved on by Forget and the learning switch: a job that started before either stores nothing
68
+ let activeSampleToken = null; // concurrency cap of 1 active voice-sample-worker to avoid >450MB OOM spikes
68
69
 
69
70
  /** Runs src/voice-worker.js over three blob paths. Resolves {pairs, examples}, or null when it fails, crashes or times out (then it is ended). */
70
71
  function analyze(versions) {
@@ -106,6 +107,7 @@ export async function learnFromSend(job) {
106
107
 
107
108
  /** Ends every running learning worker (the server is stopping). */
108
109
  export function stopVoice() {
110
+ activeSampleToken = null;
109
111
  for (const w of [...running]) w.terminate();
110
112
  running.clear();
111
113
  }
@@ -221,8 +223,11 @@ function agentPairs(s) {
221
223
  return runs.slice(-AGENT_PAIRS_MAX).map((u) => ({ before: s.snapshots.blob(u.before), after: s.snapshots.blob(u.after) }));
222
224
  }
223
225
 
224
- /** Runs src/voice-sample-worker.js. Resolves {sources} or {error: code}; a crash or a timeout (60 s) is "unreadable" and the worker is ended. */
226
+ /** Runs src/voice-sample-worker.js. Resolves {sources} or {error: code}; at most 1 active worker at a time. */
225
227
  function sampleInWorker(job) {
228
+ if (activeSampleToken !== null) return Promise.resolve({ error: "busy", busy: true });
229
+ const token = {};
230
+ activeSampleToken = token;
226
231
  return new Promise((resolve) => {
227
232
  let worker;
228
233
  let settled = false;
@@ -230,6 +235,7 @@ function sampleInWorker(job) {
230
235
  const finish = (r) => {
231
236
  if (settled) return;
232
237
  settled = true;
238
+ if (activeSampleToken === token) activeSampleToken = null;
233
239
  clearTimeout(timer);
234
240
  running.delete(worker);
235
241
  worker?.terminate();
@@ -276,6 +282,9 @@ export async function buildVoiceRequest(sessions, key, body) {
276
282
  const saved = readGuide();
277
283
  if (body.improve && !saved.text) throw new VoiceError("There is no saved guide to improve.");
278
284
 
285
+ if ((body.open_file || files.length || body.texts.length) && activeSampleToken !== null) {
286
+ throw httpError(429, "Another voice guide is being built right now. Wait a moment, then try again.", "busy");
287
+ }
279
288
  let open = null;
280
289
  if (body.open_file) {
281
290
  let bytes;
@@ -298,6 +307,7 @@ export async function buildVoiceRequest(sessions, key, body) {
298
307
  let sampled = { sources: [] };
299
308
  if (sources) {
300
309
  sampled = await sampleInWorker({ open, files, texts: body.texts, shares: shares.source });
310
+ if (sampled.busy) throw httpError(429, "Another voice guide is being built right now. Wait a moment, then try again.", "busy");
301
311
  if (sampled.error) throw new VoiceError(UNREADABLE);
302
312
  }
303
313
  const evidence = {
@@ -96,12 +96,15 @@ export function readEvidence() {
96
96
  const raw = readObject(evidenceFile());
97
97
  if (raw === "missing") return emptyEvidence();
98
98
  if (!raw || typeof raw.learning !== "boolean") return { ...emptyEvidence(), learning: false };
99
+ const pairs = list(raw.pairs, cleanPair, MAX_PAIRS);
99
100
  const examples = list(raw.examples, cleanExample, Infinity);
100
101
  const unique = examples.filter((e, i) => examples.findLastIndex((x) => x.id === e.id) === i).slice(-MAX_EXAMPLES);
102
+ const totalPairs = Number.isSafeInteger(raw.totalPairs) && raw.totalPairs >= pairs.length ? raw.totalPairs : undefined;
101
103
  return {
102
104
  v: 1,
103
105
  learning: raw.learning,
104
- pairs: list(raw.pairs, cleanPair, MAX_PAIRS),
106
+ pairs,
107
+ ...(totalPairs !== undefined ? { totalPairs } : {}),
105
108
  hits: list(raw.hits, cleanHit, MAX_HITS),
106
109
  examples: unique,
107
110
  answers: cleanAnswers(raw.answers),
@@ -123,10 +126,13 @@ export function appendRecords({ pairs = [], hits = [], examples = [] } = {}, now
123
126
  const ev = readEvidence();
124
127
  if (!ev.learning) return ev;
125
128
  const add = (items, clean) => (Array.isArray(items) ? items.map((x) => clean(plain(x) ? { ...x, at: now } : x)).filter(Boolean) : []);
129
+ const newPairs = add(pairs, cleanPair);
126
130
  const newHits = add((Array.isArray(hits) ? hits : []).map((trait) => ({ trait })), cleanHit);
131
+ const prevTotal = Number.isSafeInteger(ev.totalPairs) && ev.totalPairs >= ev.pairs.length ? ev.totalPairs : ev.pairs.length;
127
132
  const next = {
128
133
  ...ev,
129
- pairs: [...ev.pairs, ...add(pairs, cleanPair)].slice(-MAX_PAIRS),
134
+ pairs: [...ev.pairs, ...newPairs].slice(-MAX_PAIRS),
135
+ totalPairs: prevTotal + newPairs.length,
130
136
  hits: [...ev.hits, ...newHits].slice(-MAX_HITS),
131
137
  examples: [...ev.examples, ...add(examples, cleanExample)],
132
138
  };
@@ -134,12 +140,13 @@ export function appendRecords({ pairs = [], hits = [], examples = [] } = {}, now
134
140
  return save(next);
135
141
  }
136
142
 
137
- /** "Not now" ({later: pairs kept now}) or "Don't suggest this" ({never}) for one trait. An unknown trait or action is a 400, nothing written. */
143
+ /** "Not now" ({later: total pairs seen so far}) or "Don't suggest this" ({never}) for one trait. An unknown trait or action is a 400, nothing written. */
138
144
  export function setAnswer(trait, action) {
139
145
  if (typeof trait !== "string" || !TRAIT_IDS.includes(trait)) throw new VoiceError(`trait must be one of: ${TRAIT_IDS.join(", ")}`);
140
146
  if (typeof action !== "string" || !ANSWERS.includes(action)) throw new VoiceError(`action must be one of: ${ANSWERS.join(", ")}`);
141
147
  const ev = readEvidence();
142
- return save({ ...ev, answers: { ...ev.answers, [trait]: action === "never" ? { never: true } : { later: ev.pairs.length } } });
148
+ const total = Number.isSafeInteger(ev.totalPairs) && ev.totalPairs >= ev.pairs.length ? ev.totalPairs : ev.pairs.length;
149
+ return save({ ...ev, answers: { ...ev.answers, [trait]: action === "never" ? { never: true } : { later: total } } });
143
150
  }
144
151
 
145
152
  /** Forget the kept edits, comment hits and examples. The learning switch stays; "never" answers stay (a choice, not evidence); "later" goes (it counted kept pairs). */
package/src/voice.js CHANGED
@@ -271,11 +271,12 @@ export function suggestions(evidence, style) {
271
271
  const hits = Array.isArray(ev.hits) ? ev.hits.filter((h) => h && typeof h === "object" && COMMENT_TRAITS.includes(h.trait)) : [];
272
272
  const answers = ev.answers && typeof ev.answers === "object" && !Array.isArray(ev.answers) ? ev.answers : {};
273
273
  const on = new Set(Array.isArray(style?.traits) ? style.traits : []);
274
+ const totalPairs = Number.isInteger(ev.totalPairs) ? ev.totalPairs : pairs.length;
274
275
  const held = (trait) => {
275
276
  const a = Object.hasOwn(answers, trait) ? answers[trait] : null;
276
277
  if (!a || typeof a !== "object") return false;
277
278
  if (a.never === true) return true;
278
- return Number.isInteger(a.later) && pairs.length - a.later < LATER_AFTER;
279
+ return Number.isInteger(a.later) && totalPairs - a.later < LATER_AFTER;
279
280
  };
280
281
  const found = [];
281
282
  const win = pairs.slice(-WINDOW);
package/src/xlsx.js CHANGED
@@ -216,6 +216,8 @@ function readSheet(zip, sheet, sharedStrings, budget = bookBudget()) {
216
216
  let field = null; // v | f | is
217
217
  let inIs = false;
218
218
  let phon = 0;
219
+ let rowIdx = -1;
220
+ let colIdx = 0;
219
221
  walkXml(xml, {
220
222
  onOpen(n, a) {
221
223
  const l = local(n);
@@ -223,12 +225,30 @@ function readSheet(zip, sheet, sharedStrings, budget = bookBudget()) {
223
225
  if (++budget.merges > MAX_MERGES) throw tooMany(budget, MAX_MERGES, "merged ranges");
224
226
  const q = parseA1(a.ref ?? "");
225
227
  if (q && (q.r1 > q.r0 || q.c1 > q.c0)) out.merges.push(q);
228
+ } else if (l === "row") {
229
+ rowIdx = a.r && /^\d+$/.test(a.r) ? Number(a.r) - 1 : rowIdx + 1;
230
+ colIdx = 0;
226
231
  } else if (l === "c" && ++budget.cells > MAX_CELLS) throw tooMany(budget, MAX_CELLS, "cells");
227
- else if (l === "c" && a.r) {
228
- const m = /^([A-Z]{1,3})(\d+)$/.exec(a.r);
229
- cur = m ? { c: [...m[1]].reduce((x, ch) => x * 26 + ch.charCodeAt(0) - 64, 0) - 1, r: Number(m[2]) - 1, t: a.t || "n", s: Number(a.s) || 0, v: "", f: null, is: "" } : null;
232
+ else if (l === "c") {
233
+ let r = -1;
234
+ let c = -1;
235
+ if (a.r) {
236
+ const m = /^([A-Z]{1,3})(\d+)$/.exec(a.r);
237
+ if (m) {
238
+ c = [...m[1]].reduce((x, ch) => x * 26 + ch.charCodeAt(0) - 64, 0) - 1;
239
+ r = Number(m[2]) - 1;
240
+ colIdx = c + 1;
241
+ }
242
+ } else {
243
+ r = rowIdx;
244
+ c = colIdx++;
245
+ }
246
+ cur = r >= 0 && r < 1_048_576 ? { c, r, t: a.t || "n", s: Number(a.s) || 0, v: "", hasV: false, f: null, is: "" } : null;
230
247
  } else if (!cur) return;
231
- else if (l === "v") field = "v";
248
+ else if (l === "v") {
249
+ field = "v";
250
+ cur.hasV = true;
251
+ }
232
252
  else if (l === "f") {
233
253
  field = "f";
234
254
  cur.f = "";
@@ -252,7 +272,7 @@ function readSheet(zip, sheet, sharedStrings, budget = bookBudget()) {
252
272
  if ((budget.sharedText -= text.length) < 0) throw tooMany(budget, MAX_SHARED_CELL_TEXT, "characters of text in its cells");
253
273
  }
254
274
  else if (cur.t === "inlineStr") text = cur.is;
255
- else if (cur.t === "b") text = cur.v === "1" ? "TRUE" : "FALSE";
275
+ else if (cur.t === "b") text = !cur.hasV ? "" : cur.v === "1" || cur.v === "true" ? "TRUE" : "FALSE";
256
276
  else if (cur.t === "n" && NUM.test(cur.v)) text = String(Number(cur.v));
257
277
  else text = cur.v;
258
278
  if (text !== "" && cur.c < 16384) {
package/src/zip-write.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import zlib from "node:zlib";
2
2
  import { gated } from "./stop-gate.js";
3
- import { openZip, ZipError } from "./zip.js";
3
+ import { crc32, openZip, ZipError } from "./zip.js";
4
4
 
5
5
  // Rewrites one entry of a ZIP held in memory and copies every other entry as it is: same order, same compressed bytes, same CRC, same
6
6
  // central-directory fields (times, attributes, extra fields, comments). The local headers are written fresh from the central-directory
@@ -19,7 +19,7 @@ export function replaceEntry(bytes, name, data) {
19
19
  if (!zip.entries.has(name)) throw new ZipError(`no entry ${JSON.stringify(name)} to replace`);
20
20
  const newData = Buffer.from(data);
21
21
  const newComp = gated(() => zlib.deflateRawSync(newData)); // never stopped inside zlib (src/stop-gate.js)
22
- const newCrc = zlib.crc32(newData) >>> 0;
22
+ const newCrc = crc32(newData);
23
23
  const parts = [];
24
24
  const central = [];
25
25
  let offset = 0;
@@ -38,9 +38,11 @@ export function replaceEntry(bytes, name, data) {
38
38
  const crc = mine ? newCrc : e.crc;
39
39
  const size = mine ? newData.length : e.size;
40
40
  const flags = e.flags & ~0x8; // sizes are in the local header now: no data descriptor
41
+ if (offset > 0xfffffffe || comp.length > 0xfffffffe || size > 0xfffffffe) throw new ZipError("the archive would need ZIP64");
42
+ const verNeeded = method === 8 ? Math.max(20, record.readUInt16LE(6)) : record.readUInt16LE(6);
41
43
  const lh = Buffer.alloc(30);
42
44
  lh.writeUInt32LE(0x04034b50, 0);
43
- lh.writeUInt16LE(record.readUInt16LE(6), 4); // version needed
45
+ lh.writeUInt16LE(verNeeded, 4); // version needed
44
46
  lh.writeUInt16LE(flags, 6);
45
47
  lh.writeUInt16LE(method, 8);
46
48
  lh.writeUInt16LE(record.readUInt16LE(12), 10); // time
@@ -51,6 +53,7 @@ export function replaceEntry(bytes, name, data) {
51
53
  lh.writeUInt16LE(nameLen, 26);
52
54
  lh.writeUInt16LE(0, 28); // no local extra field
53
55
  parts.push(lh, nameBuf, comp);
56
+ record.writeUInt16LE(verNeeded, 6);
54
57
  record.writeUInt16LE(flags, 8);
55
58
  record.writeUInt16LE(method, 10);
56
59
  record.writeUInt32LE(crc, 16);
@@ -59,8 +62,10 @@ export function replaceEntry(bytes, name, data) {
59
62
  record.writeUInt32LE(offset, 42);
60
63
  central.push(record);
61
64
  offset += 30 + nameLen + comp.length;
65
+ if (offset > 0xfffffffe) throw new ZipError("the archive would need ZIP64");
62
66
  }
63
67
  const cd = Buffer.concat(central);
68
+ if (offset + cd.length > 0xfffffffe) throw new ZipError("the archive would need ZIP64");
64
69
  const commentLen = buf.readUInt16LE(zip.eocd + 20);
65
70
  const end = Buffer.alloc(22);
66
71
  sig(0x06054b50).copy(end, 0);
@@ -70,6 +75,5 @@ export function replaceEntry(bytes, name, data) {
70
75
  end.writeUInt32LE(offset, 16);
71
76
  end.writeUInt16LE(commentLen, 20);
72
77
  const comment = buf.subarray(zip.eocd + 22, zip.eocd + 22 + commentLen);
73
- if (offset + cd.length > 0xfffffffe) throw new ZipError("the archive would need ZIP64");
74
78
  return Buffer.concat([...parts, cd, end, comment]);
75
79
  }