onionskin-cli 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +98 -0
- package/README.md +4 -4
- package/package.json +1 -1
- package/scripts/checker/check.py +24 -5
- package/scripts/checker/requirements.txt +482 -5
- package/scripts/requirements.txt +482 -5
- package/scripts/rewriter/apple/main.swift +1 -1
- package/scripts/rewriter/apple/onionskin-fm +0 -0
- package/scripts/rewriter/apple/onionskin-fm.json +1 -1
- package/skills/onionskin/SKILL.md +4 -4
- package/skills/onionskin/reference/ops.md +1 -1
- package/src/agent-runner.js +94 -15
- package/src/checker.js +238 -33
- package/src/cli.js +173 -33
- package/src/doctor.js +12 -3
- package/src/docx.js +130 -63
- package/src/fsutil.js +13 -5
- package/src/history.js +11 -2
- package/src/html-render.js +74 -15
- package/src/html-utf8.js +1 -1
- package/src/html.js +3 -1
- package/src/merge-package.js +82 -26
- package/src/offload.js +25 -6
- package/src/ops/docx-table.js +5 -3
- package/src/ops/pptx.js +8 -2
- package/src/ops/xlsx.js +114 -8
- package/src/package.js +17 -2
- package/src/pptx.js +244 -8
- package/src/question.js +10 -3
- package/src/quick.js +1 -1
- package/src/redline-check.js +12 -2
- package/src/rewriter.js +38 -8
- package/src/server-entry.js +4 -3
- package/src/server.js +97 -28
- package/src/session.js +13 -2
- package/src/setup.js +20 -2
- package/src/update.js +248 -19
- package/src/voice-guide.js +12 -7
- package/src/voice-learn.js +11 -1
- package/src/voice-store.js +11 -4
- package/src/voice.js +2 -1
- package/src/xlsx.js +25 -5
- package/src/zip-write.js +8 -4
- package/src/zip.js +64 -13
- package/web/dist/assets/{index-DytKkMdi.js → index-CjsTf5gj.js} +108 -106
- package/web/dist/assets/index-DnbLtHhG.css +1 -0
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-DkNx0LSH.css +0 -1
package/src/update.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// SHA-256 check). No prompts, no sudo; snapshots and the state folder are never deleted here.
|
|
5
5
|
//
|
|
6
6
|
// Everything this command needs after get.sh starts is imported at the top: install.sh deletes the app folder this process runs from.
|
|
7
|
-
import { spawnSync } from "node:child_process";
|
|
7
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
8
8
|
import fs from "node:fs";
|
|
9
9
|
import os from "node:os";
|
|
10
10
|
import path from "node:path";
|
|
@@ -31,8 +31,14 @@ export function baseUrlOk(u) {
|
|
|
31
31
|
return m[1] === "https" || /^127\.0\.0\.1(?::\d+)?$/.test(m[2]);
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
/** curl's --proto for a URL baseUrlOk accepted:
|
|
35
|
-
export const curlProto = (u) =>
|
|
34
|
+
/** curl's --proto for a URL baseUrlOk accepted: only the scheme of the URL itself (redirects stay https only, CURL). */
|
|
35
|
+
export const curlProto = (u) => {
|
|
36
|
+
const s = String(u ?? "");
|
|
37
|
+
if (s.startsWith("https://")) return ["--proto", "=https"];
|
|
38
|
+
if (s.startsWith("file://")) return ["--proto", "=file"];
|
|
39
|
+
if (/^http:\/\/127\.0\.0\.1(?::\d+)?(?:\/|$)/.test(s)) return ["--proto", "=http"];
|
|
40
|
+
return ["--proto", "=https"];
|
|
41
|
+
};
|
|
36
42
|
|
|
37
43
|
const VERSION_RE = /^(\d{1,6})\.(\d{1,6})\.(\d{1,6})(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/;
|
|
38
44
|
const MAX_VERSION = 40; // characters of a whole version
|
|
@@ -151,7 +157,7 @@ export function parseUpdateArgs(args) {
|
|
|
151
157
|
/**
|
|
152
158
|
* @param {string[]} args the arguments after `update`
|
|
153
159
|
* @param {object} d version (running), appRoot, home, env, print(line), and the parts that act:
|
|
154
|
-
* fetchLatest(base), server: {healthy(), openWork() -> {status, body}}, stop(), runGetSh(base) -> exit status,
|
|
160
|
+
* fetchLatest(base), server: {healthy({reportMismatch: true}) -> boolean | "incompatible", openWork() -> {status, body}}, stop(), runGetSh(base) -> exit status,
|
|
155
161
|
* launcherVersion(route, root) -> string|null (the installed command's --version), run(cmd, args) -> exit status (npm --yes)
|
|
156
162
|
* @returns {Promise<number>} the exit code
|
|
157
163
|
*/
|
|
@@ -226,26 +232,48 @@ export async function updateCommand(args, d) {
|
|
|
226
232
|
|
|
227
233
|
// An update restarts the server: a tab open now cannot reconnect (the new server listens on a new port). Refuse while work is open.
|
|
228
234
|
let stopped = false;
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
235
|
+
const health = await d.server.healthy({ reportMismatch: true });
|
|
236
|
+
if (health) {
|
|
237
|
+
if (health === "incompatible" && !a.force) {
|
|
238
|
+
say(`A running Onionskin server from another version may hold unsent work: run ${BIN} stop, or ${BIN} update --force.`);
|
|
239
|
+
return 1;
|
|
240
|
+
}
|
|
241
|
+
const w = health === true ? await d.server.openWork() : null;
|
|
242
|
+
const sessions = Number.isInteger(w?.body?.sessions) ? w.body.sessions : null;
|
|
232
243
|
// a server that does not count its open tabs (open_sessions) cannot vouch for edits typed in a tab: like one without the route
|
|
233
|
-
const risk = w
|
|
244
|
+
const risk = w?.status === 200 && Number.isInteger(w.body?.open_sessions) ? workAtRisk(w.body) : null;
|
|
234
245
|
const reason = risk ? (risk.length ? `open work that the update would cut off: ${risk.join(", ")}` : null) : sessions === 0 ? null : "a server from an older version that cannot say whether its open sessions hold unsent work";
|
|
235
246
|
if (reason && !a.force) {
|
|
236
247
|
say(`Not updated: ${PRODUCT.name} is running with ${reason}. The update restarts ${PRODUCT.name}, and a browser tab that is open now cannot reconnect to it, so anything not sent from it would be lost. Send or finish that work, then press Close session in the file's page (or close its tab and wait 20 minutes), and run ${BIN} update again. To update anyway, run ${BIN} update --force. Nothing was changed.`);
|
|
237
248
|
return 1;
|
|
238
249
|
}
|
|
239
|
-
|
|
250
|
+
try {
|
|
251
|
+
await d.stop();
|
|
252
|
+
} catch (err) {
|
|
253
|
+
say(`Not updated: could not stop the running ${PRODUCT.name} server (${err?.message || err}). Nothing was changed.`);
|
|
254
|
+
say(`AGENT: the update did not run because the running ${PRODUCT.name} server could not be stopped. Tell the person nothing was changed and suggest running \`${BIN} stop\` or \`${BIN} doctor\`.`);
|
|
255
|
+
return 1;
|
|
256
|
+
}
|
|
240
257
|
stopped = true;
|
|
241
258
|
say(`Stopped the ${PRODUCT.name} server for the update${reason ? ` (--force: ${reason})` : ""}.`);
|
|
242
259
|
}
|
|
243
260
|
|
|
244
261
|
if (route === "npm") {
|
|
262
|
+
const limit = 10 * 60_000;
|
|
263
|
+
const deadline = Date.now() + limit;
|
|
264
|
+
const run = d.run || runInGroup;
|
|
245
265
|
for (const c of NPM_COMMANDS) {
|
|
246
266
|
say(`$ ${c}`);
|
|
247
267
|
const [cmd, ...rest] = c.split(" ");
|
|
248
|
-
const
|
|
268
|
+
const remaining = deadline - Date.now();
|
|
269
|
+
let timedOut = remaining <= 0;
|
|
270
|
+
const opts = { env, timeout: remaining, totalTimeout: limit, onTimeout: () => { timedOut = true; } };
|
|
271
|
+
const args = cmd === BIN ? [path.join(root, "src", "cli.js"), ...rest] : rest;
|
|
272
|
+
const status = timedOut ? 1 : await run(cmd === BIN ? process.execPath : cmd, args, opts);
|
|
273
|
+
if (timedOut) {
|
|
274
|
+
say("The npm update timed out after 10 minutes.");
|
|
275
|
+
return 1;
|
|
276
|
+
}
|
|
249
277
|
if (status !== 0) {
|
|
250
278
|
say(`Not updated: \`${c}\` failed (exit ${status}). Run the two commands above yourself to see why.`);
|
|
251
279
|
return 1;
|
|
@@ -263,10 +291,47 @@ export async function updateCommand(args, d) {
|
|
|
263
291
|
|
|
264
292
|
// the installer copy: get.sh from the same release folder, piped into sh as the README does
|
|
265
293
|
say(`Running the installer from ${base}/get.sh`);
|
|
266
|
-
|
|
294
|
+
let skillSetupFailed = false;
|
|
295
|
+
let outBuf = "";
|
|
296
|
+
let timedOutMs = 0;
|
|
297
|
+
const scanLine = (line) => {
|
|
298
|
+
if (line.includes("SKILL_SETUP_FAILED")) skillSetupFailed = true;
|
|
299
|
+
};
|
|
300
|
+
let noChecker = true;
|
|
301
|
+
try {
|
|
302
|
+
noChecker = !fs.lstatSync(path.join(home, PRODUCT.stateDirName, "venv", "pyvenv.cfg")).isFile();
|
|
303
|
+
} catch {
|
|
304
|
+
// An inaccessible or missing configuration means there is no environment to update.
|
|
305
|
+
}
|
|
306
|
+
const rawRes = await d.runGetSh(base, {
|
|
307
|
+
noChecker,
|
|
308
|
+
onOutput: (chunk) => {
|
|
309
|
+
outBuf += String(chunk);
|
|
310
|
+
let nl;
|
|
311
|
+
while ((nl = outBuf.indexOf("\n")) !== -1) {
|
|
312
|
+
scanLine(outBuf.slice(0, nl));
|
|
313
|
+
outBuf = outBuf.slice(nl + 1);
|
|
314
|
+
}
|
|
315
|
+
},
|
|
316
|
+
onTimeout: (ms) => {
|
|
317
|
+
timedOutMs = ms;
|
|
318
|
+
},
|
|
319
|
+
});
|
|
320
|
+
if (outBuf) scanLine(outBuf);
|
|
321
|
+
const status = typeof rawRes === "object" && rawRes !== null ? rawRes.status : rawRes;
|
|
322
|
+
if (typeof rawRes === "object" && rawRes !== null) {
|
|
323
|
+
if (rawRes.skillSetupFailed) skillSetupFailed = true;
|
|
324
|
+
if (rawRes.timedOut) timedOutMs = rawRes.timeoutMs || 360_000;
|
|
325
|
+
}
|
|
267
326
|
const now = d.launcherVersion(route, root);
|
|
268
327
|
if (status !== 0) {
|
|
269
|
-
|
|
328
|
+
if (timedOutMs) {
|
|
329
|
+
const secs = Math.max(1, Math.round(timedOutMs / 1000));
|
|
330
|
+
const dur = secs >= 60 && secs % 60 === 0 ? `${secs / 60} ${secs === 60 ? "minute" : "minutes"}` : `${secs}s`;
|
|
331
|
+
say(`Not updated: the installer timed out after ${dur}. The command now runs ${PRODUCT.name} ${now || "(version unknown)"}.`);
|
|
332
|
+
} else {
|
|
333
|
+
say(`Not updated: the installer stopped (exit ${status}; its lines above say why). The command now runs ${PRODUCT.name} ${now || "(version unknown)"}.`);
|
|
334
|
+
}
|
|
270
335
|
return 1;
|
|
271
336
|
}
|
|
272
337
|
if (now !== latest.version) {
|
|
@@ -274,15 +339,179 @@ export async function updateCommand(args, d) {
|
|
|
274
339
|
return 1;
|
|
275
340
|
}
|
|
276
341
|
say("");
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
342
|
+
if (skillSetupFailed) {
|
|
343
|
+
say(`UPDATED (with warning): ${PRODUCT.name} ${d.version} -> ${now}, but skill setup failed. Run \`${BIN} setup\` to finish installing the agent skill. Snapshots and settings in ~/.onionskin were kept.`);
|
|
344
|
+
if (stopped) say(`The server was stopped; it starts again on the next ${BIN} command. Open your file again to keep working.`);
|
|
345
|
+
say(`AGENT: this was an update to ${now}, but skill setup failed (SKILL_SETUP_FAILED). Tell the person ${PRODUCT.name} is updated to ${now}, and that skill setup did not finish so they should run \`${BIN} setup\`.`);
|
|
346
|
+
} else {
|
|
347
|
+
say(`UPDATED: ${PRODUCT.name} ${d.version} -> ${now}. Snapshots and settings in ~/.onionskin were kept.`);
|
|
348
|
+
if (stopped) say(`The server was stopped; it starts again on the next ${BIN} command. Open your file again to keep working.`);
|
|
349
|
+
say(`AGENT: this was an update. Tell the person this one sentence, and not the installer's sentence above: ${PRODUCT.name} is updated to ${now}.`);
|
|
350
|
+
}
|
|
280
351
|
return 0;
|
|
281
352
|
}
|
|
282
353
|
|
|
283
|
-
|
|
284
|
-
|
|
354
|
+
const SIG_CODES = { SIGHUP: 129, SIGINT: 130, SIGTERM: 143 };
|
|
355
|
+
|
|
356
|
+
/** Runs a command in its own process group; forwards signals and kills the entire process group on timeout or parent exit. */
|
|
357
|
+
export function runInGroup(cmd, args, { env = process.env, timeout, totalTimeout = timeout, graceMs = 500, signalGraceMs = 10_000, onOutput, onTimeout }) {
|
|
358
|
+
return new Promise((resolve) => {
|
|
359
|
+
let settled = false;
|
|
360
|
+
let timedOut = false;
|
|
361
|
+
let signaled = null;
|
|
362
|
+
let killTimer = null;
|
|
363
|
+
let exitTimer = null;
|
|
364
|
+
let stdoutBuf = "";
|
|
365
|
+
let stderrBuf = "";
|
|
366
|
+
|
|
367
|
+
let child;
|
|
368
|
+
try {
|
|
369
|
+
child = spawn(cmd, args, {
|
|
370
|
+
detached: true,
|
|
371
|
+
stdio: onOutput ? ["ignore", "pipe", "pipe"] : ["ignore", "inherit", "inherit"],
|
|
372
|
+
env,
|
|
373
|
+
});
|
|
374
|
+
} catch {
|
|
375
|
+
return resolve(1);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
const killGroup = (sig) => {
|
|
379
|
+
if (!child.pid) return;
|
|
380
|
+
try {
|
|
381
|
+
process.kill(-child.pid, sig);
|
|
382
|
+
} catch {}
|
|
383
|
+
};
|
|
384
|
+
|
|
385
|
+
const emitLine = (stream, line, nl) => {
|
|
386
|
+
if (onOutput) onOutput(line + nl);
|
|
387
|
+
const out = line.includes("SKILL_SETUP_FAILED")
|
|
388
|
+
? ` Warning: skill setup did not finish; run '${BIN} setup' to finish installing the skill.`
|
|
389
|
+
: line;
|
|
390
|
+
stream.write(out + nl);
|
|
391
|
+
};
|
|
392
|
+
|
|
393
|
+
const feed = (stream, which, chunk) => {
|
|
394
|
+
let buf = (which === "out" ? stdoutBuf : stderrBuf) + chunk.toString("utf8");
|
|
395
|
+
let idx;
|
|
396
|
+
while ((idx = buf.indexOf("\n")) !== -1) {
|
|
397
|
+
emitLine(stream, buf.slice(0, idx), "\n");
|
|
398
|
+
buf = buf.slice(idx + 1);
|
|
399
|
+
}
|
|
400
|
+
if (which === "out") stdoutBuf = buf;
|
|
401
|
+
else stderrBuf = buf;
|
|
402
|
+
};
|
|
403
|
+
|
|
404
|
+
const flushStreams = () => {
|
|
405
|
+
if (stdoutBuf) {
|
|
406
|
+
const s = stdoutBuf;
|
|
407
|
+
stdoutBuf = "";
|
|
408
|
+
emitLine(process.stdout, s, "");
|
|
409
|
+
}
|
|
410
|
+
if (stderrBuf) {
|
|
411
|
+
const s = stderrBuf;
|
|
412
|
+
stderrBuf = "";
|
|
413
|
+
emitLine(process.stderr, s, "");
|
|
414
|
+
}
|
|
415
|
+
};
|
|
416
|
+
|
|
417
|
+
const onParentExit = () => {
|
|
418
|
+
killGroup("SIGTERM");
|
|
419
|
+
killGroup("SIGKILL");
|
|
420
|
+
};
|
|
421
|
+
|
|
422
|
+
const onSignal = (sig) => {
|
|
423
|
+
if (settled || signaled) return;
|
|
424
|
+
signaled = sig;
|
|
425
|
+
killGroup(sig);
|
|
426
|
+
clearTimeout(killTimer);
|
|
427
|
+
const hardStop = () => {
|
|
428
|
+
killGroup("SIGKILL");
|
|
429
|
+
try { child.stdout?.destroy(); } catch {}
|
|
430
|
+
try { child.stderr?.destroy(); } catch {}
|
|
431
|
+
finish(SIG_CODES[sig] || 1);
|
|
432
|
+
};
|
|
433
|
+
if (signalGraceMs === 0) return hardStop();
|
|
434
|
+
killTimer = setTimeout(hardStop, signalGraceMs);
|
|
435
|
+
killTimer.unref?.();
|
|
436
|
+
};
|
|
437
|
+
|
|
438
|
+
const sigHandlers = ["SIGINT", "SIGTERM", "SIGHUP"].map((sig) => {
|
|
439
|
+
const fn = () => onSignal(sig);
|
|
440
|
+
process.on(sig, fn);
|
|
441
|
+
return [sig, fn];
|
|
442
|
+
});
|
|
443
|
+
process.on("exit", onParentExit);
|
|
444
|
+
|
|
445
|
+
const cleanupListeners = () => {
|
|
446
|
+
for (const [sig, fn] of sigHandlers) process.off(sig, fn);
|
|
447
|
+
process.off("exit", onParentExit);
|
|
448
|
+
};
|
|
449
|
+
|
|
450
|
+
const finish = (code) => {
|
|
451
|
+
if (settled) return;
|
|
452
|
+
settled = true;
|
|
453
|
+
clearTimeout(timer);
|
|
454
|
+
clearTimeout(killTimer);
|
|
455
|
+
clearTimeout(exitTimer);
|
|
456
|
+
cleanupListeners();
|
|
457
|
+
// If the leader exited on timeout/signal while a descendant ignored SIGTERM/SIGINT, reap the group.
|
|
458
|
+
if (timedOut || signaled) killGroup("SIGKILL");
|
|
459
|
+
flushStreams();
|
|
460
|
+
if (timedOut) return resolve(1);
|
|
461
|
+
if (signaled) return resolve(SIG_CODES[signaled] || 1);
|
|
462
|
+
resolve(code ?? 1);
|
|
463
|
+
};
|
|
464
|
+
|
|
465
|
+
child.stdout?.on("data", (buf) => feed(process.stdout, "out", buf));
|
|
466
|
+
child.stderr?.on("data", (buf) => feed(process.stderr, "err", buf));
|
|
467
|
+
child.once("error", () => finish(1));
|
|
468
|
+
child.once("exit", (code) => {
|
|
469
|
+
if (settled) return;
|
|
470
|
+
exitTimer = setTimeout(() => {
|
|
471
|
+
try { child.stdout?.destroy(); } catch {}
|
|
472
|
+
try { child.stderr?.destroy(); } catch {}
|
|
473
|
+
finish(code);
|
|
474
|
+
}, graceMs);
|
|
475
|
+
exitTimer.unref?.();
|
|
476
|
+
});
|
|
477
|
+
child.once("close", (code) => finish(code));
|
|
478
|
+
|
|
479
|
+
const timer = setTimeout(() => {
|
|
480
|
+
timedOut = true;
|
|
481
|
+
if (onTimeout) onTimeout(totalTimeout);
|
|
482
|
+
killGroup("SIGTERM");
|
|
483
|
+
killTimer = setTimeout(() => {
|
|
484
|
+
killGroup("SIGKILL");
|
|
485
|
+
try { child.stdout?.destroy(); } catch {}
|
|
486
|
+
try { child.stderr?.destroy(); } catch {}
|
|
487
|
+
finish(1);
|
|
488
|
+
}, graceMs);
|
|
489
|
+
killTimer.unref?.();
|
|
490
|
+
}, timeout);
|
|
491
|
+
timer.unref?.();
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/** The README one-liner, run for `base` (already checked by baseUrlOk): downloads get.sh to a 0600 temp file, then executes it with /bin/sh. */
|
|
496
|
+
export async function runGetShDefault(base, env = process.env, { timeout = 360_000, graceMs = 500, onOutput = () => {}, onTimeout, noChecker = false } = {}) {
|
|
285
497
|
const url = String(base).replace(/\/+$/, "");
|
|
286
|
-
const
|
|
287
|
-
|
|
498
|
+
const tmpDir = fs.mkdtempSync(path.join(env.TMPDIR || os.tmpdir(), "onionskin-get-"));
|
|
499
|
+
const tmpFile = path.join(tmpDir, "get.sh");
|
|
500
|
+
const deadline = Date.now() + timeout;
|
|
501
|
+
try {
|
|
502
|
+
const fd = fs.openSync(tmpFile, "wx", 0o600);
|
|
503
|
+
fs.closeSync(fd);
|
|
504
|
+
const dl = spawnSync(
|
|
505
|
+
"curl",
|
|
506
|
+
["-fsSL", "--connect-timeout", "20", "--max-time", "60", ...curlProto(url), "--proto-redir", "=https", "-o", tmpFile, `${url}/get.sh`],
|
|
507
|
+
{ stdio: ["ignore", "inherit", "inherit"], env, timeout: Math.min(70_000, timeout) }
|
|
508
|
+
);
|
|
509
|
+
if (dl.error || dl.status !== 0) return dl.status ?? 1;
|
|
510
|
+
fs.chmodSync(tmpFile, 0o600);
|
|
511
|
+
const remaining = Math.max(1, deadline - Date.now());
|
|
512
|
+
const args = noChecker ? [tmpFile, "--no-checker"] : [tmpFile];
|
|
513
|
+
return await runInGroup("/bin/sh", args, { env: { ...env, ONIONSKIN_BASE_URL: url }, timeout: remaining, totalTimeout: timeout, graceMs, onOutput, onTimeout });
|
|
514
|
+
} finally {
|
|
515
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
516
|
+
}
|
|
288
517
|
}
|
package/src/voice-guide.js
CHANGED
|
@@ -71,19 +71,24 @@ const BLANK_RUN_MAX = 2;
|
|
|
71
71
|
const TLD = /^\p{L}{2,}$/u;
|
|
72
72
|
const NAME_CHAR = /[\p{L}\p{N}-]/u;
|
|
73
73
|
const PORT = /:\d{1,5}$/u;
|
|
74
|
+
const LOOKALIKE_DOTS = /[\u3002\uFF61\uFF0E\u2024]/gu;
|
|
75
|
+
const INVISIBLE = /[\u00AD\u034F\u061C\u115F\u1160\u17B4\u17B5\u180E\u200B-\u200F\u202A-\u202E\u2060-\u2064\u2066-\u2069\u2800\u3164\uFEFF\uFFA0]/gu;
|
|
76
|
+
/** NFKC, invisible characters stripped, and look-alike dots folded to ASCII "." for link checks. */
|
|
77
|
+
export const foldLinkText = (s) => nfc(String(s ?? "").replace(INVISIBLE, ""), "NFKC").replace(INVISIBLE, "").replace(LOOKALIKE_DOTS, ".");
|
|
78
|
+
|
|
74
79
|
/**
|
|
75
|
-
* A domain with a path
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
* "e.g. 3/4" pass. One pass over each word.
|
|
80
|
+
* A domain with a path (0.0.5 rule), linear-time on 32 KB. Expects `foldLinkText(s)`: any part of a word before one of its slashes
|
|
81
|
+
* that ends in a name character, a dot and two or more letters, with an optional port after it ("evil.example/x",
|
|
82
|
+
* "//evil.example/login", "x/evil.example/", "evil.example:8080/x"). Bare domains without a slash are intentionally not matched.
|
|
83
|
+
* "4.5/5", "and/or", "e.g. 3/4", "main.py:42", "Socket.IO", and "Finished.No" pass. One pass over each word.
|
|
79
84
|
*/
|
|
80
|
-
function domainPath(s) {
|
|
85
|
+
export function domainPath(s) {
|
|
81
86
|
for (const word of s.split(/\s+/u)) {
|
|
82
87
|
const parts = word.split("/");
|
|
83
88
|
for (let i = 0; i < parts.length - 1; i++) {
|
|
84
89
|
const name = parts[i].replace(PORT, "");
|
|
85
90
|
const dot = name.lastIndexOf(".");
|
|
86
|
-
if (dot > 0 && NAME_CHAR.test(name
|
|
91
|
+
if (dot > 0 && NAME_CHAR.test([...name.slice(0, dot)].at(-1)) && TLD.test(name.slice(dot + 1))) return true;
|
|
87
92
|
}
|
|
88
93
|
}
|
|
89
94
|
return false;
|
|
@@ -141,7 +146,7 @@ export function cleanVoiceDraft(value) {
|
|
|
141
146
|
if (Buffer.byteLength(value, "utf8") > DRAFT_MAX_BYTES) return refuse("too_long", `the draft is over ${DRAFT_MAX_BYTES} bytes`);
|
|
142
147
|
const text = nfc(value.replace(/\r\n?/gu, "\n")).replace(VS, "");
|
|
143
148
|
if (HIDDEN.test(text)) return refuse("hidden_characters", "the draft holds a hidden or control character (tabs included)");
|
|
144
|
-
const nfkc =
|
|
149
|
+
const nfkc = foldLinkText(text);
|
|
145
150
|
if (MARKUP.some((re) => re.test(nfkc)) || domainPath(nfkc)) return refuse("link_or_markup", "the draft holds a link, an address, @, HTML or Markdown markup, or a code fence");
|
|
146
151
|
const lines = text.split("\n");
|
|
147
152
|
// padding: a run of blank lines could push a planted line far below the part of a box the person reads first
|
package/src/voice-learn.js
CHANGED
|
@@ -65,6 +65,7 @@ export function voiceJob(s, { last = null, diskSha = null, writtenSha = null, se
|
|
|
65
65
|
|
|
66
66
|
const running = new Set(); // live workers, ended by stopVoice
|
|
67
67
|
let epoch = 0; // moved on by Forget and the learning switch: a job that started before either stores nothing
|
|
68
|
+
let activeSampleToken = null; // concurrency cap of 1 active voice-sample-worker to avoid >450MB OOM spikes
|
|
68
69
|
|
|
69
70
|
/** Runs src/voice-worker.js over three blob paths. Resolves {pairs, examples}, or null when it fails, crashes or times out (then it is ended). */
|
|
70
71
|
function analyze(versions) {
|
|
@@ -106,6 +107,7 @@ export async function learnFromSend(job) {
|
|
|
106
107
|
|
|
107
108
|
/** Ends every running learning worker (the server is stopping). */
|
|
108
109
|
export function stopVoice() {
|
|
110
|
+
activeSampleToken = null;
|
|
109
111
|
for (const w of [...running]) w.terminate();
|
|
110
112
|
running.clear();
|
|
111
113
|
}
|
|
@@ -221,8 +223,11 @@ function agentPairs(s) {
|
|
|
221
223
|
return runs.slice(-AGENT_PAIRS_MAX).map((u) => ({ before: s.snapshots.blob(u.before), after: s.snapshots.blob(u.after) }));
|
|
222
224
|
}
|
|
223
225
|
|
|
224
|
-
/** Runs src/voice-sample-worker.js. Resolves {sources} or {error: code};
|
|
226
|
+
/** Runs src/voice-sample-worker.js. Resolves {sources} or {error: code}; at most 1 active worker at a time. */
|
|
225
227
|
function sampleInWorker(job) {
|
|
228
|
+
if (activeSampleToken !== null) return Promise.resolve({ error: "busy", busy: true });
|
|
229
|
+
const token = {};
|
|
230
|
+
activeSampleToken = token;
|
|
226
231
|
return new Promise((resolve) => {
|
|
227
232
|
let worker;
|
|
228
233
|
let settled = false;
|
|
@@ -230,6 +235,7 @@ function sampleInWorker(job) {
|
|
|
230
235
|
const finish = (r) => {
|
|
231
236
|
if (settled) return;
|
|
232
237
|
settled = true;
|
|
238
|
+
if (activeSampleToken === token) activeSampleToken = null;
|
|
233
239
|
clearTimeout(timer);
|
|
234
240
|
running.delete(worker);
|
|
235
241
|
worker?.terminate();
|
|
@@ -276,6 +282,9 @@ export async function buildVoiceRequest(sessions, key, body) {
|
|
|
276
282
|
const saved = readGuide();
|
|
277
283
|
if (body.improve && !saved.text) throw new VoiceError("There is no saved guide to improve.");
|
|
278
284
|
|
|
285
|
+
if ((body.open_file || files.length || body.texts.length) && activeSampleToken !== null) {
|
|
286
|
+
throw httpError(429, "Another voice guide is being built right now. Wait a moment, then try again.", "busy");
|
|
287
|
+
}
|
|
279
288
|
let open = null;
|
|
280
289
|
if (body.open_file) {
|
|
281
290
|
let bytes;
|
|
@@ -298,6 +307,7 @@ export async function buildVoiceRequest(sessions, key, body) {
|
|
|
298
307
|
let sampled = { sources: [] };
|
|
299
308
|
if (sources) {
|
|
300
309
|
sampled = await sampleInWorker({ open, files, texts: body.texts, shares: shares.source });
|
|
310
|
+
if (sampled.busy) throw httpError(429, "Another voice guide is being built right now. Wait a moment, then try again.", "busy");
|
|
301
311
|
if (sampled.error) throw new VoiceError(UNREADABLE);
|
|
302
312
|
}
|
|
303
313
|
const evidence = {
|
package/src/voice-store.js
CHANGED
|
@@ -96,12 +96,15 @@ export function readEvidence() {
|
|
|
96
96
|
const raw = readObject(evidenceFile());
|
|
97
97
|
if (raw === "missing") return emptyEvidence();
|
|
98
98
|
if (!raw || typeof raw.learning !== "boolean") return { ...emptyEvidence(), learning: false };
|
|
99
|
+
const pairs = list(raw.pairs, cleanPair, MAX_PAIRS);
|
|
99
100
|
const examples = list(raw.examples, cleanExample, Infinity);
|
|
100
101
|
const unique = examples.filter((e, i) => examples.findLastIndex((x) => x.id === e.id) === i).slice(-MAX_EXAMPLES);
|
|
102
|
+
const totalPairs = Number.isSafeInteger(raw.totalPairs) && raw.totalPairs >= pairs.length ? raw.totalPairs : undefined;
|
|
101
103
|
return {
|
|
102
104
|
v: 1,
|
|
103
105
|
learning: raw.learning,
|
|
104
|
-
pairs
|
|
106
|
+
pairs,
|
|
107
|
+
...(totalPairs !== undefined ? { totalPairs } : {}),
|
|
105
108
|
hits: list(raw.hits, cleanHit, MAX_HITS),
|
|
106
109
|
examples: unique,
|
|
107
110
|
answers: cleanAnswers(raw.answers),
|
|
@@ -123,10 +126,13 @@ export function appendRecords({ pairs = [], hits = [], examples = [] } = {}, now
|
|
|
123
126
|
const ev = readEvidence();
|
|
124
127
|
if (!ev.learning) return ev;
|
|
125
128
|
const add = (items, clean) => (Array.isArray(items) ? items.map((x) => clean(plain(x) ? { ...x, at: now } : x)).filter(Boolean) : []);
|
|
129
|
+
const newPairs = add(pairs, cleanPair);
|
|
126
130
|
const newHits = add((Array.isArray(hits) ? hits : []).map((trait) => ({ trait })), cleanHit);
|
|
131
|
+
const prevTotal = Number.isSafeInteger(ev.totalPairs) && ev.totalPairs >= ev.pairs.length ? ev.totalPairs : ev.pairs.length;
|
|
127
132
|
const next = {
|
|
128
133
|
...ev,
|
|
129
|
-
pairs: [...ev.pairs, ...
|
|
134
|
+
pairs: [...ev.pairs, ...newPairs].slice(-MAX_PAIRS),
|
|
135
|
+
totalPairs: prevTotal + newPairs.length,
|
|
130
136
|
hits: [...ev.hits, ...newHits].slice(-MAX_HITS),
|
|
131
137
|
examples: [...ev.examples, ...add(examples, cleanExample)],
|
|
132
138
|
};
|
|
@@ -134,12 +140,13 @@ export function appendRecords({ pairs = [], hits = [], examples = [] } = {}, now
|
|
|
134
140
|
return save(next);
|
|
135
141
|
}
|
|
136
142
|
|
|
137
|
-
/** "Not now" ({later: pairs
|
|
143
|
+
/** "Not now" ({later: total pairs seen so far}) or "Don't suggest this" ({never}) for one trait. An unknown trait or action is a 400, nothing written. */
|
|
138
144
|
export function setAnswer(trait, action) {
|
|
139
145
|
if (typeof trait !== "string" || !TRAIT_IDS.includes(trait)) throw new VoiceError(`trait must be one of: ${TRAIT_IDS.join(", ")}`);
|
|
140
146
|
if (typeof action !== "string" || !ANSWERS.includes(action)) throw new VoiceError(`action must be one of: ${ANSWERS.join(", ")}`);
|
|
141
147
|
const ev = readEvidence();
|
|
142
|
-
|
|
148
|
+
const total = Number.isSafeInteger(ev.totalPairs) && ev.totalPairs >= ev.pairs.length ? ev.totalPairs : ev.pairs.length;
|
|
149
|
+
return save({ ...ev, answers: { ...ev.answers, [trait]: action === "never" ? { never: true } : { later: total } } });
|
|
143
150
|
}
|
|
144
151
|
|
|
145
152
|
/** Forget the kept edits, comment hits and examples. The learning switch stays; "never" answers stay (a choice, not evidence); "later" goes (it counted kept pairs). */
|
package/src/voice.js
CHANGED
|
@@ -271,11 +271,12 @@ export function suggestions(evidence, style) {
|
|
|
271
271
|
const hits = Array.isArray(ev.hits) ? ev.hits.filter((h) => h && typeof h === "object" && COMMENT_TRAITS.includes(h.trait)) : [];
|
|
272
272
|
const answers = ev.answers && typeof ev.answers === "object" && !Array.isArray(ev.answers) ? ev.answers : {};
|
|
273
273
|
const on = new Set(Array.isArray(style?.traits) ? style.traits : []);
|
|
274
|
+
const totalPairs = Number.isInteger(ev.totalPairs) ? ev.totalPairs : pairs.length;
|
|
274
275
|
const held = (trait) => {
|
|
275
276
|
const a = Object.hasOwn(answers, trait) ? answers[trait] : null;
|
|
276
277
|
if (!a || typeof a !== "object") return false;
|
|
277
278
|
if (a.never === true) return true;
|
|
278
|
-
return Number.isInteger(a.later) &&
|
|
279
|
+
return Number.isInteger(a.later) && totalPairs - a.later < LATER_AFTER;
|
|
279
280
|
};
|
|
280
281
|
const found = [];
|
|
281
282
|
const win = pairs.slice(-WINDOW);
|
package/src/xlsx.js
CHANGED
|
@@ -216,6 +216,8 @@ function readSheet(zip, sheet, sharedStrings, budget = bookBudget()) {
|
|
|
216
216
|
let field = null; // v | f | is
|
|
217
217
|
let inIs = false;
|
|
218
218
|
let phon = 0;
|
|
219
|
+
let rowIdx = -1;
|
|
220
|
+
let colIdx = 0;
|
|
219
221
|
walkXml(xml, {
|
|
220
222
|
onOpen(n, a) {
|
|
221
223
|
const l = local(n);
|
|
@@ -223,12 +225,30 @@ function readSheet(zip, sheet, sharedStrings, budget = bookBudget()) {
|
|
|
223
225
|
if (++budget.merges > MAX_MERGES) throw tooMany(budget, MAX_MERGES, "merged ranges");
|
|
224
226
|
const q = parseA1(a.ref ?? "");
|
|
225
227
|
if (q && (q.r1 > q.r0 || q.c1 > q.c0)) out.merges.push(q);
|
|
228
|
+
} else if (l === "row") {
|
|
229
|
+
rowIdx = a.r && /^\d+$/.test(a.r) ? Number(a.r) - 1 : rowIdx + 1;
|
|
230
|
+
colIdx = 0;
|
|
226
231
|
} else if (l === "c" && ++budget.cells > MAX_CELLS) throw tooMany(budget, MAX_CELLS, "cells");
|
|
227
|
-
else if (l === "c"
|
|
228
|
-
|
|
229
|
-
|
|
232
|
+
else if (l === "c") {
|
|
233
|
+
let r = -1;
|
|
234
|
+
let c = -1;
|
|
235
|
+
if (a.r) {
|
|
236
|
+
const m = /^([A-Z]{1,3})(\d+)$/.exec(a.r);
|
|
237
|
+
if (m) {
|
|
238
|
+
c = [...m[1]].reduce((x, ch) => x * 26 + ch.charCodeAt(0) - 64, 0) - 1;
|
|
239
|
+
r = Number(m[2]) - 1;
|
|
240
|
+
colIdx = c + 1;
|
|
241
|
+
}
|
|
242
|
+
} else {
|
|
243
|
+
r = rowIdx;
|
|
244
|
+
c = colIdx++;
|
|
245
|
+
}
|
|
246
|
+
cur = r >= 0 && r < 1_048_576 ? { c, r, t: a.t || "n", s: Number(a.s) || 0, v: "", hasV: false, f: null, is: "" } : null;
|
|
230
247
|
} else if (!cur) return;
|
|
231
|
-
else if (l === "v")
|
|
248
|
+
else if (l === "v") {
|
|
249
|
+
field = "v";
|
|
250
|
+
cur.hasV = true;
|
|
251
|
+
}
|
|
232
252
|
else if (l === "f") {
|
|
233
253
|
field = "f";
|
|
234
254
|
cur.f = "";
|
|
@@ -252,7 +272,7 @@ function readSheet(zip, sheet, sharedStrings, budget = bookBudget()) {
|
|
|
252
272
|
if ((budget.sharedText -= text.length) < 0) throw tooMany(budget, MAX_SHARED_CELL_TEXT, "characters of text in its cells");
|
|
253
273
|
}
|
|
254
274
|
else if (cur.t === "inlineStr") text = cur.is;
|
|
255
|
-
else if (cur.t === "b") text = cur.v === "1" ? "TRUE" : "FALSE";
|
|
275
|
+
else if (cur.t === "b") text = !cur.hasV ? "" : cur.v === "1" || cur.v === "true" ? "TRUE" : "FALSE";
|
|
256
276
|
else if (cur.t === "n" && NUM.test(cur.v)) text = String(Number(cur.v));
|
|
257
277
|
else text = cur.v;
|
|
258
278
|
if (text !== "" && cur.c < 16384) {
|
package/src/zip-write.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import zlib from "node:zlib";
|
|
2
2
|
import { gated } from "./stop-gate.js";
|
|
3
|
-
import { openZip, ZipError } from "./zip.js";
|
|
3
|
+
import { crc32, openZip, ZipError } from "./zip.js";
|
|
4
4
|
|
|
5
5
|
// Rewrites one entry of a ZIP held in memory and copies every other entry as it is: same order, same compressed bytes, same CRC, same
|
|
6
6
|
// central-directory fields (times, attributes, extra fields, comments). The local headers are written fresh from the central-directory
|
|
@@ -19,7 +19,7 @@ export function replaceEntry(bytes, name, data) {
|
|
|
19
19
|
if (!zip.entries.has(name)) throw new ZipError(`no entry ${JSON.stringify(name)} to replace`);
|
|
20
20
|
const newData = Buffer.from(data);
|
|
21
21
|
const newComp = gated(() => zlib.deflateRawSync(newData)); // never stopped inside zlib (src/stop-gate.js)
|
|
22
|
-
const newCrc =
|
|
22
|
+
const newCrc = crc32(newData);
|
|
23
23
|
const parts = [];
|
|
24
24
|
const central = [];
|
|
25
25
|
let offset = 0;
|
|
@@ -38,9 +38,11 @@ export function replaceEntry(bytes, name, data) {
|
|
|
38
38
|
const crc = mine ? newCrc : e.crc;
|
|
39
39
|
const size = mine ? newData.length : e.size;
|
|
40
40
|
const flags = e.flags & ~0x8; // sizes are in the local header now: no data descriptor
|
|
41
|
+
if (offset > 0xfffffffe || comp.length > 0xfffffffe || size > 0xfffffffe) throw new ZipError("the archive would need ZIP64");
|
|
42
|
+
const verNeeded = method === 8 ? Math.max(20, record.readUInt16LE(6)) : record.readUInt16LE(6);
|
|
41
43
|
const lh = Buffer.alloc(30);
|
|
42
44
|
lh.writeUInt32LE(0x04034b50, 0);
|
|
43
|
-
lh.writeUInt16LE(
|
|
45
|
+
lh.writeUInt16LE(verNeeded, 4); // version needed
|
|
44
46
|
lh.writeUInt16LE(flags, 6);
|
|
45
47
|
lh.writeUInt16LE(method, 8);
|
|
46
48
|
lh.writeUInt16LE(record.readUInt16LE(12), 10); // time
|
|
@@ -51,6 +53,7 @@ export function replaceEntry(bytes, name, data) {
|
|
|
51
53
|
lh.writeUInt16LE(nameLen, 26);
|
|
52
54
|
lh.writeUInt16LE(0, 28); // no local extra field
|
|
53
55
|
parts.push(lh, nameBuf, comp);
|
|
56
|
+
record.writeUInt16LE(verNeeded, 6);
|
|
54
57
|
record.writeUInt16LE(flags, 8);
|
|
55
58
|
record.writeUInt16LE(method, 10);
|
|
56
59
|
record.writeUInt32LE(crc, 16);
|
|
@@ -59,8 +62,10 @@ export function replaceEntry(bytes, name, data) {
|
|
|
59
62
|
record.writeUInt32LE(offset, 42);
|
|
60
63
|
central.push(record);
|
|
61
64
|
offset += 30 + nameLen + comp.length;
|
|
65
|
+
if (offset > 0xfffffffe) throw new ZipError("the archive would need ZIP64");
|
|
62
66
|
}
|
|
63
67
|
const cd = Buffer.concat(central);
|
|
68
|
+
if (offset + cd.length > 0xfffffffe) throw new ZipError("the archive would need ZIP64");
|
|
64
69
|
const commentLen = buf.readUInt16LE(zip.eocd + 20);
|
|
65
70
|
const end = Buffer.alloc(22);
|
|
66
71
|
sig(0x06054b50).copy(end, 0);
|
|
@@ -70,6 +75,5 @@ export function replaceEntry(bytes, name, data) {
|
|
|
70
75
|
end.writeUInt32LE(offset, 16);
|
|
71
76
|
end.writeUInt16LE(commentLen, 20);
|
|
72
77
|
const comment = buf.subarray(zip.eocd + 22, zip.eocd + 22 + commentLen);
|
|
73
|
-
if (offset + cd.length > 0xfffffffe) throw new ZipError("the archive would need ZIP64");
|
|
74
78
|
return Buffer.concat([...parts, cd, end, comment]);
|
|
75
79
|
}
|