onionskin-cli 0.0.0-stage → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +136 -0
  2. package/INSTALL.md +17 -0
  3. package/LICENSE +21 -0
  4. package/NOTICE +46 -0
  5. package/README.md +88 -2
  6. package/docs/contracts/batch.schema.json +839 -0
  7. package/docs/contracts/history.schema.json +126 -0
  8. package/docs/contracts/ops.schema.json +541 -0
  9. package/package.json +54 -4
  10. package/scripts/checker/check.py +411 -0
  11. package/scripts/checker/requirements.txt +5 -0
  12. package/scripts/install-check.mjs +9 -0
  13. package/scripts/requirements.txt +5 -0
  14. package/scripts/rewriter/apple/main.swift +135 -0
  15. package/scripts/rewriter/apple/onionskin-fm +0 -0
  16. package/scripts/rewriter/apple/onionskin-fm.json +6 -0
  17. package/skills/onionskin/SKILL.md +117 -0
  18. package/skills/onionskin/reference/batch.md +94 -0
  19. package/skills/onionskin/reference/formats.md +15 -0
  20. package/skills/onionskin/reference/ops.md +253 -0
  21. package/skills/onionskin/reference/troubleshooting.md +73 -0
  22. package/skills/onionskin/reference/voice-guide.md +86 -0
  23. package/skills/onionskin/voice.example.md +51 -0
  24. package/src/agent-detect.js +149 -0
  25. package/src/agent-events.js +159 -0
  26. package/src/agent-prompt.js +27 -0
  27. package/src/agent-runner.js +593 -0
  28. package/src/analyze.js +17 -0
  29. package/src/anchors.js +190 -0
  30. package/src/apply.js +387 -0
  31. package/src/auth.js +130 -0
  32. package/src/batch.js +322 -0
  33. package/src/check-worker.js +8 -0
  34. package/src/checker.js +293 -0
  35. package/src/cli.js +516 -0
  36. package/src/constants.js +20 -0
  37. package/src/diff.js +267 -0
  38. package/src/doctor.js +247 -0
  39. package/src/docx.js +438 -0
  40. package/src/fmt.js +30 -0
  41. package/src/fsutil.js +348 -0
  42. package/src/history-worker.js +16 -0
  43. package/src/history.js +576 -0
  44. package/src/lease.js +79 -0
  45. package/src/local.js +242 -0
  46. package/src/merge-pptx.js +198 -0
  47. package/src/merge-xlsx.js +453 -0
  48. package/src/merge.js +410 -0
  49. package/src/offload.js +34 -0
  50. package/src/ops/common.js +48 -0
  51. package/src/ops/docx-table.js +323 -0
  52. package/src/ops/docx.js +503 -0
  53. package/src/ops/engine.js +41 -0
  54. package/src/ops/pptx.js +336 -0
  55. package/src/ops/run.js +115 -0
  56. package/src/ops/runs.js +127 -0
  57. package/src/ops/schema.js +109 -0
  58. package/src/ops/xlsx.js +277 -0
  59. package/src/package.js +189 -0
  60. package/src/paths.js +64 -0
  61. package/src/person.js +66 -0
  62. package/src/pptx.js +194 -0
  63. package/src/question.js +47 -0
  64. package/src/quick-plan.js +107 -0
  65. package/src/quick.js +164 -0
  66. package/src/redline-check.js +251 -0
  67. package/src/redline-worker.js +29 -0
  68. package/src/redline.js +438 -0
  69. package/src/reply.js +410 -0
  70. package/src/review.js +274 -0
  71. package/src/rewrite-check.js +1014 -0
  72. package/src/rewriter-cleanup.js +105 -0
  73. package/src/rewriter-setup.js +104 -0
  74. package/src/rewriter.js +396 -0
  75. package/src/sanitize.js +42 -0
  76. package/src/sentence-span.js +25 -0
  77. package/src/server-entry.js +13 -0
  78. package/src/server.js +576 -0
  79. package/src/session.js +1910 -0
  80. package/src/settings.js +161 -0
  81. package/src/setup.js +160 -0
  82. package/src/stamp-words.js +10 -0
  83. package/src/stamps.js +59 -0
  84. package/src/style.js +110 -0
  85. package/src/suggest-reply.js +84 -0
  86. package/src/suggest.js +261 -0
  87. package/src/text.js +31 -0
  88. package/src/tracked-words.js +43 -0
  89. package/src/voice-guide.js +182 -0
  90. package/src/voice-learn.js +314 -0
  91. package/src/voice-sample-worker.js +95 -0
  92. package/src/voice-samples.js +329 -0
  93. package/src/voice-store.js +305 -0
  94. package/src/voice-worker.js +28 -0
  95. package/src/voice.js +297 -0
  96. package/src/xlsx.js +454 -0
  97. package/src/xml-tree.js +118 -0
  98. package/src/xml.js +127 -0
  99. package/src/zip-write.js +74 -0
  100. package/src/zip.js +88 -0
  101. package/vendor/engine/DocViewer-BZLXUcxJ.js +15137 -0
  102. package/vendor/engine/THIRD_PARTY_LICENSES.txt +79 -0
  103. package/vendor/engine/collab-BVynXMqc.js +2591 -0
  104. package/vendor/engine/collab.js +2 -0
  105. package/vendor/engine/core.js +4 -0
  106. package/vendor/engine/default-worker-DogNT09A.js +35625 -0
  107. package/vendor/engine/embed.js +107 -0
  108. package/vendor/engine/index.js +5 -0
  109. package/vendor/engine/office-engine.embed.js +184 -0
  110. package/vendor/engine/office-engine.worker.js +36030 -0
  111. package/vendor/engine/xlsx-ops-SYAv6V0f.js +939 -0
  112. package/vendor/engine-node/LICENSE +21 -0
  113. package/vendor/engine-node/engine-node.mjs +46618 -0
  114. package/vendor/engine.lock.json +26 -0
  115. package/web/dist/assets/index-CFISATkn.js +197 -0
  116. package/web/dist/assets/index-spXE4oTz.css +1 -0
  117. package/web/dist/index.html +14 -0
@@ -0,0 +1,105 @@
1
+ // One-time cleanup of the older writing helper (IBM Granite on MLX, about 1.3 GB). It removes exactly the paths that version created inside the
2
+ // state folder, and only when they look like what it created. Run by `onionskin setup-rewriter` (every kind) and by install.sh
3
+ // (`setup-rewriter --clean-old`). It never throws: a failure to remove something leaves it for the next run and is not a reason to stop a setup.
4
+ // <state>/rewriter-venv/ a Python virtual environment (it has a pyvenv.cfg)
5
+ // <state>/models/granite/ the model (it has a config.json); <state>/models/.granite.partial/ is a download that did not finish
6
+ // <state>/rewriter.json the settings file, when it names the Granite back end (backend "mlx" or model "granite")
7
+ // Safety: a path that is a link is skipped (never followed, never deleted through); a folder is removed only when its real path is exactly where it
8
+ // should be inside the real state folder; sizes are counted without following links. The settings file goes last, so a run that stops early
9
+ // finds its marker again next time. Apple's own settings file (backend "apple") is never touched.
10
+ import fs from "node:fs";
11
+ import path from "node:path";
12
+ import { stateDir } from "./paths.js";
13
+
14
+ const lstat = (p) => fs.lstatSync(p, { throwIfNoEntry: false });
15
+ const realDir = (p) => {
16
+ const st = lstat(p);
17
+ return st?.isDirectory() && !st.isSymbolicLink() ? st : null;
18
+ };
19
+ const realFile = (p) => {
20
+ const st = lstat(p);
21
+ return st?.isFile() && !st.isSymbolicLink() ? st : null;
22
+ };
23
+
24
+ /** Bytes of the regular files under `p`, links not followed. */
25
+ function treeBytes(p) {
26
+ const st = lstat(p);
27
+ if (!st || st.isSymbolicLink()) return 0;
28
+ if (st.isFile()) return st.size;
29
+ if (!st.isDirectory()) return 0;
30
+ let n = 0;
31
+ for (const e of fs.readdirSync(p)) n += treeBytes(path.join(p, e));
32
+ return n;
33
+ }
34
+
35
+ /** "1.3 GB", "42 MB", "under 1 MB". */
36
+ export function sizeWords(bytes) {
37
+ if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(1)} GB`;
38
+ if (bytes >= 1e6) return `${Math.round(bytes / 1e6)} MB`;
39
+ return "under 1 MB";
40
+ }
41
+
42
+ /**
43
+ * Removes what the older version left in `state` (default: the state folder). Returns {bytes, removed: [names]} (removed is empty when there was
44
+ * nothing to do) and `line`, the one plain sentence for the person, when something was removed.
45
+ */
46
+ export function cleanOldHelper(state = stateDir()) {
47
+ const out = { bytes: 0, removed: [] };
48
+ let root;
49
+ try {
50
+ if (!realDir(state)) return out;
51
+ root = fs.realpathSync(state);
52
+ } catch {
53
+ return out;
54
+ }
55
+ const inside = (p, ...names) => {
56
+ try {
57
+ return fs.realpathSync(p) === path.join(root, ...names);
58
+ } catch {
59
+ return false;
60
+ }
61
+ };
62
+ const remove = (p, names, gate) => {
63
+ try {
64
+ if (!realDir(p) || !inside(p, ...names) || !gate()) return;
65
+ const bytes = treeBytes(p);
66
+ fs.rmSync(p, { recursive: true, force: true }); // unlinks a link inside the folder, never follows it
67
+ if (!lstat(p)) {
68
+ out.bytes += bytes;
69
+ out.removed.push(names.join("/"));
70
+ }
71
+ } catch {}
72
+ };
73
+
74
+ const config = path.join(state, "rewriter.json");
75
+ let oldConfig = false;
76
+ try {
77
+ if (realFile(config)) {
78
+ const cfg = JSON.parse(fs.readFileSync(config, "utf8"));
79
+ oldConfig = cfg?.backend === "mlx" || cfg?.model === "granite";
80
+ }
81
+ } catch {}
82
+
83
+ const venv = path.join(state, "rewriter-venv");
84
+ remove(venv, ["rewriter-venv"], () => !!realFile(path.join(venv, "pyvenv.cfg")));
85
+ const models = path.join(state, "models");
86
+ if (realDir(models) && inside(models, "models")) {
87
+ const granite = path.join(models, "granite");
88
+ remove(granite, ["models", "granite"], () => !!realFile(path.join(granite, "config.json")));
89
+ remove(path.join(models, ".granite.partial"), ["models", ".granite.partial"], () => true);
90
+ try {
91
+ if (!fs.readdirSync(models).length) fs.rmdirSync(models); // only when nothing else is in it
92
+ } catch {}
93
+ }
94
+ if (oldConfig) {
95
+ try {
96
+ fs.rmSync(config, { force: true });
97
+ out.removed.push("rewriter.json");
98
+ } catch {}
99
+ }
100
+ if (out.removed.length) {
101
+ out.freed = `Removed the old writing helper ${out.bytes >= 1e6 ? `download (${sizeWords(out.bytes)})` : "settings"}.`; // the first sentence alone, for install.sh
102
+ out.line = `${out.freed} Quick fixes now use Apple's model already on this Mac.`;
103
+ }
104
+ return out;
105
+ }
@@ -0,0 +1,104 @@
1
+ // `onionskin setup-rewriter [--remove]`: turns on quick fixes on this Mac (milestone 2, idea 10), with Apple's built-in
2
+ // model. Nothing is downloaded and nothing is compiled: the helper ships ready-made in this app copy (scripts/rewriter/apple/onionskin-fm, built
3
+ // by scripts/build-apple-helper.mjs) and runs from there. Setup checks it the way the server does before every start (src/rewriter.js
4
+ // appleHelperState: macOS 26 or later, Apple silicon, the shipped hash, no quarantine mark; it clears a mark install.sh could not), asks it
5
+ // whether Apple's model is available, and sends it one test sentence. rewriter.json is written LAST, so a half setup reads as off.
6
+ // Every run first removes what the older Granite download left in the state folder (src/rewriter-cleanup.js).
7
+ import { execFile } from "node:child_process";
8
+ import fs from "node:fs";
9
+ import os from "node:os";
10
+ import path from "node:path";
11
+ import { ensureStateDir, writeStateFile } from "./paths.js";
12
+ import { PROBE_REASONS, appleHelperState, appleStateLine, clearQuarantine, createWorker, readRewriterConfig, rewriterPaths, sandboxed, workerEnv } from "./rewriter.js";
13
+ import { cleanOldHelper } from "./rewriter-cleanup.js";
14
+
15
+ const PROBE_MS = 120_000;
16
+ const AVAILABILITY_MS = 30_000;
17
+ export const SETUP_MODELS = ["apple"];
18
+ export const DEFAULT_SETUP_MODEL = "apple";
19
+
20
+ const fail = (reason, message) => Object.assign(new Error(message), { reason });
21
+
22
+ /** A real sentence through the worker that was just set up. Resolves {ok:true,text} or {ok:false,code}. Replaceable in tests. */
23
+ export async function probeLaunch(launch, { text = "This are a test of the writing helper.", limitMs = PROBE_MS } = {}) {
24
+ const worker = createWorker(launch, { startMs: limitMs });
25
+ try {
26
+ const until = Date.now() + limitMs;
27
+ for (;;) {
28
+ const r = await worker.rewrite("grammar", text, { timeoutMs: 20_000 });
29
+ if (r.code !== "warming_up") return r;
30
+ if (Date.now() > until) return { ok: false, code: "timeout" };
31
+ await new Promise((res) => setTimeout(res, 250));
32
+ }
33
+ } finally {
34
+ worker.close();
35
+ }
36
+ }
37
+
38
+ async function removeAll() {
39
+ const p = rewriterPaths();
40
+ fs.rmSync(p.config, { force: true }); // first: from here on the rewriter reads as off
41
+ fs.rmSync(p.appleBin, { force: true }); // what an older version built here
42
+ }
43
+
44
+ /** `onionskin-fm --probe`, in the sandbox the worker gets: {status, stdout}. Replaceable in tests. */
45
+ export function probeHelper(bin) {
46
+ const [cmd, args] = sandboxed(bin, ["--probe"]);
47
+ return new Promise((resolve) => {
48
+ const state = path.dirname(rewriterPaths().config);
49
+ const cwd = fs.existsSync(state) ? state : os.tmpdir(); // the doctor may ask before the state folder exists
50
+ execFile(cmd, args, { encoding: "utf8", timeout: AVAILABILITY_MS, env: workerEnv(), cwd }, (err, stdout) => resolve({ status: err ? (typeof err.code === "number" ? err.code : null) : 0, stdout: String(stdout || "") }));
51
+ });
52
+ }
53
+
54
+ async function setupApple(log, probe, deps) {
55
+ log("Setting up quick fixes with Apple's built-in model. Nothing is downloaded.");
56
+ let h = appleHelperState(deps.apple);
57
+ // install.sh clears the download mark; a copy it did not (unzipped and run in place, or copied by hand) is cleared here, on this one file
58
+ if (h.reason === "quarantined") {
59
+ if ((deps.clearQuarantine ?? clearQuarantine)(h.bin)) h = appleHelperState(deps.apple); // the very file that was checked
60
+ }
61
+ if (!h.ok) throw fail(h.reason === "helper_changed" ? "helper_missing" : h.reason, appleStateLine(h.reason));
62
+ const pr = await (deps.probeHelper ?? probeHelper)(h.bin);
63
+ if (pr.status !== 0) {
64
+ const reason = PROBE_REASONS[/^unavailable: (\w+)/.exec(String(pr.stdout || "").trim())?.[1]] ?? "model_unavailable";
65
+ throw fail(reason, appleStateLine(reason) ?? "Apple's built-in model is not available on this Mac.");
66
+ }
67
+ const config = { backend: "apple", model: "apple", sha256: h.sha256 };
68
+ await finish(config, log, probe, { cmd: h.bin, args: [] });
69
+ return "Apple's built-in model";
70
+ }
71
+
72
+ /** The real test sentence, then rewriter.json (0600, atomic). A failed test leaves the setup off. */
73
+ async function finish(config, log, probe, launch) {
74
+ fs.rmSync(rewriterPaths().config, { force: true }); // a setup that fails below must not leave the old one standing
75
+ log("Trying it once on a test sentence...");
76
+ const r = await probe(launch);
77
+ if (!r?.ok) throw fail("probe_failed", `the writing helper did not answer a test sentence (${r?.code || "no answer"})`);
78
+ writeStateFile(rewriterPaths().config, JSON.stringify(config, null, 2) + "\n");
79
+ }
80
+
81
+ /**
82
+ * Returns {name, removed?}. Throws an Error with `reason` (a key of APPLE_STATES in src/rewriter.js, model_unavailable, probe_failed, bad_model)
83
+ * and a plain message on failure; nothing is configured then. deps (tests): probe(launch) -> {ok}; probeHelper(bin) -> {status, stdout};
84
+ * apple (goes to appleHelperState: dir, host, quarantined); clearQuarantine(file) -> boolean.
85
+ */
86
+ export async function setupRewriter({ model = DEFAULT_SETUP_MODEL, remove = false } = {}, log = (s) => process.stderr.write(s + "\n"), deps = {}) {
87
+ const probe = deps.probe ?? probeLaunch;
88
+ ensureStateDir();
89
+ const old = cleanOldHelper(); // the older Granite download, once; the one plain line says what was freed
90
+ if (old.line) log(old.line);
91
+ if (remove) {
92
+ await removeAll();
93
+ return { removed: true };
94
+ }
95
+ if (!SETUP_MODELS.includes(model)) throw fail("bad_model", `There is no "${model}" writing helper any more. Quick fixes use Apple's built-in model, so run setup-rewriter without --model.`);
96
+ const have = readRewriterConfig(undefined, deps.apple);
97
+ if (have.ok && have.model === model) {
98
+ log("Quick fixes are already set up.");
99
+ return { name: have.name };
100
+ }
101
+ fs.mkdirSync(path.dirname(rewriterPaths().config), { recursive: true, mode: 0o700 });
102
+ const name = await setupApple(log, probe, deps);
103
+ return { name };
104
+ }
@@ -0,0 +1,396 @@
1
+ // The local rewriter (milestone 2, idea 10): Apple's built-in on-device model rewrites ONE sentence (Fix grammar; the other tasks are for tests).
2
+ // This file is the seam and the worker. It makes no decision about what may be written: the caller must pass every answer through
3
+ // checkRewrite() (src/rewrite-check.js) and the normal write path. An answer from here is untrusted text, like any agent's.
4
+ //
5
+ // getRewriter() -> {name, backend, rewrite(task, text, {timeoutMs}), warm()} | null null: not set up (or its files are not intact)
6
+ // rewrite() always resolves (never rejects): {ok:true, text} | {ok:false, code}. The codes mean "do not use this, hand the request
7
+ // to the agent": warming_up (the model is still loading, it loads in the background), busy, timeout, unavailable, worker_exited,
8
+ // bad_task, bad_text, bad_output, plus the worker's own (refused, generate_failed, bad_request, ...).
9
+ //
10
+ // The one back end is chosen by <state>/rewriter.json (0600, written last by `onionskin setup-rewriter`):
11
+ // {backend:"apple", model:"apple", sha256} scripts/rewriter/apple/onionskin-fm of this app copy, run in place
12
+ // The command to run is never read from that file: it is derived from fixed paths, and the helper must match the hash this app copy shipped
13
+ // with (onionskin-fm.json), or the rewriter reads as off. The helper ships ready-made in the zip and is run from the app folder (an upgrade
14
+ // replaces the folder, the helper and its hash together, so nothing is copied into the state folder to go stale). Before it is ever started
15
+ // (setup's probe, the server's worker, the doctor) appleHelperState() decides from the macOS version, the processor, the hash and the
16
+ // com.apple.quarantine mark, without running it: a helper that still carries that mark would make macOS show a hidden "Not Opened" dialog
17
+ // and hang. A rewriter.json of the older Granite download reads as off until setup removes the old files (src/rewriter-cleanup.js).
18
+ // Test only: ONIONSKIN_REWRITER_STUB=<json file> maps a sentence (or "<task>\t<sentence>") to its rewrite and replaces every back end.
19
+ // Test only: ONIONSKIN_APPLE_HELPER_DIR=<folder> holds a stand-in helper (and its onionskin-fm.json) in place of the shipped one, and
20
+ // ONIONSKIN_APPLE_HOST_MACOS=<version> pretends this Mac runs that macOS, so the browser tests reach every "not available" state.
21
+ //
22
+ // The worker is long-lived (a cold start loads the model), speaks JSON lines, is started on first use, stops after 15 idle minutes and on
23
+ // close, answers one request at a time, and runs offline: a minimal environment, the state folder as its working folder, and
24
+ // /usr/bin/sandbox-exec with the network denied. Ceiling: one request at a time for the whole server (a second one gets "busy" and goes
25
+ // to the agent), and a worker that does not answer within 60 s after a timeout is killed and starts cold again.
26
+ import { spawn, spawnSync } from "node:child_process";
27
+ import crypto from "node:crypto";
28
+ import fs from "node:fs";
29
+ import os from "node:os";
30
+ import path from "node:path";
31
+ import { fileURLToPath } from "node:url";
32
+ import { env } from "./constants.js";
33
+ import { stateDir } from "./paths.js";
34
+ import { REWRITE_TASKS } from "./rewrite-check.js";
35
+ import { untrusted } from "./sanitize.js";
36
+
37
+ const here = path.dirname(fileURLToPath(import.meta.url));
38
+ export const REWRITER_DIR = path.join(here, "..", "scripts", "rewriter");
39
+ export const APPLE_HELPER_DIR = path.join(REWRITER_DIR, "apple");
40
+ export const QUARANTINE = "com.apple.quarantine";
41
+ const XATTR = "/usr/bin/xattr";
42
+ const SANDBOX_EXEC = "/usr/bin/sandbox-exec";
43
+ const SANDBOX_PROFILE = "(version 1)(allow default)(deny network*)";
44
+ const MAX_TEXT_IN = 1000;
45
+ const MAX_TEXT_OUT = 2000;
46
+ const MAX_LINE = 256 * 1024;
47
+ const DEFAULT_TIMEOUT_MS = 4000;
48
+ const START_LIMIT_MS = 120_000; // a cold model load took 16 s on the test Mac; this is the give-up point
49
+ const STUCK_MS = 60_000;
50
+ const RETRY_AFTER_FAILURE_MS = 60_000;
51
+ const idleMs = () => Number(env("REWRITER_IDLE_MS", 15 * 60_000));
52
+
53
+ /** Where everything lives. `state` is the state folder (the doctor passes its own; everything else uses the server's). */
54
+ export const rewriterPaths = (state = stateDir()) => ({
55
+ config: path.join(state, "rewriter.json"),
56
+ appleBin: path.join(state, "bin", "onionskin-fm"), // what an older version built here; only `setup-rewriter --remove` still deletes it
57
+ });
58
+ export const fileSha256 = (file) => crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex");
59
+
60
+ const NAMES = { apple: "Apple's built-in model" };
61
+
62
+ /** Why Apple's model gives no instant suggestion, in the words a person reads (the page, the doctor, setup). */
63
+ export const APPLE_STATES = {
64
+ macos_too_old: "Quick fixes need macOS 26 or later.",
65
+ device_not_eligible: "This Mac cannot run Apple's on-device writing model.",
66
+ apple_intelligence_not_enabled: "Turn on Apple Intelligence in System Settings to get quick spelling and grammar fixes.",
67
+ model_not_ready: "Apple's writing model is still getting ready. Try again in a few minutes.",
68
+ quarantined: "macOS is holding back the small helper for quick fixes because it came from a download. Install Onionskin again to clear that.",
69
+ helper_missing: "The small helper for quick fixes is missing or damaged in this copy of Onionskin. Install Onionskin again to fix it.",
70
+ };
71
+ /** The probe's words (scripts/rewriter/apple/main.swift --probe) to the state codes above. */
72
+ export const PROBE_REASONS = { deviceNotEligible: "device_not_eligible", appleIntelligenceNotEnabled: "apple_intelligence_not_enabled", modelNotReady: "model_not_ready" };
73
+
74
+ /**
75
+ * Does the file carry the com.apple.quarantine mark? Asked of /usr/bin/xattr, never by running the file. true | false; anything but a clean
76
+ * "No such xattr" answer counts as marked (fail closed).
77
+ */
78
+ export function isQuarantined(file) {
79
+ const r = spawnSync(XATTR, ["-p", QUARANTINE, file], { encoding: "utf8", timeout: 5000 });
80
+ if (r.status === 0) return true;
81
+ return !(r.status === 1 && /No such xattr/.test(r.stderr || ""));
82
+ }
83
+ /** Removes the mark from exactly this file. true when the file carries no mark afterwards. */
84
+ export function clearQuarantine(file) {
85
+ spawnSync(XATTR, ["-d", QUARANTINE, file], { encoding: "utf8", timeout: 5000 });
86
+ return !isQuarantined(file);
87
+ }
88
+
89
+ let hostCache = null;
90
+ /** {macos: "26.1" | null, appleSilicon: boolean}, read once per process (sw_vers, and sysctl only for a Node that is not arm64). */
91
+ export function hostFacts() {
92
+ if (hostCache) return hostCache;
93
+ const out = (cmd, args) => {
94
+ try {
95
+ const r = spawnSync(cmd, args, { encoding: "utf8", timeout: 5000 });
96
+ return r.status === 0 ? r.stdout.trim() : null;
97
+ } catch {
98
+ return null;
99
+ }
100
+ };
101
+ const mac = process.platform === "darwin";
102
+ if (env("APPLE_HOST_MACOS", "")) return (hostCache = { macos: env("APPLE_HOST_MACOS", ""), appleSilicon: true }); // browser tests: pretend this macOS version
103
+ hostCache = { macos: mac ? out("/usr/bin/sw_vers", ["-productVersion"]) : null, appleSilicon: mac && (process.arch === "arm64" || out("/usr/sbin/sysctl", ["-n", "hw.optional.arm64"]) === "1") };
104
+ return hostCache;
105
+ }
106
+
107
+ /**
108
+ * May the shipped Apple helper be started at all? Decided without running it, in this order: macOS 26 or later, Apple silicon, the helper and
109
+ * its onionskin-fm.json are there and the hash matches, no quarantine mark. {ok:true, bin, sha256} | {ok:false, reason} (a key of APPLE_STATES,
110
+ * or helper_changed, which reads as helper_missing). `o` (tests): dir, host ({macos, appleSilicon}), quarantined(file).
111
+ */
112
+ export function appleHelperState(o = {}) {
113
+ const dir = o.dir ?? (env("APPLE_HELPER_DIR", "") || APPLE_HELPER_DIR); // the folder variable is for the browser tests, with a stand-in helper
114
+ const host = o.host ?? hostFacts();
115
+ const marked = o.quarantined ?? isQuarantined;
116
+ const major = Number(String(host.macos ?? "").split(".")[0]);
117
+ if (!Number.isFinite(major) || major < 26) return { ok: false, reason: "macos_too_old" };
118
+ if (!host.appleSilicon) return { ok: false, reason: "device_not_eligible" };
119
+ const bin = path.join(dir, "onionskin-fm");
120
+ let rec;
121
+ try {
122
+ rec = JSON.parse(fs.readFileSync(path.join(dir, "onionskin-fm.json"), "utf8"));
123
+ if (!fs.lstatSync(bin).isFile()) return { ok: false, reason: "helper_missing" };
124
+ } catch {
125
+ return { ok: false, reason: "helper_missing" };
126
+ }
127
+ if (typeof rec?.sha256 !== "string" || fileSha256(bin) !== rec.sha256) return { ok: false, reason: "helper_changed" };
128
+ if (marked(bin)) return { ok: false, reason: "quarantined", bin };
129
+ return { ok: true, bin, sha256: rec.sha256 };
130
+ }
131
+ /** The person's words for a reason code, or null. helper_changed reads as helper_missing. */
132
+ export const appleStateLine = (reason) => APPLE_STATES[reason === "helper_changed" ? "helper_missing" : reason] ?? null;
133
+
134
+ /**
135
+ * Reads <state>/rewriter.json and decides whether it is intact. {ok:true, backend, model, name, key, launch:{cmd,args}} or {ok:false, reason}.
136
+ * Cheap enough to run on every call (a few stat calls, one /usr/bin/xattr call; the Apple helper is about 110 KB to hash).
137
+ * `apple` (tests) goes to appleHelperState. A file of the older Granite setup (backend "mlx") is unknown_backend: off.
138
+ */
139
+ export function readRewriterConfig(state, apple = {}) {
140
+ const p = rewriterPaths(state);
141
+ let cfg;
142
+ try {
143
+ cfg = JSON.parse(fs.readFileSync(p.config, "utf8"));
144
+ } catch {
145
+ return { ok: false, reason: "not_set_up" };
146
+ }
147
+ if (cfg?.backend === "apple") {
148
+ if (cfg.model !== "apple") return { ok: false, reason: "config_mismatch" };
149
+ const h = appleHelperState(apple);
150
+ if (!h.ok) return { ok: false, reason: h.reason };
151
+ return { ok: true, backend: "apple", model: "apple", name: NAMES.apple, key: `apple:${h.sha256}`, launch: { cmd: h.bin, args: [] } };
152
+ }
153
+ return { ok: false, reason: "unknown_backend" };
154
+ }
155
+
156
+ /** {on:true, name, backend} | {on:false, reason} for the doctor and the setup command. `apple` (tests) goes to appleHelperState. */
157
+ export function rewriterStatus(state, apple = {}) {
158
+ if (env("REWRITER_STUB", "")) return { on: true, name: "test stand-in", backend: "stub" };
159
+ const c = readRewriterConfig(state, apple);
160
+ return c.ok ? { on: true, name: c.name, backend: c.backend } : { on: false, reason: c.reason };
161
+ }
162
+
163
+ /** [command, args] with the network denied for the process (and everything it starts). Without sandbox-exec the command runs as is. */
164
+ export function sandboxed(cmd, args, sandboxExec = SANDBOX_EXEC) {
165
+ return fs.existsSync(sandboxExec) ? [sandboxExec, ["-p", SANDBOX_PROFILE, cmd, ...args]] : [cmd, args];
166
+ }
167
+
168
+ /** The environment the worker gets: nothing of the server's, the state folder as home. */
169
+ export function workerEnv() {
170
+ return { PATH: "/usr/bin:/bin", HOME: stateDir(), TMPDIR: os.tmpdir() };
171
+ }
172
+
173
+ /**
174
+ * One worker process behind a request function. `launch` is {cmd, args, env?, cwd?}; the real ones come from getRewriter(), tests pass a fake script.
175
+ * Options: idleMs (default 15 minutes), startMs, stuckMs, retryMs.
176
+ */
177
+ export function createWorker(launch, opts = {}) {
178
+ const o = { idleMs: idleMs(), startMs: START_LIMIT_MS, stuckMs: STUCK_MS, retryMs: RETRY_AFTER_FAILURE_MS, ...opts };
179
+ let child = null;
180
+ let ready = false;
181
+ let failedAt = 0;
182
+ let startCode = null; // the worker's own reason for not starting ({"ready":false,"code"}), answered until the next try
183
+ let nextId = 1;
184
+ let busy = null; // {id, settle}: the request the worker is still working on (also after its caller timed out)
185
+ let idleTimer = null;
186
+ let startTimer = null;
187
+ let stuckTimer = null;
188
+
189
+ const stop = () => {
190
+ clearTimeout(idleTimer);
191
+ clearTimeout(startTimer);
192
+ clearTimeout(stuckTimer);
193
+ const c = child;
194
+ child = null;
195
+ ready = false;
196
+ if (c) {
197
+ try {
198
+ c.stdin.end();
199
+ } catch {}
200
+ c.kill("SIGTERM");
201
+ }
202
+ if (busy) {
203
+ busy.settle({ ok: false, code: "worker_exited" });
204
+ busy = null;
205
+ }
206
+ };
207
+ const touchIdle = () => {
208
+ clearTimeout(idleTimer);
209
+ idleTimer = setTimeout(stop, o.idleMs);
210
+ idleTimer.unref?.();
211
+ };
212
+
213
+ function start() {
214
+ let c;
215
+ try {
216
+ c = spawn(launch.cmd, launch.args || [], { stdio: ["pipe", "pipe", "ignore"], env: launch.env ?? workerEnv(), cwd: launch.cwd ?? stateDir() });
217
+ } catch {
218
+ failedAt = Date.now();
219
+ return;
220
+ }
221
+ child = c;
222
+ c.unref();
223
+ let buf = "";
224
+ const gone = () => {
225
+ if (child !== c) return;
226
+ if (!ready) failedAt = Date.now();
227
+ stop();
228
+ };
229
+ c.once("error", gone);
230
+ c.once("exit", gone);
231
+ c.stdin.on("error", () => {});
232
+ c.stdout.setEncoding("utf8");
233
+ c.stdout.on("data", (d) => {
234
+ buf += d;
235
+ if (buf.length > MAX_LINE && !buf.includes("\n")) return gone(); // a worker that never ends a line is not speaking our protocol
236
+ let at;
237
+ while ((at = buf.indexOf("\n")) >= 0) {
238
+ const line = buf.slice(0, at);
239
+ buf = buf.slice(at + 1);
240
+ if (child === c) onLine(line);
241
+ }
242
+ });
243
+ startTimer = setTimeout(() => {
244
+ if (child === c && !ready) {
245
+ failedAt = Date.now();
246
+ stop();
247
+ }
248
+ }, o.startMs);
249
+ startTimer.unref?.();
250
+ }
251
+
252
+ function onLine(line) {
253
+ let m;
254
+ try {
255
+ m = JSON.parse(line);
256
+ } catch {
257
+ return;
258
+ }
259
+ if (!m || typeof m !== "object") return;
260
+ if ("ready" in m) {
261
+ if (m.ready === true && !ready) {
262
+ ready = true;
263
+ startCode = null; // a start that worked (also one begun by warm()) replaces an earlier reason
264
+ clearTimeout(startTimer);
265
+ touchIdle();
266
+ } else if (m.ready === false) {
267
+ failedAt = Date.now();
268
+ startCode = Object.values(PROBE_REASONS).includes(m.code) ? m.code : null; // Apple's three reasons; anything else reads as "unavailable"
269
+ stop();
270
+ }
271
+ return;
272
+ }
273
+ if (!busy || m.id !== busy.id) return;
274
+ const b = busy;
275
+ busy = null;
276
+ clearTimeout(stuckTimer);
277
+ touchIdle();
278
+ if (m.ok === true && typeof m.text === "string" && m.text.length <= MAX_TEXT_OUT) b.settle({ ok: true, text: m.text });
279
+ else if (m.ok === true) b.settle({ ok: false, code: "bad_output" });
280
+ else b.settle({ ok: false, code: goodCode(m.code) ?? "failed", ...(typeof m.reason === "string" && m.reason ? { reason: untrusted(m.reason, 200) } : {}) });
281
+ }
282
+
283
+ return {
284
+ /** Starts the model in the background if it is not running. */
285
+ warm() {
286
+ if (!child && Date.now() - failedAt >= o.retryMs) start();
287
+ },
288
+ rewrite(task, text, { timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
289
+ if (!REWRITE_TASKS.includes(task)) return Promise.resolve({ ok: false, code: "bad_task" });
290
+ if (typeof text !== "string" || !text.trim() || text.length > MAX_TEXT_IN) return Promise.resolve({ ok: false, code: "bad_text" });
291
+ if (!child) {
292
+ if (Date.now() - failedAt < o.retryMs) return Promise.resolve({ ok: false, code: startCode ?? "unavailable" });
293
+ startCode = null;
294
+ start();
295
+ }
296
+ if (!child) return Promise.resolve({ ok: false, code: "unavailable" });
297
+ if (!ready) return Promise.resolve({ ok: false, code: "warming_up" });
298
+ if (busy) return Promise.resolve({ ok: false, code: "busy" });
299
+ return new Promise((resolve) => {
300
+ const id = nextId++;
301
+ let timer;
302
+ const settle = (r) => {
303
+ clearTimeout(timer);
304
+ resolve(r);
305
+ };
306
+ busy = { id, settle };
307
+ timer = setTimeout(() => settle({ ok: false, code: "timeout" }), timeoutMs);
308
+ // the worker may still be working on it; "busy" holds until it answers, and a worker that never does is stopped
309
+ stuckTimer = setTimeout(() => {
310
+ if (busy?.id === id) {
311
+ failedAt = Date.now();
312
+ stop();
313
+ }
314
+ }, timeoutMs + o.stuckMs);
315
+ stuckTimer.unref?.();
316
+ try {
317
+ child.stdin.write(JSON.stringify({ id, task, text }) + "\n");
318
+ } catch {
319
+ busy = null;
320
+ settle({ ok: false, code: "worker_exited" });
321
+ }
322
+ });
323
+ },
324
+ close: stop,
325
+ /** For tests and the doctor: is a model loaded right now? */
326
+ get ready() {
327
+ return ready;
328
+ },
329
+ /** The worker's own reason for its last failed start (device_not_eligible, ...), or null. */
330
+ get startCode() {
331
+ return startCode;
332
+ },
333
+ };
334
+ }
335
+
336
+ const goodCode = (c) => (typeof c === "string" && /^[a-z_]{1,40}$/.test(c) ? c : null);
337
+
338
+ // ---- the stub back end (tests) ---------------------------------------------------------------------------------------
339
+ function stubRewriter(file) {
340
+ return {
341
+ name: "test stand-in",
342
+ backend: "stub",
343
+ warm() {},
344
+ close() {},
345
+ async rewrite(task, text, { timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
346
+ if (!REWRITE_TASKS.includes(task)) return { ok: false, code: "bad_task" };
347
+ if (typeof text !== "string" || !text.trim() || text.length > MAX_TEXT_IN) return { ok: false, code: "bad_text" };
348
+ let map;
349
+ try {
350
+ map = JSON.parse(fs.readFileSync(file, "utf8"));
351
+ } catch {
352
+ return { ok: false, code: "unavailable" };
353
+ }
354
+ const hit = Object.hasOwn(map, `${task}\t${text}`) ? map[`${task}\t${text}`] : Object.hasOwn(map, text) ? map[text] : undefined;
355
+ if (hit === undefined) return { ok: false, code: "no_stub" };
356
+ const e = typeof hit === "string" ? { text: hit } : hit;
357
+ if (e.delayMs > timeoutMs) {
358
+ await new Promise((r) => setTimeout(r, timeoutMs));
359
+ return { ok: false, code: "timeout" };
360
+ }
361
+ if (e.delayMs) await new Promise((r) => setTimeout(r, e.delayMs));
362
+ if (typeof e.code === "string") return { ok: false, code: e.code };
363
+ return typeof e.text === "string" && e.text.length <= MAX_TEXT_OUT ? { ok: true, text: e.text } : { ok: false, code: "bad_output" };
364
+ },
365
+ };
366
+ }
367
+
368
+ // ---- the one worker of this server ---------------------------------------------------------------------------------------
369
+ let current = null; // {key, worker}
370
+
371
+ /** Stops the worker (server shutdown, or the configuration changed). Safe to call any time. */
372
+ export function stopRewriter() {
373
+ current?.worker.close();
374
+ current = null;
375
+ }
376
+
377
+ /**
378
+ * The rewriter, or null when it is not set up, its files are not intact, or (Apple) its helper may not be started (appleHelperState). A removed or
379
+ * replaced setup stops the worker that was running. `apple` (tests) goes to appleHelperState.
380
+ */
381
+ export function getRewriter(apple = {}) {
382
+ const stub = env("REWRITER_STUB", "");
383
+ if (stub) return stubRewriter(stub);
384
+ const c = readRewriterConfig(undefined, apple);
385
+ if (!c.ok) {
386
+ stopRewriter();
387
+ return null;
388
+ }
389
+ if (current && current.key !== c.key) stopRewriter();
390
+ if (!current) {
391
+ const [cmd, args] = sandboxed(c.launch.cmd, c.launch.args);
392
+ current = { key: c.key, worker: createWorker({ cmd, args }) };
393
+ }
394
+ const { worker } = current;
395
+ return { name: c.name, backend: c.backend, rewrite: (task, text, o) => worker.rewrite(task, text, o), warm: () => worker.warm(), close: stopRewriter, startCode: () => worker.startCode };
396
+ }
@@ -0,0 +1,42 @@
1
+ // Built with new RegExp from escaped strings so no raw separator characters live in this source file.
2
+ // C0 (\n \t \r become spaces first), DEL, C1, bidi controls (LRM/RLM/ALM, embeddings, overrides, isolates), zero-width and BOM.
3
+ const STRIP = new RegExp("[\\u0000-\\u001F\\u007F-\\u009F\\u061C\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u2069\\uFEFF]", "g");
4
+ const WS = new RegExp("[\\t\\n\\r\\u2028\\u2029]+", "g");
5
+ // the user's own words keep newlines (\u000A) and tabs; everything else control/bidi is dropped
6
+ const USER_STRIP = new RegExp("[\\u0000-\\u0008\\u000B-\\u001F\\u007F-\\u009F\\u061C\\u200E\\u200F\\u202A-\\u202E\\u2066-\\u2069\\uFEFF]", "g");
7
+ const MAX_UNTRUSTED = 500;
8
+ const MARK = "…[truncated]";
9
+
10
+ /** Untrusted document-derived text: strip control and bidi characters, NFC, collapse whitespace, cap at 500 chars including the marker. */
11
+ export function untrusted(value, max = MAX_UNTRUSTED) {
12
+ if (value == null) return "";
13
+ let s = String(value).replace(WS, " ").replace(STRIP, "").normalize("NFC");
14
+ s = s.replace(/ {2,}/g, " ").trim();
15
+ const cps = [...s];
16
+ if (cps.length > max) s = cps.slice(0, max - [...MARK].length).join("") + MARK;
17
+ return s;
18
+ }
19
+
20
+ /** The user's own words: keep newlines, strip other control and bidi characters, cap generously. */
21
+ export function userText(value, max = 5000) {
22
+ if (value == null) return "";
23
+ const s = String(value).replace(/\r\n?/g, "\n").replace(USER_STRIP, "").normalize("NFC");
24
+ const cps = [...s];
25
+ return cps.length > max ? cps.slice(0, max).join("") + MARK : s;
26
+ }
27
+
28
+ const FLAG_KEEP = new Set(["code", "severity"]);
29
+ /**
30
+ * The one chokepoint for flags that leave the package checks, the session or apply. Every string except `code` and `severity` came from
31
+ * the document (a part name, a relationship target, an error message) or from an agent, so it passes through `untrusted()` (no control
32
+ * or bidi characters, one line, at most 200 characters). Arrays are cleaned element by element; numbers and booleans pass unchanged.
33
+ */
34
+ export function cleanFlag(flag) {
35
+ const clean = (v) => (typeof v === "string" ? untrusted(v, 200) : Array.isArray(v) ? v.map(clean) : v);
36
+ const out = {};
37
+ for (const [k, v] of Object.entries(flag)) out[k] = FLAG_KEEP.has(k) ? v : clean(v);
38
+ return out;
39
+ }
40
+
41
+ /** True when `value` holds a control, bidi or zero-width character (the set `untrusted()` strips, newlines included). */
42
+ export const hasControlChars = (value) => new RegExp(STRIP.source).test(String(value)) || new RegExp(WS.source).test(String(value));