onionskin-cli 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +98 -0
  2. package/README.md +4 -4
  3. package/package.json +1 -1
  4. package/scripts/checker/check.py +24 -5
  5. package/scripts/checker/requirements.txt +482 -5
  6. package/scripts/requirements.txt +482 -5
  7. package/scripts/rewriter/apple/main.swift +1 -1
  8. package/scripts/rewriter/apple/onionskin-fm +0 -0
  9. package/scripts/rewriter/apple/onionskin-fm.json +1 -1
  10. package/skills/onionskin/SKILL.md +4 -4
  11. package/skills/onionskin/reference/ops.md +1 -1
  12. package/src/agent-runner.js +94 -15
  13. package/src/checker.js +238 -33
  14. package/src/cli.js +173 -33
  15. package/src/doctor.js +12 -3
  16. package/src/docx.js +130 -63
  17. package/src/fsutil.js +13 -5
  18. package/src/history.js +11 -2
  19. package/src/html-render.js +74 -15
  20. package/src/html-utf8.js +1 -1
  21. package/src/html.js +3 -1
  22. package/src/merge-package.js +82 -26
  23. package/src/offload.js +25 -6
  24. package/src/ops/docx-table.js +5 -3
  25. package/src/ops/pptx.js +8 -2
  26. package/src/ops/xlsx.js +114 -8
  27. package/src/package.js +17 -2
  28. package/src/pptx.js +244 -8
  29. package/src/question.js +10 -3
  30. package/src/quick.js +1 -1
  31. package/src/redline-check.js +12 -2
  32. package/src/rewriter.js +38 -8
  33. package/src/server-entry.js +4 -3
  34. package/src/server.js +97 -28
  35. package/src/session.js +13 -2
  36. package/src/setup.js +20 -2
  37. package/src/update.js +248 -19
  38. package/src/voice-guide.js +12 -7
  39. package/src/voice-learn.js +11 -1
  40. package/src/voice-store.js +11 -4
  41. package/src/voice.js +2 -1
  42. package/src/xlsx.js +25 -5
  43. package/src/zip-write.js +8 -4
  44. package/src/zip.js +64 -13
  45. package/web/dist/assets/{index-DytKkMdi.js → index-CjsTf5gj.js} +108 -106
  46. package/web/dist/assets/index-DnbLtHhG.css +1 -0
  47. package/web/dist/index.html +2 -2
  48. package/web/dist/assets/index-DkNx0LSH.css +0 -1
package/src/cli.js CHANGED
@@ -96,6 +96,7 @@ function req(options, body) {
96
96
  res.on("close", () => !res.complete && reject(new Error("server connection closed")));
97
97
  });
98
98
  r.on("error", reject);
99
+ if (options.timeout) r.on("timeout", () => r.destroy(new Error("request timed out")));
99
100
  if (options.signal) options.signal.addEventListener("abort", () => r.destroy(new Error("aborted")), { once: true });
100
101
  // `deadline` (ms): the whole request, answer included, or it ends with `deadlineError`
101
102
  if (options.deadline) {
@@ -120,11 +121,12 @@ const json = (t) => {
120
121
  }
121
122
  };
122
123
 
123
- async function healthy() {
124
+ async function healthy({ reportMismatch = false } = {}) {
124
125
  try {
125
126
  const r = await req({ method: "GET", path: "/health", timeout: 1500 });
126
127
  const j = json(r.text);
127
- return r.status === 200 && j.protocol === SERVER_PROTOCOL;
128
+ if (r.status === 200 && j?.protocol === SERVER_PROTOCOL) return true;
129
+ return reportMismatch && r.status === 200 ? "incompatible" : false;
128
130
  } catch {
129
131
  return false;
130
132
  }
@@ -162,7 +164,14 @@ async function removeStaleSocket() {
162
164
  continue;
163
165
  }
164
166
  }
165
- if (!Number.isInteger(pid)) continue;
167
+ if (!Number.isInteger(pid) || pid <= 0) {
168
+ if (f.endsWith("start.lock")) {
169
+ try {
170
+ if (Date.now() - fs.statSync(f).mtimeMs <= 2000) return false;
171
+ } catch {}
172
+ }
173
+ continue;
174
+ }
166
175
  try {
167
176
  process.kill(pid, 0);
168
177
  return false; // that process is alive: it may be starting up on this socket
@@ -178,48 +187,180 @@ async function removeStaleSocket() {
178
187
  }
179
188
  }
180
189
 
181
- /** `stop` (and `update`): asks a running server to stop, waits for its process to end, then removes a leftover socket. */
182
- async function stopServer() {
183
- if (await healthy()) {
184
- const pid = json((await req({ method: "GET", path: "/health", timeout: 1500 }).catch(() => ({ text: "" }))).text).pid;
185
- await req({ method: "POST", path: "/cli/stop" }, {});
186
- // the server answers first and exits a moment later: "stopped" is printed only once its process is gone (waits at most 5 s)
187
- const gone = () => {
190
+ /** True when `pid` exists (EPERM also means it exists, owned by another user). */
191
+ function pidAlive(pid) {
192
+ try {
193
+ process.kill(pid, 0);
194
+ return true;
195
+ } catch (e) {
196
+ return e.code === "EPERM";
197
+ }
198
+ }
199
+
200
+ /** True when a Unix socket or TCP endpoint accepts a connection within `timeoutMs`. */
201
+ function canConnect(opts, timeoutMs = 400) {
202
+ return new Promise((resolve) => {
203
+ let done = false;
204
+ const finish = (ok) => {
205
+ if (done) return;
206
+ done = true;
207
+ clearTimeout(t);
188
208
  try {
189
- process.kill(pid, 0);
190
- return false;
191
- } catch {
192
- return true;
193
- }
209
+ s.destroy();
210
+ } catch {}
211
+ resolve(ok);
194
212
  };
195
- for (let i = 0; i < 100 && Number.isInteger(pid) && !gone(); i++) await new Promise((r) => setTimeout(r, 50));
213
+ const s = net.createConnection(opts);
214
+ const t = setTimeout(() => finish(false), timeoutMs);
215
+ t.unref?.();
216
+ s.once("connect", () => finish(true));
217
+ s.once("error", () => finish(false));
218
+ });
219
+ }
220
+
221
+ /** True when `pid`'s command line looks like an Onionskin server process. */
222
+ function looksLikeOnionskinPid(pid) {
223
+ const r = spawnSync("ps", ["-o", "command=", "-p", String(pid)], { encoding: "utf8", timeout: 1000 });
224
+ if (r.status !== 0) return false;
225
+ return /server-entry\.js|onionskin/i.test((r.stdout || "").trim());
226
+ }
227
+
228
+ /**
229
+ * Inspects server.json / start.lock when /health did not answer.
230
+ * Treats a recorded pid as the server ONLY when identity is confirmed (the socket or recorded port accepts
231
+ * connections, or the process command matches Onionskin); otherwise removes the stale record(s).
232
+ */
233
+ async function inspectRecordedServer() {
234
+ let staleRemoved = false;
235
+ let sockAccepts = null;
236
+ const checkSock = async () => {
237
+ if (sockAccepts === null) sockAccepts = await canConnect({ path: socketPath() });
238
+ return sockAccepts;
239
+ };
240
+ for (const f of [serverRecordPath(), path.join(stateDir(), "start.lock")]) {
241
+ let raw;
242
+ try {
243
+ raw = fs.readFileSync(f, "utf8");
244
+ } catch {
245
+ continue;
246
+ }
247
+ let pid = null;
248
+ let port = null;
249
+ try {
250
+ const obj = JSON.parse(raw);
251
+ pid = Number.parseInt(obj?.pid ?? raw, 10);
252
+ if (Number.isInteger(obj?.port) && obj.port > 0 && obj.port <= 65535) port = obj.port;
253
+ } catch {
254
+ pid = Number.parseInt(raw, 10);
255
+ }
256
+ if (Number.isInteger(pid) && pid > 0 && pidAlive(pid)) {
257
+ const confirmed =
258
+ (await checkSock()) ||
259
+ (port !== null && (await canConnect({ host: "127.0.0.1", port }))) ||
260
+ looksLikeOnionskinPid(pid);
261
+ if (confirmed) return { pid, staleRemoved };
262
+ // If a socket file is still present alongside a live PID, leave both untouched (a server may be starting up; test/stop.test.js)
263
+ // while still returning pid: null so `stop` does not wait 5s or report an unrelated process as stuck.
264
+ if (fs.existsSync(socketPath())) continue;
265
+ }
266
+ try {
267
+ if (f.endsWith("start.lock") && Date.now() - fs.statSync(f).mtimeMs <= 2000) continue;
268
+ fs.unlinkSync(f);
269
+ staleRemoved = true;
270
+ } catch {}
271
+ }
272
+ return { pid: null, staleRemoved };
273
+ }
274
+
275
+ /** `stop` (and `update`): asks a running server to stop, waits for its process to end, then removes a leftover socket. */
276
+ async function stopServer({ waitMs = 5000 } = {}) {
277
+ let pid = null;
278
+ let staleRemoved = false;
279
+ let shouldStop = await healthy();
280
+ if (shouldStop) {
281
+ pid = json((await req({ method: "GET", path: "/health", timeout: 1500 }).catch(() => ({ text: "" }))).text).pid;
282
+ } else {
283
+ const r = await req({ method: "GET", path: "/health", timeout: 1500 }).catch(() => null);
284
+ const j = r?.status === 200 ? json(r.text) : null;
285
+ if (j && Number.isInteger(j.pid) && j.pid > 0) {
286
+ pid = j.pid;
287
+ shouldStop = true;
288
+ }
289
+ }
290
+ if (!Number.isInteger(pid) || pid <= 0) {
291
+ const rec = await inspectRecordedServer();
292
+ staleRemoved = rec.staleRemoved;
293
+ if (rec.pid) {
294
+ pid = rec.pid;
295
+ shouldStop = true;
296
+ }
297
+ }
298
+ if (shouldStop) {
299
+ await req({ method: "POST", path: "/cli/stop", timeout: 2000 }, {}).catch(() => {});
300
+ // the server answers first and exits a moment later: "stopped" is printed only once its process is gone (waits at most waitMs)
301
+ const gone = () => !pidAlive(pid);
302
+ const steps = Math.max(1, Math.ceil(waitMs / 50));
303
+ for (let i = 0; i < steps && Number.isInteger(pid) && pid > 0 && !gone(); i++) await new Promise((r) => setTimeout(r, 50));
304
+ if (Number.isInteger(pid) && pid > 0 && !gone()) {
305
+ throw new Error(`server (pid ${pid}) did not stop within ${Math.round(waitMs / 1000)}s`);
306
+ }
196
307
  }
197
308
  await removeStaleSocket(); // a socket file nothing answers on is no use to anyone, and `doctor` would keep reporting it
309
+ return { staleRemoved, stopped: shouldStop };
198
310
  }
199
311
 
200
312
  /** Starts the detached server and waits (event-based) for its "ready" line. */
201
313
  async function ensureServer() {
202
314
  ensureStateDir();
203
315
  if (await healthy()) return;
204
- const child = spawn(process.execPath, [path.join(here, "server-entry.js")], { detached: true, stdio: ["ignore", "pipe", "ignore"], env: process.env });
205
- await new Promise((resolve, reject) => {
206
- const to = setTimeout(() => reject(new Error(`could not start the ${PRODUCT.name} server`)), 15000);
207
- child.stdout.once("data", () => (clearTimeout(to), resolve()));
208
- child.once("exit", () => (clearTimeout(to), resolve())); // lost a race to another server: it is healthy now
209
- child.once("error", (e) => (clearTimeout(to), reject(e)));
316
+ const child = spawn(process.execPath, [path.join(here, "server-entry.js")], { detached: true, stdio: ["ignore", "pipe", "pipe"], env: process.env });
317
+ const errChunks = [];
318
+ let errLen = 0;
319
+ child.stderr.on("data", (d) => {
320
+ if (errLen >= 4096) return;
321
+ const slice = d.length > 4096 - errLen ? d.subarray(0, 4096 - errLen) : d;
322
+ errChunks.push(slice);
323
+ errLen += slice.length;
210
324
  });
211
- child.stdout.destroy();
212
- child.unref();
213
- if (!(await healthy())) throw new Error(`the ${PRODUCT.name} server did not come up`);
325
+ const detail = () => {
326
+ const s = Buffer.concat(errChunks).toString("utf8").trim();
327
+ return s ? `: ${s}` : "";
328
+ };
329
+ try {
330
+ await new Promise((resolve, reject) => {
331
+ const to = setTimeout(() => reject(new Error(`could not start the ${PRODUCT.name} server${detail()}`)), 15000);
332
+ child.stdout.once("data", () => (clearTimeout(to), resolve()));
333
+ child.once("exit", () => (clearTimeout(to), resolve())); // lost a race to another server: it is healthy now
334
+ child.once("error", (e) => (clearTimeout(to), reject(new Error(`${e.message}${detail()}`))));
335
+ });
336
+ if (!(await healthy())) {
337
+ if (child.exitCode !== null && !child.stderr.readableEnded) {
338
+ await Promise.race([new Promise((r) => child.stderr.once("end", r)), new Promise((r) => setTimeout(r, 50))]);
339
+ }
340
+ throw new Error(`the ${PRODUCT.name} server did not come up${detail()}`);
341
+ }
342
+ } finally {
343
+ child.stdout.destroy();
344
+ child.stderr.destroy();
345
+ child.unref();
346
+ }
347
+ }
348
+
349
+ export function openBrowserArgs(url, platform = process.platform) {
350
+ if (platform === "darwin") return ["open", [url]];
351
+ if (platform === "win32") return ["cmd", ["/c", "start", "", url.replace(/&/g, "^&")]];
352
+ return ["xdg-open", [url]];
214
353
  }
215
354
 
216
355
  function openBrowser(url) {
217
- const cmd = process.platform === "darwin" ? ["open", [url]] : process.platform === "win32" ? ["cmd", ["/c", "start", "", url]] : ["xdg-open", [url]];
218
- const c = spawn(cmd[0], cmd[1], { detached: true, stdio: "ignore" });
356
+ const [bin, args] = openBrowserArgs(url);
357
+ const c = spawn(bin, args, { detached: true, stdio: "ignore" });
219
358
  c.on("error", () => {});
220
359
  c.unref();
221
360
  }
222
361
 
362
+ export { req, stopServer, ensureServer };
363
+
223
364
  /** `config background <preset|default>`, `config background --image <file> [--strength N] [--no-blur]`, `config get background` */
224
365
  export function parseConfigArgs(args) {
225
366
  const a = { action: "", key: "", value: "", image: "", strength: null, blur: null };
@@ -276,7 +417,7 @@ export function configCommand(args) {
276
417
  return `Background set to your image. It is copied as it is; the page scales it (and the copy keeps its metadata on disk). Reload the page to see it.\n`;
277
418
  }
278
419
  writeBackground({ kind: "preset", name: a.value, ...keep });
279
- return `Background set to ${a.value}. Reload the page to see it.\n`;
420
+ return `Background set to ${a.reset ? `the default (${a.value})` : a.value}. Reload the page to see it.\n`;
280
421
  }
281
422
 
282
423
  const out = (text) => new Promise((r) => process.stdout.write(text, r));
@@ -416,7 +557,7 @@ async function main() {
416
557
  process.exit(r.exit);
417
558
  }
418
559
  if (argv[0] === "setup-checker") {
419
- const py = setupChecker();
560
+ const py = await setupChecker();
420
561
  return out(`checker ready: ${py}\n`);
421
562
  }
422
563
  if (argv[0] === "setup-rewriter") {
@@ -460,9 +601,8 @@ async function main() {
460
601
  print: (line) => process.stdout.write(line + "\n"),
461
602
  server: { healthy, openWork: async () => { const r = await req({ method: "GET", path: "/cli/open-work", timeout: 3000 }); return { status: r.status, body: json(r.text) }; } },
462
603
  stop: stopServer,
463
- runGetSh: (base) => runGetShDefault(base),
604
+ runGetSh: (base, opts) => runGetShDefault(base, process.env, opts),
464
605
  launcherVersion: (route, root) => (route === "npm" ? versionOf(process.execPath, [path.join(root, "src", "cli.js"), "--version"]) : versionOf(path.join(home, ".local", "bin", BIN), ["--version"])),
465
- run: (cmd, args) => spawnSync(cmd, args, { stdio: ["ignore", "inherit", "inherit"] }).status ?? 1,
466
606
  });
467
607
  process.exit(code);
468
608
  }
@@ -557,8 +697,8 @@ async function main() {
557
697
  if (a.summary) return out(JSON.stringify({ ok: !!j.ok, flags: (j.flags || []).map((f) => ({ code: f.code, severity: f.severity })), ...(j.checks?.notice ? { checks_notice: j.checks.notice } : {}) }) + "\n");
558
698
  return out(JSON.stringify(j, null, 2) + "\n");
559
699
  } else if (a.cmd === "stop") {
560
- await stopServer();
561
- return out("stopped\n");
700
+ const res = await stopServer();
701
+ return out(res?.staleRemoved ? "no running server (stale record removed)\n" : res?.stopped ? "stopped\n" : "no server was running\n");
562
702
  } else throw new Error(`unknown command ${a.cmd}\n${HELP}`);
563
703
  }
564
704
 
package/src/doctor.js CHANGED
@@ -90,6 +90,15 @@ const run = (cmd, args, env) => {
90
90
  }
91
91
  };
92
92
 
93
+ /** Parse the target cli.js path from an installer-written launcher (single- or double-quoted). */
94
+ export function parseLauncherTarget(text) {
95
+ const sq = text.match(/exec node '((?:[^']|'\\'')+)'/);
96
+ if (sq) return sq[1].replace(/'\\''/g, "'");
97
+ const dq = text.match(/exec node "([^"]+)"/);
98
+ if (dq) return dq[1];
99
+ return null;
100
+ }
101
+
93
102
  /**
94
103
  * @param {object} o home, appRoot, env, verbose: where to look. probes: injectable `defaultBrowser()` and `freeBytes(dir)` for tests.
95
104
  * @returns {Promise<{schema:string, ok:boolean, version:string|null, checks:Array<{status:string,name:string,detail:string,fix?:string,required:boolean}>}>}
@@ -149,8 +158,8 @@ export async function runDoctor(o = {}) {
149
158
  const found = which(BIN);
150
159
  const launcher = found || path.join(home, ".local", "bin", BIN);
151
160
  if (fs.existsSync(launcher)) {
152
- const target = ((fs.readFileSync(launcher, "utf8").match(/exec node "([^"]+)"/)) || [])[1];
153
- // the installer writes `exec node "<app>/src/cli.js"`; any other launcher (a symlink, npm link) is judged by whether it runs
161
+ const target = parseLauncherTarget(fs.readFileSync(launcher, "utf8"));
162
+ // the installer writes `exec node '<app>/src/cli.js'` (or double-quoted in older versions); any other launcher (a symlink, npm link) is judged by whether it runs
154
163
  add(!target || fs.existsSync(target) ? "OK" : "MISSING", "launcher", `${launcher}${target ? ` -> ${target}` : " (custom launcher)"}`, { required: true, fix: REINSTALL_FIX });
155
164
  const v = run(launcher, ["--version"], env);
156
165
  add(v ? "OK" : "MISSING", `${BIN} --version`, v || "the launcher did not run", { required: true, fix: REINSTALL_FIX });
@@ -166,7 +175,7 @@ export async function runDoctor(o = {}) {
166
175
  };
167
176
  const runs = (p) => {
168
177
  try {
169
- const t = (fs.readFileSync(p, "utf8").slice(0, 400).match(/exec node "([^"]+)"/) || [])[1]; // the installer's 2-line launcher
178
+ const t = parseLauncherTarget(fs.readFileSync(p, "utf8").slice(0, 400)); // the installer's 2-line launcher
170
179
  return real(t || p);
171
180
  } catch {
172
181
  return real(p);
package/src/docx.js CHANGED
@@ -4,6 +4,9 @@ import { readBudget, tooMany, walkXml } from "./xml.js";
4
4
  import { OCCURRENCE, affords, bigramCodes, diceOf, hitsOf, normPaid, normText, searchCost, similarity, spend, textHash, weigh, weightOf } from "./text.js";
5
5
  import { openZip } from "./zip.js";
6
6
  import { summarizeRuns } from "./fmt.js";
7
+ import { HAS_NON_PLAIN_WS, cuttable, normOccurrences } from "./pptx.js";
8
+
9
+ export { cuttable };
7
10
 
8
11
  // One classifier for `protected`: the engine's own (default deny), bundled for Node. The batch, `apply` and the editor agree because they ask the same code.
9
12
  const { protectReasons } = await import(pathToFileURL(engineNodePath()).href);
@@ -74,7 +77,9 @@ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget()
74
77
  let tableCount = 0; // 1-based order among top-level tables (the label users read), unlike body_path[0] which counts every body item
75
78
  let cellK = 0;
76
79
  let cur = null;
77
- let delDepth = 0;
80
+ const revStack = [];
81
+ let mcFallbackDepth = 0;
82
+ const visible = () => !revStack.includes("del") && (revStack.length === 0 || revStack[revStack.length - 1] !== "moveFrom");
78
83
  let inT = false;
79
84
  let inInstr = false;
80
85
  let inBody = false;
@@ -87,9 +92,14 @@ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget()
87
92
  walkXml(xml, {
88
93
  onOpen(name, attrs) {
89
94
  stack.push(name);
95
+ if (name === "mc:Fallback") {
96
+ mcFallbackDepth++;
97
+ return;
98
+ }
99
+ if (mcFallbackDepth > 0) return;
90
100
  if (name === "w:body") inBody = true;
91
101
  if (!inBody) return;
92
- if (name === "w:tbl") {
102
+ if (name === "w:tbl" && pDepth === 0) {
93
103
  tblDepth++;
94
104
  if (tblDepth === 1) {
95
105
  bodyIndex++;
@@ -98,16 +108,15 @@ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget()
98
108
  info.row = -1;
99
109
  info.col = -1;
100
110
  }
101
- } else if (name === "w:tr" && tblDepth === 1) {
111
+ } else if (name === "w:tr" && tblDepth === 1 && pDepth === 0) {
102
112
  info.row++;
103
113
  info.col = -1;
104
- } else if (name === "w:tc" && tblDepth === 1) {
114
+ } else if (name === "w:tc" && tblDepth === 1 && pDepth === 0) {
105
115
  info.col++;
106
116
  cellK = 0;
107
117
  } else if (name === "w:p") {
108
118
  pDepth++;
109
- const nested = stack.slice(0, -1).includes("w:p"); // a text box paragraph inside a paragraph
110
- if (nested) return;
119
+ if (pDepth > 1) return;
111
120
  let path = null;
112
121
  let tableNo = null;
113
122
  if (tblDepth === 0) {
@@ -121,15 +130,15 @@ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget()
121
130
  cur = { ordinal: paragraphs.length, path, tableNo, paraId: /^[0-9A-Fa-f]{8}$/.test(attrs["w14:paraId"] || "") ? attrs["w14:paraId"].toUpperCase() : null, text: "", instr: "", protected: false, style: null, align: "", runs: [] };
122
131
  if (detail) (Object.assign(cur, { spans: [], list: null, ppr: [], sect: false, loose: false }), (marks = new Set()), (pprTop = null));
123
132
  paragraphs.push(cur);
124
- } else if (cur) {
133
+ } else if (cur && pDepth === 1) {
125
134
  if (detail) {
126
135
  const n = stack.length;
127
- if (pDepth === 1 && stack[n - 2] === "w:pPr" && stack[n - 3] === "w:p") {
136
+ if (stack[n - 2] === "w:pPr" && stack[n - 3] === "w:p") {
128
137
  pprTop = name;
129
138
  if (name === "w:sectPr") cur.sect = true;
130
139
  }
131
140
  if (pprTop && !PPR_OWN.has(pprTop)) cur.ppr.push(written(name, attrs));
132
- else if (pprTop === "w:numPr" && pDepth === 1 && stack[n - 2] === "w:numPr" && (name === "w:numId" || name === "w:ilvl")) (cur.list ||= { numId: "", ilvl: 0 })[name === "w:numId" ? "numId" : "ilvl"] = name === "w:numId" ? attrs["w:val"] || "" : Number(attrs["w:val"]) || 0;
141
+ else if (pprTop === "w:numPr" && stack[n - 2] === "w:numPr" && (name === "w:numId" || name === "w:ilvl")) (cur.list ||= { numId: "", ilvl: 0 })[name === "w:numId" ? "numId" : "ilvl"] = name === "w:numId" ? attrs["w:val"] || "" : Number(attrs["w:val"]) || 0;
133
142
  const mark = RANGE.exec(name);
134
143
  if (mark && "w:id" in attrs) {
135
144
  const key = `${mark[1]} ${attrs["w:id"]}`;
@@ -147,35 +156,33 @@ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget()
147
156
  else if (name === "w:r" && !run) run = { text: false, p: {}, at: cur.text.length, depth: stack.length, set: {}, rest: [] };
148
157
  else if (name === "w:rPr") rprDepth++;
149
158
  else if (run && rprDepth === 1 && stack.includes("w:rPr")) runProp(run.p, name, attrs);
150
- if (name === "w:del") delDepth++;
151
- else if (name === "w:t" && delDepth === 0) (inT = true, run && (run.text = true));
159
+ if (name === "w:del") revStack.push("del");
160
+ else if (name === "w:moveFrom") revStack.push("moveFrom");
161
+ else if (name === "w:ins") revStack.push("ins");
162
+ else if (name === "w:t" && visible()) (inT = true, run && (run.text = true));
152
163
  else if (name === "w:instrText") inInstr = true;
153
- else if (name === "w:tab" && delDepth === 0) cur.text += "\t";
164
+ else if (name === "w:tab" && visible()) cur.text += "\t";
154
165
  else if (name === "w:br" || name === "w:cr") {
155
- if (delDepth === 0 && attrs["w:type"] !== "page" && attrs["w:type"] !== "column") cur.text += "\n";
166
+ if (visible() && attrs["w:type"] !== "page" && attrs["w:type"] !== "column") cur.text += "\n";
156
167
  } else if (name === "w:drawing" || name === "w:pict") {
157
- if (delDepth === 0) cur.text += OBJ;
168
+ if (visible()) cur.text += OBJ;
158
169
  }
159
170
  }
160
171
  },
161
172
  onText(t) {
162
- if (cur && inT) cur.text += t;
163
- else if (cur && inInstr && cur.instr.length < 200) cur.instr += t;
173
+ if (mcFallbackDepth > 0) return;
174
+ if (cur && pDepth === 1 && inT) cur.text += t;
175
+ else if (cur && pDepth === 1 && inInstr && cur.instr.length < 200) cur.instr += t;
164
176
  },
165
177
  onClose(name) {
166
178
  stack.pop();
167
- if (name === "w:t") inT = false;
168
- else if (name === "w:instrText") inInstr = false;
169
- else if (name === "w:del") delDepth = Math.max(0, delDepth - 1);
170
- else if (name === "w:tbl") tblDepth--;
171
- else if (name === "w:rPr") rprDepth = Math.max(0, rprDepth - 1);
172
- else if (name === "w:r" && run) {
173
- if (run.text && cur) cur.runs.push(run.p);
174
- if (detail && cur && cur.text.length > run.at) cur.spans.push({ n: cur.text.length - run.at, set: run.set, rest: run.rest.join(" ") });
175
- run = null;
176
- } else if (name === "w:p") {
179
+ if (mcFallbackDepth > 0) {
180
+ if (name === "mc:Fallback") mcFallbackDepth--;
181
+ return;
182
+ }
183
+ if (name === "w:p") {
177
184
  pDepth--;
178
- if (stack.includes("w:p")) return;
185
+ if (pDepth > 0) return;
179
186
  if (cur) (cur.fmt = summarizeRuns(cur.runs), delete cur.runs);
180
187
  if (cur && detail) {
181
188
  if (cur.spans.reduce((n, x) => n + x.n, 0) !== cur.text.length) cur.spans = null;
@@ -183,8 +190,21 @@ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget()
183
190
  cur.loose = marks.size > 0;
184
191
  }
185
192
  cur = null;
193
+ revStack.length = 0;
194
+ } else if (name === "w:tbl" && pDepth === 0) {
195
+ tblDepth--;
196
+ } else if (cur && pDepth === 1) {
197
+ if (name === "w:t") inT = false;
198
+ else if (name === "w:instrText") inInstr = false;
199
+ else if (name === "w:del" || name === "w:moveFrom" || name === "w:ins") revStack.pop();
200
+ else if (name === "w:rPr") rprDepth = Math.max(0, rprDepth - 1);
201
+ else if (name === "w:r" && run) {
202
+ if (run.text && cur) cur.runs.push(run.p);
203
+ if (detail && cur && cur.text.length > run.at) cur.spans.push({ n: cur.text.length - run.at, set: run.set, rest: run.rest.join(" ") });
204
+ run = null;
205
+ }
206
+ if (detail && name === pprTop && stack[stack.length - 1] === "w:pPr") pprTop = null;
186
207
  }
187
- if (detail && name === pprTop && stack[stack.length - 1] === "w:pPr") pprTop = null;
188
208
  },
189
209
  }, budget);
190
210
  for (const p of paragraphs) p.hash = textHash(p.text);
@@ -587,23 +607,6 @@ function normHead(text, n) {
587
607
  // Once the window holds at least as many normalized characters as the prefix, it answers for the whole text; else it doubles.
588
608
  // Every character read (the walk to a cut and the window normalized) is spent from the budget of the request (src/text.js): null when it
589
609
  // runs out. An outside review: CJK text has no character below U+0300, every window grew to the whole paragraph, 3.65 s for one anchor.
590
- const MARK = /\p{M}/u;
591
- /**
592
- * May a window start (or end) right before text[i]? Yes below U+0300 (no such character joins the one before it under NFC); never inside a
593
- * surrogate pair or before a combining mark; otherwise when the character normalizes the same after the three before it as on its own
594
- * (the longest chain of starters that compose, Hangul L V T and its like, is three: test/parse-limits.test.js checks every code point).
595
- * Exported with the two window readers for that test.
596
- */
597
- export function cuttable(text, i) {
598
- if (i <= 0 || i >= text.length) return true;
599
- const c = text.charCodeAt(i);
600
- if (c < 0x300) return true;
601
- if (c >= 0xdc00 && c <= 0xdfff) return false;
602
- const ch = String.fromCodePoint(text.codePointAt(i));
603
- if (MARK.test(ch)) return false;
604
- const before = text.slice(Math.max(0, i - 3), i);
605
- return (before + ch).normalize("NFC") === before.normalize("NFC") + ch.normalize("NFC");
606
- }
607
610
  /** The nearest place at or before (dir -1) or after (dir 1) i where a window may be cut, each character walked spent; null past the budget. */
608
611
  function cutNear(text, i, dir) {
609
612
  let n = 0;
@@ -635,6 +638,15 @@ export function startsWithNorm(text, from, suffix) {
635
638
  * `nth` is the occurrence index remembered at capture. Returns {at: the offset, k: which occurrence it is}, or {at: -1, k: null} when the
636
639
  * text does not occur or the budget ran out.
637
640
  */
641
+ /** does the text around the occurrence at `at` (of `qLen` UTF-16 units) read as prefix and suffix (normalized, as an anchor keeps them)? null: the budget ran out */
642
+ function fitsAt(text, qLen, at, { prefix = "", suffix = "" }) {
643
+ prefix = String(prefix || "").slice(-CTX); // what an anchor keeps (durableFor); a longer one from elsewhere is read as its last CTX
644
+ suffix = String(suffix || "").slice(0, CTX);
645
+ const pre = !prefix || endsWithNorm(text, at, prefix);
646
+ if (pre === null) return null;
647
+ const suf = pre && (!suffix || startsWithNorm(text, at + qLen, suffix));
648
+ return suf === null ? null : !!(pre && suf);
649
+ }
638
650
  function pickOccurrence(p, q, opts = {}) {
639
651
  const text = p.text;
640
652
  // the remembered occurrence first, listing only the occurrences up to it: when its surroundings still fit, it is the answer every rule
@@ -649,29 +661,45 @@ function pickOccurrence(p, q, opts = {}) {
649
661
  const head = occurrences(p, q, remembered + 1);
650
662
  if (head === null) return { at: -1, k: null };
651
663
  if (head.length === remembered + 1) {
652
- const ok = fitsAt(text, q, head[remembered], opts);
664
+ const ok = fitsAt(text, q.length, head[remembered], opts);
653
665
  if (ok === null) return { at: -1, k: null };
654
666
  if (ok) return { at: head[remembered], k: remembered };
655
667
  }
656
668
  }
657
669
  const occ = occurrencesIn(p, q);
658
- const at = occ?.length ? pickFrom(text, q, occ, { ...opts, nth: remembered }) : -1;
670
+ const at = occ?.length ? pickFrom(text, q.length, occ, { ...opts, nth: remembered }) : -1;
659
671
  return at < 0 ? { at: -1, k: null } : { at, k: occ.indexOf(at) };
660
672
  }
661
- /** does the text around the occurrence at `at` read as prefix and suffix (normalized, as an anchor keeps them)? null: the budget ran out */
662
- function fitsAt(text, q, at, { prefix = "", suffix = "" }) {
663
- prefix = String(prefix || "").slice(-CTX); // what an anchor keeps (durableFor); a longer one from elsewhere is read as its last CTX
664
- suffix = String(suffix || "").slice(0, CTX);
665
- const pre = !prefix || endsWithNorm(text, at, prefix);
666
- if (pre === null) return null;
667
- const suf = pre && (!suffix || startsWithNorm(text, at + q.length, suffix));
668
- return suf === null ? null : !!(pre && suf);
673
+ function pickNormOccurrence(p, q, opts = {}) {
674
+ const occ = normOccurrences(p, q);
675
+ if (!occ?.length) return { at: -1, len: 0, k: null };
676
+ let { nth = null } = opts;
677
+ if (!(Number.isInteger(nth) && nth >= 0 && nth < occ.length)) nth = null;
678
+ const prefix = String(opts.prefix || "").slice(-CTX);
679
+ const suffix = String(opts.suffix || "").slice(0, CTX);
680
+ if ((prefix || suffix) && occ.length > 1 && !spend(occ.length * 2 * (prefix.length + suffix.length + 16))) return { at: -1, len: 0, k: null };
681
+ const guarded = [];
682
+ for (let i = 0; i < occ.length; i++) {
683
+ const o = occ[i];
684
+ const ok = fitsAt(p.text, o.end - o.start, o.start, { prefix, suffix });
685
+ if (ok === null) return { at: -1, len: 0, k: null };
686
+ if (ok) guarded.push(i);
687
+ }
688
+ let idx;
689
+ if (guarded.length === 1) idx = guarded[0];
690
+ else if (nth != null && guarded.includes(nth)) idx = nth;
691
+ else if (Number.isInteger(opts.hintStart) && guarded.length) {
692
+ idx = guarded.reduce((best, i) => (Math.abs(occ[i].start - opts.hintStart) < Math.abs(occ[best].start - opts.hintStart) ? i : best), guarded[0]);
693
+ } else if (guarded.length) idx = guarded[0];
694
+ else idx = nth != null ? nth : 0;
695
+ const hit = occ[idx];
696
+ return { at: hit.start, len: hit.end - hit.start, k: idx };
669
697
  }
670
- function pickFrom(text, q, occ, { prefix = "", suffix = "", nth = null }) {
698
+ function pickFrom(text, qLen, occ, { prefix = "", suffix = "", nth = null, hintStart = null }) {
671
699
  if (!(Number.isInteger(nth) && nth >= 0 && nth < occ.length)) nth = null;
672
700
  prefix = String(prefix || "").slice(-CTX);
673
701
  suffix = String(suffix || "").slice(0, CTX);
674
- const fits = (at) => fitsAt(text, q, at, { prefix, suffix });
702
+ const fits = (at) => fitsAt(text, qLen, at, { prefix, suffix });
675
703
  // each occurrence reads about CTX characters on each side, out of the budget of the request (src/text.js): a paragraph of 100,000
676
704
  // "a" read the whole text before every one of them and took 10 seconds in an outside review. Past the budget, no offset.
677
705
  if ((prefix || suffix) && occ.length > 1 && !spend(occ.length * 2 * (prefix.length + suffix.length + 16))) return -1;
@@ -683,6 +711,9 @@ function pickFrom(text, q, occ, { prefix = "", suffix = "", nth = null }) {
683
711
  }
684
712
  if (guarded.length === 1) return guarded[0];
685
713
  if (nth != null && nth < occ.length && guarded.includes(occ[nth])) return occ[nth];
714
+ if (Number.isInteger(hintStart) && guarded.length) {
715
+ return guarded.reduce((best, at) => (Math.abs(at - hintStart) < Math.abs(best - hintStart) ? at : best), guarded[0]);
716
+ }
686
717
  if (guarded.length) return guarded[0];
687
718
  return nth != null && nth < occ.length ? occ[nth] : occ[0];
688
719
  }
@@ -697,12 +728,31 @@ function durableFor(paragraphs, p, engineTop, ctx, loc = {}) {
697
728
  let nth = null;
698
729
  if (q) {
699
730
  // the engine's own offsets when they point at the quote; else the occurrence whose surroundings match prefix and suffix
700
- const own = Number.isInteger(loc.start) && Number.isInteger(loc.end) && p.text.slice(loc.start, loc.end) === ctx.quote;
701
- const k = own ? occurrencesIn(p, ctx.quote)?.indexOf(loc.start) ?? -1 : -1;
702
- const picked = own ? { at: loc.start, k: k >= 0 ? k : null } : pickOccurrence(p, ctx.quote, { prefix, suffix });
731
+ const hintStart = Number.isInteger(loc.start) ? loc.start : null;
732
+ const preferNorm = (ctx.quote !== q && ctx.quote.trim() !== q) || (q.includes(" ") && HAS_NON_PLAIN_WS.test(p.text));
733
+ const own = hintStart !== null && Number.isInteger(loc.end) && p.text.slice(loc.start, loc.end) === ctx.quote;
734
+ let picked;
735
+ if (own) {
736
+ let k = -1;
737
+ if (preferNorm) {
738
+ const nocc = normOccurrences(p, q);
739
+ if (nocc) {
740
+ k = nocc.findIndex((o) => o.start === loc.start && o.end === loc.end);
741
+ if (k < 0) k = nocc.findIndex((o) => o.start >= loc.start && o.end <= loc.end);
742
+ }
743
+ } else {
744
+ k = occurrencesIn(p, ctx.quote)?.indexOf(loc.start) ?? -1;
745
+ }
746
+ picked = { at: loc.start, k: k >= 0 ? k : null, len: ctx.quote.length };
747
+ } else if (preferNorm) {
748
+ picked = pickNormOccurrence(p, q, { prefix, suffix, hintStart });
749
+ } else {
750
+ picked = { ...pickOccurrence(p, ctx.quote, { prefix, suffix, hintStart }), len: ctx.quote.length };
751
+ if (picked.at < 0) picked = pickNormOccurrence(p, q, { prefix, suffix, hintStart });
752
+ }
703
753
  if (picked.at >= 0) {
704
754
  start = picked.at;
705
- end = picked.at + ctx.quote.length;
755
+ end = picked.at + picked.len;
706
756
  nth = picked.k;
707
757
  }
708
758
  }
@@ -739,10 +789,27 @@ export function resolveDurable(paragraphs, d) {
739
789
  const q = d.quote || "";
740
790
  let start = null;
741
791
  let end = null;
742
- const { at, k } = q ? pickOccurrence(p, q, { prefix: d.prefix, suffix: d.suffix, nth: d.nth }) : { at: -1, k: null };
792
+ const ctxOpts = { prefix: d.prefix ?? d.before ?? "", suffix: d.suffix ?? d.after ?? "", nth: d.nth ?? d.occ ?? null };
793
+ const preferNorm = !!q && q === normText(q) && q.includes(" ") && HAS_NON_PLAIN_WS.test(p.text);
794
+ let at = -1;
795
+ let len = q.length;
796
+ let k = null;
797
+ if (q) {
798
+ if (preferNorm) {
799
+ const picked = pickNormOccurrence(p, q, ctxOpts);
800
+ if (picked.at >= 0) ({ at, len, k } = picked);
801
+ else ({ at, k } = pickOccurrence(p, q, ctxOpts));
802
+ } else {
803
+ ({ at, k } = pickOccurrence(p, q, ctxOpts));
804
+ if (at < 0 || (Number.isInteger(ctxOpts.nth) && ctxOpts.nth > 0 && k !== ctxOpts.nth)) {
805
+ const picked = pickNormOccurrence(p, q, ctxOpts);
806
+ if (picked.at >= 0) ({ at, len, k } = picked);
807
+ }
808
+ }
809
+ }
743
810
  if (at >= 0) {
744
811
  start = at;
745
- end = at + q.length;
812
+ end = at + len;
746
813
  }
747
814
  return { method: hit.method, score: hit.score, ordinal: p.ordinal, path: p.path, tableNo: p.tableNo ?? null, paraId: p.paraId, textHash: p.hash, protected: !!p.protected, ...(p.protected ? { protectInfo: protectInfo(p) } : {}), dupCount: indexOf(paragraphs).byHash.get(p.hash)?.length ?? 0, nth: at >= 0 ? k : null, prevHash: paragraphs[p.ordinal - 1]?.hash || null, nextHash: paragraphs[p.ordinal + 1]?.hash || null, text: normHead(p.text, PARA_TEXT_KEEP) ?? "", start, end, engineDelta: d.engineDelta || 0 };
748
815
  }