onionskin-cli 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +98 -0
- package/README.md +4 -4
- package/package.json +1 -1
- package/scripts/checker/check.py +24 -5
- package/scripts/checker/requirements.txt +482 -5
- package/scripts/requirements.txt +482 -5
- package/scripts/rewriter/apple/main.swift +1 -1
- package/scripts/rewriter/apple/onionskin-fm +0 -0
- package/scripts/rewriter/apple/onionskin-fm.json +1 -1
- package/skills/onionskin/SKILL.md +4 -4
- package/skills/onionskin/reference/ops.md +1 -1
- package/src/agent-runner.js +94 -15
- package/src/checker.js +238 -33
- package/src/cli.js +173 -33
- package/src/doctor.js +12 -3
- package/src/docx.js +130 -63
- package/src/fsutil.js +13 -5
- package/src/history.js +11 -2
- package/src/html-render.js +74 -15
- package/src/html-utf8.js +1 -1
- package/src/html.js +3 -1
- package/src/merge-package.js +82 -26
- package/src/offload.js +25 -6
- package/src/ops/docx-table.js +5 -3
- package/src/ops/pptx.js +8 -2
- package/src/ops/xlsx.js +114 -8
- package/src/package.js +17 -2
- package/src/pptx.js +244 -8
- package/src/question.js +10 -3
- package/src/quick.js +1 -1
- package/src/redline-check.js +12 -2
- package/src/rewriter.js +38 -8
- package/src/server-entry.js +4 -3
- package/src/server.js +97 -28
- package/src/session.js +13 -2
- package/src/setup.js +20 -2
- package/src/update.js +248 -19
- package/src/voice-guide.js +12 -7
- package/src/voice-learn.js +11 -1
- package/src/voice-store.js +11 -4
- package/src/voice.js +2 -1
- package/src/xlsx.js +25 -5
- package/src/zip-write.js +8 -4
- package/src/zip.js +64 -13
- package/web/dist/assets/{index-DytKkMdi.js → index-CjsTf5gj.js} +108 -106
- package/web/dist/assets/index-DnbLtHhG.css +1 -0
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-DkNx0LSH.css +0 -1
package/src/cli.js
CHANGED
|
@@ -96,6 +96,7 @@ function req(options, body) {
|
|
|
96
96
|
res.on("close", () => !res.complete && reject(new Error("server connection closed")));
|
|
97
97
|
});
|
|
98
98
|
r.on("error", reject);
|
|
99
|
+
if (options.timeout) r.on("timeout", () => r.destroy(new Error("request timed out")));
|
|
99
100
|
if (options.signal) options.signal.addEventListener("abort", () => r.destroy(new Error("aborted")), { once: true });
|
|
100
101
|
// `deadline` (ms): the whole request, answer included, or it ends with `deadlineError`
|
|
101
102
|
if (options.deadline) {
|
|
@@ -120,11 +121,12 @@ const json = (t) => {
|
|
|
120
121
|
}
|
|
121
122
|
};
|
|
122
123
|
|
|
123
|
-
async function healthy() {
|
|
124
|
+
async function healthy({ reportMismatch = false } = {}) {
|
|
124
125
|
try {
|
|
125
126
|
const r = await req({ method: "GET", path: "/health", timeout: 1500 });
|
|
126
127
|
const j = json(r.text);
|
|
127
|
-
|
|
128
|
+
if (r.status === 200 && j?.protocol === SERVER_PROTOCOL) return true;
|
|
129
|
+
return reportMismatch && r.status === 200 ? "incompatible" : false;
|
|
128
130
|
} catch {
|
|
129
131
|
return false;
|
|
130
132
|
}
|
|
@@ -162,7 +164,14 @@ async function removeStaleSocket() {
|
|
|
162
164
|
continue;
|
|
163
165
|
}
|
|
164
166
|
}
|
|
165
|
-
if (!Number.isInteger(pid))
|
|
167
|
+
if (!Number.isInteger(pid) || pid <= 0) {
|
|
168
|
+
if (f.endsWith("start.lock")) {
|
|
169
|
+
try {
|
|
170
|
+
if (Date.now() - fs.statSync(f).mtimeMs <= 2000) return false;
|
|
171
|
+
} catch {}
|
|
172
|
+
}
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
166
175
|
try {
|
|
167
176
|
process.kill(pid, 0);
|
|
168
177
|
return false; // that process is alive: it may be starting up on this socket
|
|
@@ -178,48 +187,180 @@ async function removeStaleSocket() {
|
|
|
178
187
|
}
|
|
179
188
|
}
|
|
180
189
|
|
|
181
|
-
/**
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
190
|
+
/** True when `pid` exists (EPERM also means it exists, owned by another user). */
|
|
191
|
+
function pidAlive(pid) {
|
|
192
|
+
try {
|
|
193
|
+
process.kill(pid, 0);
|
|
194
|
+
return true;
|
|
195
|
+
} catch (e) {
|
|
196
|
+
return e.code === "EPERM";
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** True when a Unix socket or TCP endpoint accepts a connection within `timeoutMs`. */
|
|
201
|
+
function canConnect(opts, timeoutMs = 400) {
|
|
202
|
+
return new Promise((resolve) => {
|
|
203
|
+
let done = false;
|
|
204
|
+
const finish = (ok) => {
|
|
205
|
+
if (done) return;
|
|
206
|
+
done = true;
|
|
207
|
+
clearTimeout(t);
|
|
188
208
|
try {
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
return true;
|
|
193
|
-
}
|
|
209
|
+
s.destroy();
|
|
210
|
+
} catch {}
|
|
211
|
+
resolve(ok);
|
|
194
212
|
};
|
|
195
|
-
|
|
213
|
+
const s = net.createConnection(opts);
|
|
214
|
+
const t = setTimeout(() => finish(false), timeoutMs);
|
|
215
|
+
t.unref?.();
|
|
216
|
+
s.once("connect", () => finish(true));
|
|
217
|
+
s.once("error", () => finish(false));
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/** True when `pid`'s command line looks like an Onionskin server process. */
|
|
222
|
+
function looksLikeOnionskinPid(pid) {
|
|
223
|
+
const r = spawnSync("ps", ["-o", "command=", "-p", String(pid)], { encoding: "utf8", timeout: 1000 });
|
|
224
|
+
if (r.status !== 0) return false;
|
|
225
|
+
return /server-entry\.js|onionskin/i.test((r.stdout || "").trim());
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Inspects server.json / start.lock when /health did not answer.
|
|
230
|
+
* Treats a recorded pid as the server ONLY when identity is confirmed (the socket or recorded port accepts
|
|
231
|
+
* connections, or the process command matches Onionskin); otherwise removes the stale record(s).
|
|
232
|
+
*/
|
|
233
|
+
async function inspectRecordedServer() {
|
|
234
|
+
let staleRemoved = false;
|
|
235
|
+
let sockAccepts = null;
|
|
236
|
+
const checkSock = async () => {
|
|
237
|
+
if (sockAccepts === null) sockAccepts = await canConnect({ path: socketPath() });
|
|
238
|
+
return sockAccepts;
|
|
239
|
+
};
|
|
240
|
+
for (const f of [serverRecordPath(), path.join(stateDir(), "start.lock")]) {
|
|
241
|
+
let raw;
|
|
242
|
+
try {
|
|
243
|
+
raw = fs.readFileSync(f, "utf8");
|
|
244
|
+
} catch {
|
|
245
|
+
continue;
|
|
246
|
+
}
|
|
247
|
+
let pid = null;
|
|
248
|
+
let port = null;
|
|
249
|
+
try {
|
|
250
|
+
const obj = JSON.parse(raw);
|
|
251
|
+
pid = Number.parseInt(obj?.pid ?? raw, 10);
|
|
252
|
+
if (Number.isInteger(obj?.port) && obj.port > 0 && obj.port <= 65535) port = obj.port;
|
|
253
|
+
} catch {
|
|
254
|
+
pid = Number.parseInt(raw, 10);
|
|
255
|
+
}
|
|
256
|
+
if (Number.isInteger(pid) && pid > 0 && pidAlive(pid)) {
|
|
257
|
+
const confirmed =
|
|
258
|
+
(await checkSock()) ||
|
|
259
|
+
(port !== null && (await canConnect({ host: "127.0.0.1", port }))) ||
|
|
260
|
+
looksLikeOnionskinPid(pid);
|
|
261
|
+
if (confirmed) return { pid, staleRemoved };
|
|
262
|
+
// If a socket file is still present alongside a live PID, leave both untouched (a server may be starting up; test/stop.test.js)
|
|
263
|
+
// while still returning pid: null so `stop` does not wait 5s or report an unrelated process as stuck.
|
|
264
|
+
if (fs.existsSync(socketPath())) continue;
|
|
265
|
+
}
|
|
266
|
+
try {
|
|
267
|
+
if (f.endsWith("start.lock") && Date.now() - fs.statSync(f).mtimeMs <= 2000) continue;
|
|
268
|
+
fs.unlinkSync(f);
|
|
269
|
+
staleRemoved = true;
|
|
270
|
+
} catch {}
|
|
271
|
+
}
|
|
272
|
+
return { pid: null, staleRemoved };
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/** `stop` (and `update`): asks a running server to stop, waits for its process to end, then removes a leftover socket. */
|
|
276
|
+
async function stopServer({ waitMs = 5000 } = {}) {
|
|
277
|
+
let pid = null;
|
|
278
|
+
let staleRemoved = false;
|
|
279
|
+
let shouldStop = await healthy();
|
|
280
|
+
if (shouldStop) {
|
|
281
|
+
pid = json((await req({ method: "GET", path: "/health", timeout: 1500 }).catch(() => ({ text: "" }))).text).pid;
|
|
282
|
+
} else {
|
|
283
|
+
const r = await req({ method: "GET", path: "/health", timeout: 1500 }).catch(() => null);
|
|
284
|
+
const j = r?.status === 200 ? json(r.text) : null;
|
|
285
|
+
if (j && Number.isInteger(j.pid) && j.pid > 0) {
|
|
286
|
+
pid = j.pid;
|
|
287
|
+
shouldStop = true;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
if (!Number.isInteger(pid) || pid <= 0) {
|
|
291
|
+
const rec = await inspectRecordedServer();
|
|
292
|
+
staleRemoved = rec.staleRemoved;
|
|
293
|
+
if (rec.pid) {
|
|
294
|
+
pid = rec.pid;
|
|
295
|
+
shouldStop = true;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
if (shouldStop) {
|
|
299
|
+
await req({ method: "POST", path: "/cli/stop", timeout: 2000 }, {}).catch(() => {});
|
|
300
|
+
// the server answers first and exits a moment later: "stopped" is printed only once its process is gone (waits at most waitMs)
|
|
301
|
+
const gone = () => !pidAlive(pid);
|
|
302
|
+
const steps = Math.max(1, Math.ceil(waitMs / 50));
|
|
303
|
+
for (let i = 0; i < steps && Number.isInteger(pid) && pid > 0 && !gone(); i++) await new Promise((r) => setTimeout(r, 50));
|
|
304
|
+
if (Number.isInteger(pid) && pid > 0 && !gone()) {
|
|
305
|
+
throw new Error(`server (pid ${pid}) did not stop within ${Math.round(waitMs / 1000)}s`);
|
|
306
|
+
}
|
|
196
307
|
}
|
|
197
308
|
await removeStaleSocket(); // a socket file nothing answers on is no use to anyone, and `doctor` would keep reporting it
|
|
309
|
+
return { staleRemoved, stopped: shouldStop };
|
|
198
310
|
}
|
|
199
311
|
|
|
200
312
|
/** Starts the detached server and waits (event-based) for its "ready" line. */
|
|
201
313
|
async function ensureServer() {
|
|
202
314
|
ensureStateDir();
|
|
203
315
|
if (await healthy()) return;
|
|
204
|
-
const child = spawn(process.execPath, [path.join(here, "server-entry.js")], { detached: true, stdio: ["ignore", "pipe", "
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
316
|
+
const child = spawn(process.execPath, [path.join(here, "server-entry.js")], { detached: true, stdio: ["ignore", "pipe", "pipe"], env: process.env });
|
|
317
|
+
const errChunks = [];
|
|
318
|
+
let errLen = 0;
|
|
319
|
+
child.stderr.on("data", (d) => {
|
|
320
|
+
if (errLen >= 4096) return;
|
|
321
|
+
const slice = d.length > 4096 - errLen ? d.subarray(0, 4096 - errLen) : d;
|
|
322
|
+
errChunks.push(slice);
|
|
323
|
+
errLen += slice.length;
|
|
210
324
|
});
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
325
|
+
const detail = () => {
|
|
326
|
+
const s = Buffer.concat(errChunks).toString("utf8").trim();
|
|
327
|
+
return s ? `: ${s}` : "";
|
|
328
|
+
};
|
|
329
|
+
try {
|
|
330
|
+
await new Promise((resolve, reject) => {
|
|
331
|
+
const to = setTimeout(() => reject(new Error(`could not start the ${PRODUCT.name} server${detail()}`)), 15000);
|
|
332
|
+
child.stdout.once("data", () => (clearTimeout(to), resolve()));
|
|
333
|
+
child.once("exit", () => (clearTimeout(to), resolve())); // lost a race to another server: it is healthy now
|
|
334
|
+
child.once("error", (e) => (clearTimeout(to), reject(new Error(`${e.message}${detail()}`))));
|
|
335
|
+
});
|
|
336
|
+
if (!(await healthy())) {
|
|
337
|
+
if (child.exitCode !== null && !child.stderr.readableEnded) {
|
|
338
|
+
await Promise.race([new Promise((r) => child.stderr.once("end", r)), new Promise((r) => setTimeout(r, 50))]);
|
|
339
|
+
}
|
|
340
|
+
throw new Error(`the ${PRODUCT.name} server did not come up${detail()}`);
|
|
341
|
+
}
|
|
342
|
+
} finally {
|
|
343
|
+
child.stdout.destroy();
|
|
344
|
+
child.stderr.destroy();
|
|
345
|
+
child.unref();
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
export function openBrowserArgs(url, platform = process.platform) {
|
|
350
|
+
if (platform === "darwin") return ["open", [url]];
|
|
351
|
+
if (platform === "win32") return ["cmd", ["/c", "start", "", url.replace(/&/g, "^&")]];
|
|
352
|
+
return ["xdg-open", [url]];
|
|
214
353
|
}
|
|
215
354
|
|
|
216
355
|
function openBrowser(url) {
|
|
217
|
-
const
|
|
218
|
-
const c = spawn(
|
|
356
|
+
const [bin, args] = openBrowserArgs(url);
|
|
357
|
+
const c = spawn(bin, args, { detached: true, stdio: "ignore" });
|
|
219
358
|
c.on("error", () => {});
|
|
220
359
|
c.unref();
|
|
221
360
|
}
|
|
222
361
|
|
|
362
|
+
export { req, stopServer, ensureServer };
|
|
363
|
+
|
|
223
364
|
/** `config background <preset|default>`, `config background --image <file> [--strength N] [--no-blur]`, `config get background` */
|
|
224
365
|
export function parseConfigArgs(args) {
|
|
225
366
|
const a = { action: "", key: "", value: "", image: "", strength: null, blur: null };
|
|
@@ -276,7 +417,7 @@ export function configCommand(args) {
|
|
|
276
417
|
return `Background set to your image. It is copied as it is; the page scales it (and the copy keeps its metadata on disk). Reload the page to see it.\n`;
|
|
277
418
|
}
|
|
278
419
|
writeBackground({ kind: "preset", name: a.value, ...keep });
|
|
279
|
-
return `Background set to ${a.value}. Reload the page to see it.\n`;
|
|
420
|
+
return `Background set to ${a.reset ? `the default (${a.value})` : a.value}. Reload the page to see it.\n`;
|
|
280
421
|
}
|
|
281
422
|
|
|
282
423
|
const out = (text) => new Promise((r) => process.stdout.write(text, r));
|
|
@@ -416,7 +557,7 @@ async function main() {
|
|
|
416
557
|
process.exit(r.exit);
|
|
417
558
|
}
|
|
418
559
|
if (argv[0] === "setup-checker") {
|
|
419
|
-
const py = setupChecker();
|
|
560
|
+
const py = await setupChecker();
|
|
420
561
|
return out(`checker ready: ${py}\n`);
|
|
421
562
|
}
|
|
422
563
|
if (argv[0] === "setup-rewriter") {
|
|
@@ -460,9 +601,8 @@ async function main() {
|
|
|
460
601
|
print: (line) => process.stdout.write(line + "\n"),
|
|
461
602
|
server: { healthy, openWork: async () => { const r = await req({ method: "GET", path: "/cli/open-work", timeout: 3000 }); return { status: r.status, body: json(r.text) }; } },
|
|
462
603
|
stop: stopServer,
|
|
463
|
-
runGetSh: (base) => runGetShDefault(base),
|
|
604
|
+
runGetSh: (base, opts) => runGetShDefault(base, process.env, opts),
|
|
464
605
|
launcherVersion: (route, root) => (route === "npm" ? versionOf(process.execPath, [path.join(root, "src", "cli.js"), "--version"]) : versionOf(path.join(home, ".local", "bin", BIN), ["--version"])),
|
|
465
|
-
run: (cmd, args) => spawnSync(cmd, args, { stdio: ["ignore", "inherit", "inherit"] }).status ?? 1,
|
|
466
606
|
});
|
|
467
607
|
process.exit(code);
|
|
468
608
|
}
|
|
@@ -557,8 +697,8 @@ async function main() {
|
|
|
557
697
|
if (a.summary) return out(JSON.stringify({ ok: !!j.ok, flags: (j.flags || []).map((f) => ({ code: f.code, severity: f.severity })), ...(j.checks?.notice ? { checks_notice: j.checks.notice } : {}) }) + "\n");
|
|
558
698
|
return out(JSON.stringify(j, null, 2) + "\n");
|
|
559
699
|
} else if (a.cmd === "stop") {
|
|
560
|
-
await stopServer();
|
|
561
|
-
return out("stopped\n");
|
|
700
|
+
const res = await stopServer();
|
|
701
|
+
return out(res?.staleRemoved ? "no running server (stale record removed)\n" : res?.stopped ? "stopped\n" : "no server was running\n");
|
|
562
702
|
} else throw new Error(`unknown command ${a.cmd}\n${HELP}`);
|
|
563
703
|
}
|
|
564
704
|
|
package/src/doctor.js
CHANGED
|
@@ -90,6 +90,15 @@ const run = (cmd, args, env) => {
|
|
|
90
90
|
}
|
|
91
91
|
};
|
|
92
92
|
|
|
93
|
+
/** Parse the target cli.js path from an installer-written launcher (single- or double-quoted). */
|
|
94
|
+
export function parseLauncherTarget(text) {
|
|
95
|
+
const sq = text.match(/exec node '((?:[^']|'\\'')+)'/);
|
|
96
|
+
if (sq) return sq[1].replace(/'\\''/g, "'");
|
|
97
|
+
const dq = text.match(/exec node "([^"]+)"/);
|
|
98
|
+
if (dq) return dq[1];
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
|
|
93
102
|
/**
|
|
94
103
|
* @param {object} o home, appRoot, env, verbose: where to look. probes: injectable `defaultBrowser()` and `freeBytes(dir)` for tests.
|
|
95
104
|
* @returns {Promise<{schema:string, ok:boolean, version:string|null, checks:Array<{status:string,name:string,detail:string,fix?:string,required:boolean}>}>}
|
|
@@ -149,8 +158,8 @@ export async function runDoctor(o = {}) {
|
|
|
149
158
|
const found = which(BIN);
|
|
150
159
|
const launcher = found || path.join(home, ".local", "bin", BIN);
|
|
151
160
|
if (fs.existsSync(launcher)) {
|
|
152
|
-
const target = (
|
|
153
|
-
// the installer writes `exec node
|
|
161
|
+
const target = parseLauncherTarget(fs.readFileSync(launcher, "utf8"));
|
|
162
|
+
// the installer writes `exec node '<app>/src/cli.js'` (or double-quoted in older versions); any other launcher (a symlink, npm link) is judged by whether it runs
|
|
154
163
|
add(!target || fs.existsSync(target) ? "OK" : "MISSING", "launcher", `${launcher}${target ? ` -> ${target}` : " (custom launcher)"}`, { required: true, fix: REINSTALL_FIX });
|
|
155
164
|
const v = run(launcher, ["--version"], env);
|
|
156
165
|
add(v ? "OK" : "MISSING", `${BIN} --version`, v || "the launcher did not run", { required: true, fix: REINSTALL_FIX });
|
|
@@ -166,7 +175,7 @@ export async function runDoctor(o = {}) {
|
|
|
166
175
|
};
|
|
167
176
|
const runs = (p) => {
|
|
168
177
|
try {
|
|
169
|
-
const t = (fs.readFileSync(p, "utf8").slice(0, 400)
|
|
178
|
+
const t = parseLauncherTarget(fs.readFileSync(p, "utf8").slice(0, 400)); // the installer's 2-line launcher
|
|
170
179
|
return real(t || p);
|
|
171
180
|
} catch {
|
|
172
181
|
return real(p);
|
package/src/docx.js
CHANGED
|
@@ -4,6 +4,9 @@ import { readBudget, tooMany, walkXml } from "./xml.js";
|
|
|
4
4
|
import { OCCURRENCE, affords, bigramCodes, diceOf, hitsOf, normPaid, normText, searchCost, similarity, spend, textHash, weigh, weightOf } from "./text.js";
|
|
5
5
|
import { openZip } from "./zip.js";
|
|
6
6
|
import { summarizeRuns } from "./fmt.js";
|
|
7
|
+
import { HAS_NON_PLAIN_WS, cuttable, normOccurrences } from "./pptx.js";
|
|
8
|
+
|
|
9
|
+
export { cuttable };
|
|
7
10
|
|
|
8
11
|
// One classifier for `protected`: the engine's own (default deny), bundled for Node. The batch, `apply` and the editor agree because they ask the same code.
|
|
9
12
|
const { protectReasons } = await import(pathToFileURL(engineNodePath()).href);
|
|
@@ -74,7 +77,9 @@ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget()
|
|
|
74
77
|
let tableCount = 0; // 1-based order among top-level tables (the label users read), unlike body_path[0] which counts every body item
|
|
75
78
|
let cellK = 0;
|
|
76
79
|
let cur = null;
|
|
77
|
-
|
|
80
|
+
const revStack = [];
|
|
81
|
+
let mcFallbackDepth = 0;
|
|
82
|
+
const visible = () => !revStack.includes("del") && (revStack.length === 0 || revStack[revStack.length - 1] !== "moveFrom");
|
|
78
83
|
let inT = false;
|
|
79
84
|
let inInstr = false;
|
|
80
85
|
let inBody = false;
|
|
@@ -87,9 +92,14 @@ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget()
|
|
|
87
92
|
walkXml(xml, {
|
|
88
93
|
onOpen(name, attrs) {
|
|
89
94
|
stack.push(name);
|
|
95
|
+
if (name === "mc:Fallback") {
|
|
96
|
+
mcFallbackDepth++;
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
if (mcFallbackDepth > 0) return;
|
|
90
100
|
if (name === "w:body") inBody = true;
|
|
91
101
|
if (!inBody) return;
|
|
92
|
-
if (name === "w:tbl") {
|
|
102
|
+
if (name === "w:tbl" && pDepth === 0) {
|
|
93
103
|
tblDepth++;
|
|
94
104
|
if (tblDepth === 1) {
|
|
95
105
|
bodyIndex++;
|
|
@@ -98,16 +108,15 @@ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget()
|
|
|
98
108
|
info.row = -1;
|
|
99
109
|
info.col = -1;
|
|
100
110
|
}
|
|
101
|
-
} else if (name === "w:tr" && tblDepth === 1) {
|
|
111
|
+
} else if (name === "w:tr" && tblDepth === 1 && pDepth === 0) {
|
|
102
112
|
info.row++;
|
|
103
113
|
info.col = -1;
|
|
104
|
-
} else if (name === "w:tc" && tblDepth === 1) {
|
|
114
|
+
} else if (name === "w:tc" && tblDepth === 1 && pDepth === 0) {
|
|
105
115
|
info.col++;
|
|
106
116
|
cellK = 0;
|
|
107
117
|
} else if (name === "w:p") {
|
|
108
118
|
pDepth++;
|
|
109
|
-
|
|
110
|
-
if (nested) return;
|
|
119
|
+
if (pDepth > 1) return;
|
|
111
120
|
let path = null;
|
|
112
121
|
let tableNo = null;
|
|
113
122
|
if (tblDepth === 0) {
|
|
@@ -121,15 +130,15 @@ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget()
|
|
|
121
130
|
cur = { ordinal: paragraphs.length, path, tableNo, paraId: /^[0-9A-Fa-f]{8}$/.test(attrs["w14:paraId"] || "") ? attrs["w14:paraId"].toUpperCase() : null, text: "", instr: "", protected: false, style: null, align: "", runs: [] };
|
|
122
131
|
if (detail) (Object.assign(cur, { spans: [], list: null, ppr: [], sect: false, loose: false }), (marks = new Set()), (pprTop = null));
|
|
123
132
|
paragraphs.push(cur);
|
|
124
|
-
} else if (cur) {
|
|
133
|
+
} else if (cur && pDepth === 1) {
|
|
125
134
|
if (detail) {
|
|
126
135
|
const n = stack.length;
|
|
127
|
-
if (
|
|
136
|
+
if (stack[n - 2] === "w:pPr" && stack[n - 3] === "w:p") {
|
|
128
137
|
pprTop = name;
|
|
129
138
|
if (name === "w:sectPr") cur.sect = true;
|
|
130
139
|
}
|
|
131
140
|
if (pprTop && !PPR_OWN.has(pprTop)) cur.ppr.push(written(name, attrs));
|
|
132
|
-
else if (pprTop === "w:numPr" &&
|
|
141
|
+
else if (pprTop === "w:numPr" && stack[n - 2] === "w:numPr" && (name === "w:numId" || name === "w:ilvl")) (cur.list ||= { numId: "", ilvl: 0 })[name === "w:numId" ? "numId" : "ilvl"] = name === "w:numId" ? attrs["w:val"] || "" : Number(attrs["w:val"]) || 0;
|
|
133
142
|
const mark = RANGE.exec(name);
|
|
134
143
|
if (mark && "w:id" in attrs) {
|
|
135
144
|
const key = `${mark[1]} ${attrs["w:id"]}`;
|
|
@@ -147,35 +156,33 @@ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget()
|
|
|
147
156
|
else if (name === "w:r" && !run) run = { text: false, p: {}, at: cur.text.length, depth: stack.length, set: {}, rest: [] };
|
|
148
157
|
else if (name === "w:rPr") rprDepth++;
|
|
149
158
|
else if (run && rprDepth === 1 && stack.includes("w:rPr")) runProp(run.p, name, attrs);
|
|
150
|
-
if (name === "w:del")
|
|
151
|
-
else if (name === "w:
|
|
159
|
+
if (name === "w:del") revStack.push("del");
|
|
160
|
+
else if (name === "w:moveFrom") revStack.push("moveFrom");
|
|
161
|
+
else if (name === "w:ins") revStack.push("ins");
|
|
162
|
+
else if (name === "w:t" && visible()) (inT = true, run && (run.text = true));
|
|
152
163
|
else if (name === "w:instrText") inInstr = true;
|
|
153
|
-
else if (name === "w:tab" &&
|
|
164
|
+
else if (name === "w:tab" && visible()) cur.text += "\t";
|
|
154
165
|
else if (name === "w:br" || name === "w:cr") {
|
|
155
|
-
if (
|
|
166
|
+
if (visible() && attrs["w:type"] !== "page" && attrs["w:type"] !== "column") cur.text += "\n";
|
|
156
167
|
} else if (name === "w:drawing" || name === "w:pict") {
|
|
157
|
-
if (
|
|
168
|
+
if (visible()) cur.text += OBJ;
|
|
158
169
|
}
|
|
159
170
|
}
|
|
160
171
|
},
|
|
161
172
|
onText(t) {
|
|
162
|
-
if (
|
|
163
|
-
|
|
173
|
+
if (mcFallbackDepth > 0) return;
|
|
174
|
+
if (cur && pDepth === 1 && inT) cur.text += t;
|
|
175
|
+
else if (cur && pDepth === 1 && inInstr && cur.instr.length < 200) cur.instr += t;
|
|
164
176
|
},
|
|
165
177
|
onClose(name) {
|
|
166
178
|
stack.pop();
|
|
167
|
-
if (
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
else if (name === "w:r" && run) {
|
|
173
|
-
if (run.text && cur) cur.runs.push(run.p);
|
|
174
|
-
if (detail && cur && cur.text.length > run.at) cur.spans.push({ n: cur.text.length - run.at, set: run.set, rest: run.rest.join(" ") });
|
|
175
|
-
run = null;
|
|
176
|
-
} else if (name === "w:p") {
|
|
179
|
+
if (mcFallbackDepth > 0) {
|
|
180
|
+
if (name === "mc:Fallback") mcFallbackDepth--;
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
if (name === "w:p") {
|
|
177
184
|
pDepth--;
|
|
178
|
-
if (
|
|
185
|
+
if (pDepth > 0) return;
|
|
179
186
|
if (cur) (cur.fmt = summarizeRuns(cur.runs), delete cur.runs);
|
|
180
187
|
if (cur && detail) {
|
|
181
188
|
if (cur.spans.reduce((n, x) => n + x.n, 0) !== cur.text.length) cur.spans = null;
|
|
@@ -183,8 +190,21 @@ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget()
|
|
|
183
190
|
cur.loose = marks.size > 0;
|
|
184
191
|
}
|
|
185
192
|
cur = null;
|
|
193
|
+
revStack.length = 0;
|
|
194
|
+
} else if (name === "w:tbl" && pDepth === 0) {
|
|
195
|
+
tblDepth--;
|
|
196
|
+
} else if (cur && pDepth === 1) {
|
|
197
|
+
if (name === "w:t") inT = false;
|
|
198
|
+
else if (name === "w:instrText") inInstr = false;
|
|
199
|
+
else if (name === "w:del" || name === "w:moveFrom" || name === "w:ins") revStack.pop();
|
|
200
|
+
else if (name === "w:rPr") rprDepth = Math.max(0, rprDepth - 1);
|
|
201
|
+
else if (name === "w:r" && run) {
|
|
202
|
+
if (run.text && cur) cur.runs.push(run.p);
|
|
203
|
+
if (detail && cur && cur.text.length > run.at) cur.spans.push({ n: cur.text.length - run.at, set: run.set, rest: run.rest.join(" ") });
|
|
204
|
+
run = null;
|
|
205
|
+
}
|
|
206
|
+
if (detail && name === pprTop && stack[stack.length - 1] === "w:pPr") pprTop = null;
|
|
186
207
|
}
|
|
187
|
-
if (detail && name === pprTop && stack[stack.length - 1] === "w:pPr") pprTop = null;
|
|
188
208
|
},
|
|
189
209
|
}, budget);
|
|
190
210
|
for (const p of paragraphs) p.hash = textHash(p.text);
|
|
@@ -587,23 +607,6 @@ function normHead(text, n) {
|
|
|
587
607
|
// Once the window holds at least as many normalized characters as the prefix, it answers for the whole text; else it doubles.
|
|
588
608
|
// Every character read (the walk to a cut and the window normalized) is spent from the budget of the request (src/text.js): null when it
|
|
589
609
|
// runs out. An outside review: CJK text has no character below U+0300, every window grew to the whole paragraph, 3.65 s for one anchor.
|
|
590
|
-
const MARK = /\p{M}/u;
|
|
591
|
-
/**
|
|
592
|
-
* May a window start (or end) right before text[i]? Yes below U+0300 (no such character joins the one before it under NFC); never inside a
|
|
593
|
-
* surrogate pair or before a combining mark; otherwise when the character normalizes the same after the three before it as on its own
|
|
594
|
-
* (the longest chain of starters that compose, Hangul L V T and its like, is three: test/parse-limits.test.js checks every code point).
|
|
595
|
-
* Exported with the two window readers for that test.
|
|
596
|
-
*/
|
|
597
|
-
export function cuttable(text, i) {
|
|
598
|
-
if (i <= 0 || i >= text.length) return true;
|
|
599
|
-
const c = text.charCodeAt(i);
|
|
600
|
-
if (c < 0x300) return true;
|
|
601
|
-
if (c >= 0xdc00 && c <= 0xdfff) return false;
|
|
602
|
-
const ch = String.fromCodePoint(text.codePointAt(i));
|
|
603
|
-
if (MARK.test(ch)) return false;
|
|
604
|
-
const before = text.slice(Math.max(0, i - 3), i);
|
|
605
|
-
return (before + ch).normalize("NFC") === before.normalize("NFC") + ch.normalize("NFC");
|
|
606
|
-
}
|
|
607
610
|
/** The nearest place at or before (dir -1) or after (dir 1) i where a window may be cut, each character walked spent; null past the budget. */
|
|
608
611
|
function cutNear(text, i, dir) {
|
|
609
612
|
let n = 0;
|
|
@@ -635,6 +638,15 @@ export function startsWithNorm(text, from, suffix) {
|
|
|
635
638
|
* `nth` is the occurrence index remembered at capture. Returns {at: the offset, k: which occurrence it is}, or {at: -1, k: null} when the
|
|
636
639
|
* text does not occur or the budget ran out.
|
|
637
640
|
*/
|
|
641
|
+
/** does the text around the occurrence at `at` (of `qLen` UTF-16 units) read as prefix and suffix (normalized, as an anchor keeps them)? null: the budget ran out */
|
|
642
|
+
function fitsAt(text, qLen, at, { prefix = "", suffix = "" }) {
|
|
643
|
+
prefix = String(prefix || "").slice(-CTX); // what an anchor keeps (durableFor); a longer one from elsewhere is read as its last CTX
|
|
644
|
+
suffix = String(suffix || "").slice(0, CTX);
|
|
645
|
+
const pre = !prefix || endsWithNorm(text, at, prefix);
|
|
646
|
+
if (pre === null) return null;
|
|
647
|
+
const suf = pre && (!suffix || startsWithNorm(text, at + qLen, suffix));
|
|
648
|
+
return suf === null ? null : !!(pre && suf);
|
|
649
|
+
}
|
|
638
650
|
function pickOccurrence(p, q, opts = {}) {
|
|
639
651
|
const text = p.text;
|
|
640
652
|
// the remembered occurrence first, listing only the occurrences up to it: when its surroundings still fit, it is the answer every rule
|
|
@@ -649,29 +661,45 @@ function pickOccurrence(p, q, opts = {}) {
|
|
|
649
661
|
const head = occurrences(p, q, remembered + 1);
|
|
650
662
|
if (head === null) return { at: -1, k: null };
|
|
651
663
|
if (head.length === remembered + 1) {
|
|
652
|
-
const ok = fitsAt(text, q, head[remembered], opts);
|
|
664
|
+
const ok = fitsAt(text, q.length, head[remembered], opts);
|
|
653
665
|
if (ok === null) return { at: -1, k: null };
|
|
654
666
|
if (ok) return { at: head[remembered], k: remembered };
|
|
655
667
|
}
|
|
656
668
|
}
|
|
657
669
|
const occ = occurrencesIn(p, q);
|
|
658
|
-
const at = occ?.length ? pickFrom(text, q, occ, { ...opts, nth: remembered }) : -1;
|
|
670
|
+
const at = occ?.length ? pickFrom(text, q.length, occ, { ...opts, nth: remembered }) : -1;
|
|
659
671
|
return at < 0 ? { at: -1, k: null } : { at, k: occ.indexOf(at) };
|
|
660
672
|
}
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
const
|
|
668
|
-
|
|
673
|
+
function pickNormOccurrence(p, q, opts = {}) {
|
|
674
|
+
const occ = normOccurrences(p, q);
|
|
675
|
+
if (!occ?.length) return { at: -1, len: 0, k: null };
|
|
676
|
+
let { nth = null } = opts;
|
|
677
|
+
if (!(Number.isInteger(nth) && nth >= 0 && nth < occ.length)) nth = null;
|
|
678
|
+
const prefix = String(opts.prefix || "").slice(-CTX);
|
|
679
|
+
const suffix = String(opts.suffix || "").slice(0, CTX);
|
|
680
|
+
if ((prefix || suffix) && occ.length > 1 && !spend(occ.length * 2 * (prefix.length + suffix.length + 16))) return { at: -1, len: 0, k: null };
|
|
681
|
+
const guarded = [];
|
|
682
|
+
for (let i = 0; i < occ.length; i++) {
|
|
683
|
+
const o = occ[i];
|
|
684
|
+
const ok = fitsAt(p.text, o.end - o.start, o.start, { prefix, suffix });
|
|
685
|
+
if (ok === null) return { at: -1, len: 0, k: null };
|
|
686
|
+
if (ok) guarded.push(i);
|
|
687
|
+
}
|
|
688
|
+
let idx;
|
|
689
|
+
if (guarded.length === 1) idx = guarded[0];
|
|
690
|
+
else if (nth != null && guarded.includes(nth)) idx = nth;
|
|
691
|
+
else if (Number.isInteger(opts.hintStart) && guarded.length) {
|
|
692
|
+
idx = guarded.reduce((best, i) => (Math.abs(occ[i].start - opts.hintStart) < Math.abs(occ[best].start - opts.hintStart) ? i : best), guarded[0]);
|
|
693
|
+
} else if (guarded.length) idx = guarded[0];
|
|
694
|
+
else idx = nth != null ? nth : 0;
|
|
695
|
+
const hit = occ[idx];
|
|
696
|
+
return { at: hit.start, len: hit.end - hit.start, k: idx };
|
|
669
697
|
}
|
|
670
|
-
function pickFrom(text,
|
|
698
|
+
function pickFrom(text, qLen, occ, { prefix = "", suffix = "", nth = null, hintStart = null }) {
|
|
671
699
|
if (!(Number.isInteger(nth) && nth >= 0 && nth < occ.length)) nth = null;
|
|
672
700
|
prefix = String(prefix || "").slice(-CTX);
|
|
673
701
|
suffix = String(suffix || "").slice(0, CTX);
|
|
674
|
-
const fits = (at) => fitsAt(text,
|
|
702
|
+
const fits = (at) => fitsAt(text, qLen, at, { prefix, suffix });
|
|
675
703
|
// each occurrence reads about CTX characters on each side, out of the budget of the request (src/text.js): a paragraph of 100,000
|
|
676
704
|
// "a" read the whole text before every one of them and took 10 seconds in an outside review. Past the budget, no offset.
|
|
677
705
|
if ((prefix || suffix) && occ.length > 1 && !spend(occ.length * 2 * (prefix.length + suffix.length + 16))) return -1;
|
|
@@ -683,6 +711,9 @@ function pickFrom(text, q, occ, { prefix = "", suffix = "", nth = null }) {
|
|
|
683
711
|
}
|
|
684
712
|
if (guarded.length === 1) return guarded[0];
|
|
685
713
|
if (nth != null && nth < occ.length && guarded.includes(occ[nth])) return occ[nth];
|
|
714
|
+
if (Number.isInteger(hintStart) && guarded.length) {
|
|
715
|
+
return guarded.reduce((best, at) => (Math.abs(at - hintStart) < Math.abs(best - hintStart) ? at : best), guarded[0]);
|
|
716
|
+
}
|
|
686
717
|
if (guarded.length) return guarded[0];
|
|
687
718
|
return nth != null && nth < occ.length ? occ[nth] : occ[0];
|
|
688
719
|
}
|
|
@@ -697,12 +728,31 @@ function durableFor(paragraphs, p, engineTop, ctx, loc = {}) {
|
|
|
697
728
|
let nth = null;
|
|
698
729
|
if (q) {
|
|
699
730
|
// the engine's own offsets when they point at the quote; else the occurrence whose surroundings match prefix and suffix
|
|
700
|
-
const
|
|
701
|
-
const
|
|
702
|
-
const
|
|
731
|
+
const hintStart = Number.isInteger(loc.start) ? loc.start : null;
|
|
732
|
+
const preferNorm = (ctx.quote !== q && ctx.quote.trim() !== q) || (q.includes(" ") && HAS_NON_PLAIN_WS.test(p.text));
|
|
733
|
+
const own = hintStart !== null && Number.isInteger(loc.end) && p.text.slice(loc.start, loc.end) === ctx.quote;
|
|
734
|
+
let picked;
|
|
735
|
+
if (own) {
|
|
736
|
+
let k = -1;
|
|
737
|
+
if (preferNorm) {
|
|
738
|
+
const nocc = normOccurrences(p, q);
|
|
739
|
+
if (nocc) {
|
|
740
|
+
k = nocc.findIndex((o) => o.start === loc.start && o.end === loc.end);
|
|
741
|
+
if (k < 0) k = nocc.findIndex((o) => o.start >= loc.start && o.end <= loc.end);
|
|
742
|
+
}
|
|
743
|
+
} else {
|
|
744
|
+
k = occurrencesIn(p, ctx.quote)?.indexOf(loc.start) ?? -1;
|
|
745
|
+
}
|
|
746
|
+
picked = { at: loc.start, k: k >= 0 ? k : null, len: ctx.quote.length };
|
|
747
|
+
} else if (preferNorm) {
|
|
748
|
+
picked = pickNormOccurrence(p, q, { prefix, suffix, hintStart });
|
|
749
|
+
} else {
|
|
750
|
+
picked = { ...pickOccurrence(p, ctx.quote, { prefix, suffix, hintStart }), len: ctx.quote.length };
|
|
751
|
+
if (picked.at < 0) picked = pickNormOccurrence(p, q, { prefix, suffix, hintStart });
|
|
752
|
+
}
|
|
703
753
|
if (picked.at >= 0) {
|
|
704
754
|
start = picked.at;
|
|
705
|
-
end = picked.at +
|
|
755
|
+
end = picked.at + picked.len;
|
|
706
756
|
nth = picked.k;
|
|
707
757
|
}
|
|
708
758
|
}
|
|
@@ -739,10 +789,27 @@ export function resolveDurable(paragraphs, d) {
|
|
|
739
789
|
const q = d.quote || "";
|
|
740
790
|
let start = null;
|
|
741
791
|
let end = null;
|
|
742
|
-
const
|
|
792
|
+
const ctxOpts = { prefix: d.prefix ?? d.before ?? "", suffix: d.suffix ?? d.after ?? "", nth: d.nth ?? d.occ ?? null };
|
|
793
|
+
const preferNorm = !!q && q === normText(q) && q.includes(" ") && HAS_NON_PLAIN_WS.test(p.text);
|
|
794
|
+
let at = -1;
|
|
795
|
+
let len = q.length;
|
|
796
|
+
let k = null;
|
|
797
|
+
if (q) {
|
|
798
|
+
if (preferNorm) {
|
|
799
|
+
const picked = pickNormOccurrence(p, q, ctxOpts);
|
|
800
|
+
if (picked.at >= 0) ({ at, len, k } = picked);
|
|
801
|
+
else ({ at, k } = pickOccurrence(p, q, ctxOpts));
|
|
802
|
+
} else {
|
|
803
|
+
({ at, k } = pickOccurrence(p, q, ctxOpts));
|
|
804
|
+
if (at < 0 || (Number.isInteger(ctxOpts.nth) && ctxOpts.nth > 0 && k !== ctxOpts.nth)) {
|
|
805
|
+
const picked = pickNormOccurrence(p, q, ctxOpts);
|
|
806
|
+
if (picked.at >= 0) ({ at, len, k } = picked);
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
}
|
|
743
810
|
if (at >= 0) {
|
|
744
811
|
start = at;
|
|
745
|
-
end = at +
|
|
812
|
+
end = at + len;
|
|
746
813
|
}
|
|
747
814
|
return { method: hit.method, score: hit.score, ordinal: p.ordinal, path: p.path, tableNo: p.tableNo ?? null, paraId: p.paraId, textHash: p.hash, protected: !!p.protected, ...(p.protected ? { protectInfo: protectInfo(p) } : {}), dupCount: indexOf(paragraphs).byHash.get(p.hash)?.length ?? 0, nth: at >= 0 ? k : null, prevHash: paragraphs[p.ordinal - 1]?.hash || null, nextHash: paragraphs[p.ordinal + 1]?.hash || null, text: normHead(p.text, PARA_TEXT_KEEP) ?? "", start, end, engineDelta: d.engineDelta || 0 };
|
|
748
815
|
}
|