onionskin-cli 0.0.2 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -0
- package/INSTALL.md +5 -1
- package/NOTICE +34 -0
- package/README.md +101 -33
- package/docs/contracts/batch.schema.json +27 -4
- package/docs/contracts/ops.schema.json +117 -5
- package/package.json +4 -2
- package/skills/onionskin/SKILL.md +9 -6
- package/skills/onionskin/reference/batch.md +5 -3
- package/skills/onionskin/reference/formats.md +5 -1
- package/skills/onionskin/reference/ops.md +47 -3
- package/skills/onionskin/reference/troubleshooting.md +31 -0
- package/src/agent-detect.js +28 -2
- package/src/agent-prompt.js +12 -1
- package/src/agent-runner.js +77 -19
- package/src/anchor-work.js +154 -0
- package/src/anchor-worker.js +38 -0
- package/src/anchors.js +61 -11
- package/src/apply.js +119 -47
- package/src/batch.js +32 -8
- package/src/check-worker.js +8 -5
- package/src/cli.js +91 -25
- package/src/constants.js +34 -1
- package/src/diff.js +3 -1
- package/src/docx.js +368 -58
- package/src/edit-hash.js +19 -0
- package/src/fsutil.js +1 -1
- package/src/history-worker.js +8 -8
- package/src/history.js +135 -31
- package/src/html-active.js +384 -0
- package/src/html-envelope.js +51 -0
- package/src/html-parser.js +8 -0
- package/src/html-render-worker.js +25 -0
- package/src/html-render.js +880 -0
- package/src/html-utf8.js +65 -0
- package/src/html-write-worker.js +14 -0
- package/src/html-write.js +380 -0
- package/src/html.js +1257 -0
- package/src/local.js +68 -4
- package/src/merge-package.js +1110 -0
- package/src/merge-pptx.js +98 -38
- package/src/merge-proof.js +1019 -0
- package/src/merge-worker.js +60 -0
- package/src/merge-xlsx.js +84 -29
- package/src/merge.js +286 -83
- package/src/nfc.js +46 -0
- package/src/notes.js +122 -0
- package/src/offload.js +400 -1
- package/src/open-worker.js +27 -0
- package/src/ops/common.js +2 -1
- package/src/ops/docx.js +58 -29
- package/src/ops/html.js +394 -0
- package/src/ops/pptx.js +74 -21
- package/src/ops/run.js +23 -3
- package/src/ops/runs.js +215 -13
- package/src/ops/schema.js +8 -1
- package/src/ops/xlsx.js +12 -7
- package/src/ops-worker.js +17 -0
- package/src/package.js +26 -2
- package/src/person.js +8 -1
- package/src/pptx.js +85 -21
- package/src/quick-plan.js +13 -6
- package/src/redline-worker.js +10 -14
- package/src/reply.js +46 -19
- package/src/retain.js +223 -0
- package/src/review.js +32 -18
- package/src/rewrite-check.js +2 -2
- package/src/sanitize.js +16 -3
- package/src/server.js +88 -37
- package/src/session.js +1143 -208
- package/src/stop-gate.js +34 -0
- package/src/suggest-reply.js +42 -7
- package/src/suggest.js +14 -29
- package/src/text.js +161 -11
- package/src/update.js +288 -0
- package/src/voice-guide.js +4 -3
- package/src/voice-learn.js +4 -3
- package/src/voice-sample-worker.js +7 -6
- package/src/voice-worker.js +4 -6
- package/src/watch.js +188 -0
- package/src/worker-host.js +23 -0
- package/src/xlsx.js +355 -94
- package/src/xml-tree.js +6 -3
- package/src/xml.js +82 -34
- package/src/zip-write.js +2 -1
- package/src/zip.js +9 -5
- package/vendor/parse5/LICENSE.txt +32 -0
- package/vendor/parse5/parse5.mjs +7822 -0
- package/vendor/parse5.lock.json +23 -0
- package/web/dist/assets/index-DkNx0LSH.css +1 -0
- package/web/dist/assets/{index-BUtofdtr.js → index-DytKkMdi.js} +117 -107
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-Dfizlg0i.css +0 -1
package/src/ops/pptx.js
CHANGED
|
@@ -2,7 +2,7 @@ import { partRels } from "../package.js";
|
|
|
2
2
|
import { openZip } from "../zip.js";
|
|
3
3
|
import { checkExpect, engineFailure, excerpt, fail, refused } from "./common.js";
|
|
4
4
|
import { openEngine } from "./engine.js";
|
|
5
|
-
import {
|
|
5
|
+
import { countMatches, findRanges, pickMatches, rangesOf } from "./runs.js";
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* PPTX driver (headless Engine: pptx.slide to read a slide model, pptx.setText to write one shape's paragraphs, pptx.slides to
|
|
@@ -44,7 +44,18 @@ async function slideModel(st, index) {
|
|
|
44
44
|
}
|
|
45
45
|
|
|
46
46
|
const slideText = (m) => shapesOf(m.els).map(shapeText).join("\n");
|
|
47
|
-
|
|
47
|
+
/** The slide with p:sldId `id` (the first of that id) and its index (-1: none), through a map rebuilt whenever a slide op replaces st.slides (a search per op was quadratic). */
|
|
48
|
+
function slideById(st, id) {
|
|
49
|
+
return st.slides[indexOf(st, id)];
|
|
50
|
+
}
|
|
51
|
+
function indexOf(st, id) {
|
|
52
|
+
if (st.byIdOf !== st.slides) {
|
|
53
|
+
st.byId = new Map();
|
|
54
|
+
st.slides.forEach((s, i) => st.byId.has(String(s.id)) || st.byId.set(String(s.id), i));
|
|
55
|
+
st.byIdOf = st.slides;
|
|
56
|
+
}
|
|
57
|
+
return st.byId.get(String(id)) ?? -1;
|
|
58
|
+
}
|
|
48
59
|
|
|
49
60
|
// ---- the package around a slide (read from the file as delivered: only slide ops change it, and they are checked here) ----------------
|
|
50
61
|
// what a copy may share with the slide it was made from: its layout, pictures and links (notes and comments are not copied)
|
|
@@ -60,7 +71,7 @@ function deckRefusal(st) {
|
|
|
60
71
|
return null;
|
|
61
72
|
}
|
|
62
73
|
function slidePart(st, id) {
|
|
63
|
-
return st
|
|
74
|
+
return slideById(st, id)?.path || null;
|
|
64
75
|
}
|
|
65
76
|
/** a part of the deck other than the presentation and the slide's own notes that links to the slide (a "go to slide" link, a summary zoom) */
|
|
66
77
|
function linkedFrom(st, part) {
|
|
@@ -79,7 +90,7 @@ async function findShape(st, rawTarget) {
|
|
|
79
90
|
const t = shapeTarget(st, rawTarget);
|
|
80
91
|
if (t.error) return fail("anchor_not_found", t.error);
|
|
81
92
|
const target = t.target;
|
|
82
|
-
const slide = st
|
|
93
|
+
const slide = slideById(st, target.slide_id);
|
|
83
94
|
if (!slide) return fail("anchor_not_found", `no slide with id ${target.slide_id}`);
|
|
84
95
|
const m = await slideModel(st, slide.index);
|
|
85
96
|
const hits = shapesOf(m.els).filter((e) => e.sid === target.shape_id);
|
|
@@ -107,13 +118,13 @@ export async function bind(st, op) {
|
|
|
107
118
|
for (const el of shapesOf(m.els)) {
|
|
108
119
|
el.text.paras.forEach((p, pi) => {
|
|
109
120
|
const text = paraText(p);
|
|
110
|
-
const n =
|
|
111
|
-
if (n) hits.push({ slideId: String(s.id), sid: el.sid, pi, n, ro: !!p.ro, text });
|
|
121
|
+
const { n, partial } = countMatches(text, op.find, op.match_case !== false);
|
|
122
|
+
if (n) hits.push({ slideId: String(s.id), sid: el.sid, pi, n, partial, ro: !!p.ro, text });
|
|
112
123
|
});
|
|
113
124
|
}
|
|
114
125
|
}
|
|
115
126
|
const total = hits.reduce((n, x) => n + x.n, 0);
|
|
116
|
-
const extra = { partial_words: hits.reduce((n, x) => n +
|
|
127
|
+
const extra = { partial_words: hits.reduce((n, x) => n + x.partial, 0), protected: hits.reduce((n, x) => n + (x.ro ? x.n : 0), 0) };
|
|
117
128
|
if (total !== op.expect.count) return fail("expect_mismatch", `found ${total} occurrence(s), expected ${op.expect.count}`, { found: total, ...extra });
|
|
118
129
|
return { ok: true, op, hits, total, ...extra };
|
|
119
130
|
}
|
|
@@ -232,8 +243,39 @@ async function execSlide(st, op) {
|
|
|
232
243
|
return { status: "applied", detail: { slide_id: copy } }; // the copy's id: its position may change with a later op of this file
|
|
233
244
|
}
|
|
234
245
|
|
|
246
|
+
/**
|
|
247
|
+
* replaceInRuns for every match at once (`starts`, non-overlapping, of `len` characters each: one number, or one length per start): the same result as replaceInRuns applied
|
|
248
|
+
* right to left, in one pass over the runs (a pass per match: one shape of 240,000 one-letter runs took minutes). Exported for its test.
|
|
249
|
+
*/
|
|
250
|
+
export function replaceAllInRuns(runs, starts, len, replacement) {
|
|
251
|
+
if (!starts.length) return runs.map((r) => ({ ...r }));
|
|
252
|
+
const at = rangesOf(starts, len);
|
|
253
|
+
const firstOf = new Array(at.length).fill(-1);
|
|
254
|
+
let pos = 0;
|
|
255
|
+
let k = 0;
|
|
256
|
+
const out = runs.map((r, i) => {
|
|
257
|
+
const a = pos;
|
|
258
|
+
const b = pos + r.text.length;
|
|
259
|
+
pos = b;
|
|
260
|
+
while (k < at.length && at[k].end <= a) k++;
|
|
261
|
+
let text = null;
|
|
262
|
+
let cur = 0;
|
|
263
|
+
for (let j = k; j < at.length && at[j].start < b; j++) {
|
|
264
|
+
const { start: s, end: e } = at[j];
|
|
265
|
+
if (e <= a) continue;
|
|
266
|
+
if (r.fld) throw new RangeError("the match is inside a field");
|
|
267
|
+
const put = firstOf[j] < 0;
|
|
268
|
+
if (put) firstOf[j] = i;
|
|
269
|
+
text = (text ?? "") + r.text.slice(cur, Math.max(0, s - a)) + (put ? replacement : "");
|
|
270
|
+
cur = Math.min(r.text.length, e - a);
|
|
271
|
+
}
|
|
272
|
+
return text === null ? { ...r } : { ...r, text: text + r.text.slice(cur) };
|
|
273
|
+
});
|
|
274
|
+
return out.filter((r, i) => r.text !== "" || i === firstOf[0] || r.fld);
|
|
275
|
+
}
|
|
276
|
+
|
|
235
277
|
/** replaces [s,e) of a paragraph's concatenated run texts; the first run in range carries the replacement and the formatting */
|
|
236
|
-
function replaceInRuns(runs, s, e, replacement) {
|
|
278
|
+
export function replaceInRuns(runs, s, e, replacement) {
|
|
237
279
|
let pos = 0;
|
|
238
280
|
const out = runs.map((r) => ({ ...r }));
|
|
239
281
|
let first = -1;
|
|
@@ -269,19 +311,18 @@ export async function exec(st, b) {
|
|
|
269
311
|
if (!f.el) return f;
|
|
270
312
|
const slideIndex = f.slide.index;
|
|
271
313
|
const hits = [];
|
|
272
|
-
f.el.text.paras.forEach((p, pi) =>
|
|
314
|
+
f.el.text.paras.forEach((p, pi) => findRanges(paraText(p), op.find, op.match_case !== false).forEach(({ start, end }) => hits.push({ pi, at: start, len: end - start })));
|
|
273
315
|
const pick = pickMatches(hits.map((_, i) => i), op);
|
|
274
316
|
if (pick.error) return fail("anchor_not_found", pick.error);
|
|
275
317
|
const chosen = pick.starts.map((i) => hits[i]);
|
|
276
318
|
const blocked = chosen.filter((h) => f.el.text.paras[h.pi].ro);
|
|
277
319
|
if (blocked.length) return fail("skipped_protected", "the match is in a protected paragraph", { detail: { skipped_protected: blocked.map((h) => ({ paragraph: h.pi, excerpt: excerpt(paraText(f.el.text.paras[h.pi])) })) } });
|
|
278
320
|
const byPara = new Map();
|
|
279
|
-
for (const h of chosen) byPara.
|
|
321
|
+
for (const h of chosen) (byPara.get(h.pi) || byPara.set(h.pi, []).get(h.pi)).push(h);
|
|
280
322
|
const specs = specsOf(f.el, (pi, runs) => {
|
|
281
|
-
const
|
|
282
|
-
if (!
|
|
283
|
-
|
|
284
|
-
return runs;
|
|
323
|
+
const hs = byPara.get(pi);
|
|
324
|
+
if (!hs) return null;
|
|
325
|
+
return replaceAllInRuns(runs, hs.map((h) => h.at), hs.map((h) => h.len), op.replace);
|
|
285
326
|
});
|
|
286
327
|
await st.h.engine.call("pptx.setText", [slideIndex, f.el.sid, specs]);
|
|
287
328
|
return { status: "applied", detail: { replaced: chosen.length } };
|
|
@@ -291,12 +332,25 @@ export async function exec(st, b) {
|
|
|
291
332
|
let n = 0;
|
|
292
333
|
let skippedMatches = 0;
|
|
293
334
|
const bySlide = new Map();
|
|
294
|
-
for (const h of b.hits)
|
|
335
|
+
for (const h of b.hits) {
|
|
336
|
+
const k = `${h.slideId}:${h.sid}`;
|
|
337
|
+
(bySlide.get(k) || bySlide.set(k, []).get(k)).push(h);
|
|
338
|
+
}
|
|
339
|
+
// each slide's model is read once, its shapes by id (a model per shape, searched shape by shape, was quadratic on a slide of many
|
|
340
|
+
// shapes): this op writes one shape at a time, and a write never changes another shape's model
|
|
341
|
+
const models = new Map();
|
|
342
|
+
const shapeOn = async (slide, sid) => {
|
|
343
|
+
if (!models.has(slide)) {
|
|
344
|
+
const byId = new Map();
|
|
345
|
+
if (slide >= 0) for (const e of shapesOf((await slideModel(st, slide)).els)) if (!byId.has(e.sid)) byId.set(e.sid, e);
|
|
346
|
+
models.set(slide, byId);
|
|
347
|
+
}
|
|
348
|
+
return models.get(slide).get(sid);
|
|
349
|
+
};
|
|
295
350
|
for (const [key, hs] of bySlide) {
|
|
296
351
|
const [slideId, sid] = key.split(":");
|
|
297
352
|
const slide = indexOf(st, slideId); // by id: an earlier slide op of this file may have moved or removed it
|
|
298
|
-
const
|
|
299
|
-
const el = shapesOf(m.els).find((e) => e.sid === Number(sid));
|
|
353
|
+
const el = await shapeOn(slide, Number(sid));
|
|
300
354
|
if (!el) {
|
|
301
355
|
skippedMatches += hs.reduce((a, h) => a + h.n, 0);
|
|
302
356
|
skipped.push({ slide: slideId, shape_id: Number(sid), reason: slide < 0 ? "the slide is gone" : "the shape is gone" });
|
|
@@ -310,10 +364,9 @@ export async function exec(st, b) {
|
|
|
310
364
|
if (!todo.size) continue;
|
|
311
365
|
const specs = specsOf(el, (pi, runs) => {
|
|
312
366
|
if (!todo.has(pi)) return null;
|
|
313
|
-
const
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
return runs;
|
|
367
|
+
const found = findRanges(runs.map((r) => r.text).join(""), op.find, op.match_case !== false);
|
|
368
|
+
n += found.length;
|
|
369
|
+
return replaceAllInRuns(runs, found.map((r) => r.start), found.map((r) => r.end - r.start), op.replace);
|
|
317
370
|
});
|
|
318
371
|
await st.h.engine.call("pptx.setText", [slide, Number(sid), specs]);
|
|
319
372
|
}
|
package/src/ops/run.js
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { nodeChecks } from "../package.js";
|
|
2
|
+
import { byFormat } from "../constants.js";
|
|
3
|
+
import { parserLoaded } from "../html.js";
|
|
4
|
+
import { previewOffThread, runOpsOffThread } from "../offload.js";
|
|
2
5
|
import * as docx from "./docx.js";
|
|
6
|
+
import * as html from "./html.js";
|
|
3
7
|
import * as pptx from "./pptx.js";
|
|
4
8
|
import * as xlsx from "./xlsx.js";
|
|
5
9
|
|
|
6
|
-
const DRIVERS = { docx, xlsx, pptx };
|
|
10
|
+
const DRIVERS = { docx, xlsx, pptx, html };
|
|
7
11
|
// ops whose job is to remove or rewrite content: a lower count of drawings/links/bookmarks is then expected, not damage
|
|
8
12
|
const REWRITES = new Set(["delete_paragraph", "set_table_cell_text", "set_shape_text", "delete_rows", "delete_columns", "delete_table_row", "delete_table_column", "delete_slide"]);
|
|
9
13
|
const BLOCKING = new Set(["invalid_package", "part_removed", "relationship_target_missing"]);
|
|
@@ -21,6 +25,13 @@ export function demote(results, reason) {
|
|
|
21
25
|
}
|
|
22
26
|
const notApplied = (i, op, why) => ({ index: i, op: op.op, ...(op.id ? { id: op.id } : {}), status: "not_applied", reason: why });
|
|
23
27
|
|
|
28
|
+
// The engine's calls resolve as promises, so a run of many ops (a merge of every paragraph of a long document) would hold the thread
|
|
29
|
+
// from its first op to its last: the loops below give the event loop a turn once TURN_MS have passed since the last one.
|
|
30
|
+
const TURN_MS = 40;
|
|
31
|
+
const turn = () => new Promise((resolve) => setImmediate(resolve));
|
|
32
|
+
let since = 0;
|
|
33
|
+
const maybeTurn = () => (performance.now() - since > TURN_MS ? turn().then(() => (since = performance.now())) : null);
|
|
34
|
+
|
|
24
35
|
/**
|
|
25
36
|
* Runs `ops` against `bytes`. Phase 1 binds every op (anchor + expect) against the untouched file; phase 2 executes in order.
|
|
26
37
|
* The engine cannot roll back, so nothing is saved unless the set is acceptable: atomic mode stops at the first op that
|
|
@@ -28,14 +39,18 @@ const notApplied = (i, op, why) => ({ index: i, op: op.op, ...(op.id ? { id: op.
|
|
|
28
39
|
* Returns {results, bytes (null = nothing to write), warnings, blocked}.
|
|
29
40
|
*/
|
|
30
41
|
export async function runOps({ bytes, format, name, ops, partial = false, anchors = new Map() }) {
|
|
31
|
-
|
|
42
|
+
// an HTML file is parsed in worker threads only (test/html-parser-thread.test.js): this runs again in the ops worker unless the parser is here
|
|
43
|
+
if (format === "html" && !parserLoaded()) return runOpsOffThread({ bytes, format, name, ops, partial, anchors });
|
|
44
|
+
const driver = byFormat(DRIVERS, format, "runOps");
|
|
32
45
|
const excluded = new Map();
|
|
33
46
|
for (let pass = 0; pass <= ops.length; pass++) {
|
|
34
47
|
const st = await driver.open({ bytes, name, anchors });
|
|
35
48
|
try {
|
|
49
|
+
await turn();
|
|
36
50
|
const results = new Array(ops.length);
|
|
37
51
|
const bound = new Array(ops.length);
|
|
38
52
|
for (const [i, op] of ops.entries()) {
|
|
53
|
+
await maybeTurn();
|
|
39
54
|
if (excluded.has(i)) results[i] = shape(i, op, excluded.get(i));
|
|
40
55
|
else {
|
|
41
56
|
const b = await driver.bind(st, op);
|
|
@@ -47,6 +62,7 @@ export async function runOps({ bytes, format, name, ops, partial = false, anchor
|
|
|
47
62
|
let stop = bindFailed && !partial;
|
|
48
63
|
let restart = false;
|
|
49
64
|
for (let i = 0; i < ops.length && !restart; i++) {
|
|
65
|
+
await maybeTurn();
|
|
50
66
|
if (!bound[i]) continue;
|
|
51
67
|
if (stop) {
|
|
52
68
|
results[i] = notApplied(i, ops[i], "atomic mode: another op did not apply, so nothing was written");
|
|
@@ -70,7 +86,10 @@ export async function runOps({ bytes, format, name, ops, partial = false, anchor
|
|
|
70
86
|
}
|
|
71
87
|
const applied = results.filter((r) => r.status === "applied").length;
|
|
72
88
|
if (!applied) return { results, bytes: null, warnings: [], blocked: null };
|
|
89
|
+
// HTML has no engine and no package: its driver applies the splices at once and runs its own proof and write check (src/ops/html.js)
|
|
90
|
+
if (driver.finish) return await driver.finish(st, results, { partial });
|
|
73
91
|
const out = await driver.save(st);
|
|
92
|
+
await turn();
|
|
74
93
|
// the writer is checked before anything leaves this function: a lossy save blocks the whole write
|
|
75
94
|
const flags = nodeChecks(bytes, out, format);
|
|
76
95
|
const rewrites = ops.some((op, i) => results[i].status === "applied" && (REWRITES.has(op.op) || (op.op === "replace_text" && "text" in op)));
|
|
@@ -93,7 +112,8 @@ export async function runOps({ bytes, format, name, ops, partial = false, anchor
|
|
|
93
112
|
* `partial_words` (replace_all_text) is the number of matches that sit inside a longer word; `in_formulas` (XLSX) the number inside a formula.
|
|
94
113
|
*/
|
|
95
114
|
export async function previewAll(bytes, format, op, { name = `preview.${format}`, anchors = new Map() } = {}) {
|
|
96
|
-
|
|
115
|
+
if (format === "html" && !parserLoaded()) return previewOffThread(bytes, format, op);
|
|
116
|
+
const driver = Object.hasOwn(DRIVERS, format) ? DRIVERS[format] : null;
|
|
97
117
|
if (!driver) return { ok: false, status: "refused", reason: `unsupported format ${format}` };
|
|
98
118
|
if (op?.op !== "replace_all_text" && op?.op !== "format_all") return { ok: false, status: "refused", reason: `${op?.op} is not a document-wide op` };
|
|
99
119
|
const st = await driver.open({ bytes, name, anchors });
|
package/src/ops/runs.js
CHANGED
|
@@ -19,25 +19,137 @@ export function layout(runs) {
|
|
|
19
19
|
return { text, segs };
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
/**
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
22
|
+
/**
|
|
23
|
+
* Case folding for find: each code point on its own, lowercased, with the final sigma folded to sigma. Per code point, because
|
|
24
|
+
* String.prototype.toLowerCase of a whole text can change its length (U+0130 becomes two units), which moved every offset found in
|
|
25
|
+
* the folded text away from the original. Each code point's expansion stays tied to that one character (see findRanges).
|
|
26
|
+
*/
|
|
27
|
+
export function foldCase(s) {
|
|
28
|
+
let out = "";
|
|
29
|
+
for (const ch of s) out += foldChar(ch);
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
const FOLDED = new Map(); // a character -> its folding (each distinct character is folded once)
|
|
33
|
+
const foldChar = (ch) => {
|
|
34
|
+
let f = FOLDED.get(ch);
|
|
35
|
+
if (f === undefined) FOLDED.set(ch, (f = ch === "ς" ? "σ" : ch.toLowerCase()));
|
|
36
|
+
return f;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Calls fn(start, end) for each non-overlapping match of `find` in `text`, [start, end) offsets INTO `text` (end - start is the
|
|
41
|
+
* matched length, which with match_case off is not always find.length: "İ" folds to two units). Matches are found in the folded text
|
|
42
|
+
* and mapped back. Rule: a match must begin and end between original characters; a folded match that starts or ends inside one
|
|
43
|
+
* character's expansion ("i" against the "i" + U+0307 that U+0130 folds to) is not reported, because replacing it would cut a
|
|
44
|
+
* character in half. Matching is leftmost first; after a rejected candidate the search resumes one unit later.
|
|
45
|
+
* Linear, with no record per match: the folded text and its map back to the original are built once (typed arrays, one entry per
|
|
46
|
+
* folded unit) and only when match_case is off. Ceiling: a pathological text matches millions of times (8 million characters of "İ "
|
|
47
|
+
* against "i̇": 4 million), each one a call of fn.
|
|
48
|
+
*/
|
|
49
|
+
export function eachMatch(text, find, matchCase, fn) {
|
|
50
|
+
if (!find) return;
|
|
51
|
+
if (matchCase) {
|
|
52
|
+
for (let at = text.indexOf(find); at >= 0; at = text.indexOf(find, at + find.length)) fn(at, at + find.length);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
const needle = foldCase(find);
|
|
56
|
+
if (!needle) return;
|
|
57
|
+
// pass 1: the folded length; ASCII folds to one unit of its own, anything else through foldChar
|
|
58
|
+
let n = 0;
|
|
59
|
+
let plain = true;
|
|
60
|
+
for (let k = 0; k < text.length; k++) {
|
|
61
|
+
const c = text.charCodeAt(k);
|
|
62
|
+
if (c < 128) n++;
|
|
63
|
+
else {
|
|
64
|
+
plain = false;
|
|
65
|
+
const cp = text.codePointAt(k);
|
|
66
|
+
const ch = String.fromCodePoint(cp);
|
|
67
|
+
n += foldChar(ch).length;
|
|
68
|
+
if (cp > 0xffff) k++;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if (plain) {
|
|
72
|
+
// nothing but ASCII: folding keeps every offset
|
|
73
|
+
const hay = text.toLowerCase();
|
|
74
|
+
for (let at = hay.indexOf(needle); at >= 0; at = hay.indexOf(needle, at + needle.length)) fn(at, at + needle.length);
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
const from = new Int32Array(n + 1); // folded unit -> offset in text of the character it came from
|
|
78
|
+
const head = new Uint8Array(n + 1); // 1 when it is the first unit of its character's folding
|
|
79
|
+
const parts = [];
|
|
80
|
+
let chunk = "";
|
|
81
|
+
let u = 0;
|
|
82
|
+
for (let k = 0; k < text.length; ) {
|
|
83
|
+
const c = text.charCodeAt(k);
|
|
84
|
+
let f;
|
|
85
|
+
let w = 1;
|
|
86
|
+
if (c < 128) f = c >= 65 && c <= 90 ? String.fromCharCode(c + 32) : text[k];
|
|
87
|
+
else {
|
|
88
|
+
const cp = text.codePointAt(k);
|
|
89
|
+
w = cp > 0xffff ? 2 : 1;
|
|
90
|
+
f = foldChar(text.slice(k, k + w));
|
|
91
|
+
}
|
|
92
|
+
for (let j = 0; j < f.length; j++) (from[u] = k), (head[u] = j === 0 ? 1 : 0), u++;
|
|
93
|
+
chunk += f;
|
|
94
|
+
if (chunk.length > 65536) (parts.push(chunk), (chunk = ""));
|
|
95
|
+
k += w;
|
|
96
|
+
}
|
|
97
|
+
parts.push(chunk);
|
|
98
|
+
from[n] = text.length;
|
|
99
|
+
head[n] = 1;
|
|
100
|
+
const hay = parts.join("");
|
|
101
|
+
for (let at = hay.indexOf(needle); at >= 0; ) {
|
|
102
|
+
const end = at + needle.length;
|
|
103
|
+
if (head[at] && head[end]) {
|
|
104
|
+
fn(from[at], from[end]);
|
|
105
|
+
at = hay.indexOf(needle, end);
|
|
106
|
+
} else at = hay.indexOf(needle, at + 1);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Non-overlapping matches of `find` in `text` as [{start, end}] (eachMatch's rule). */
|
|
111
|
+
export function findRanges(text, find, matchCase = true) {
|
|
26
112
|
const out = [];
|
|
27
|
-
|
|
113
|
+
eachMatch(text, find, matchCase, (start, end) => out.push({ start, end }));
|
|
28
114
|
return out;
|
|
29
115
|
}
|
|
30
116
|
|
|
117
|
+
/** Start offsets of the matches findRanges reports. */
|
|
118
|
+
export const findAll = (text, find, matchCase = true) => findRanges(text, find, matchCase).map((r) => r.start);
|
|
119
|
+
|
|
31
120
|
const WORD = /[\p{L}\p{N}]/u;
|
|
32
|
-
|
|
121
|
+
const WORDY = new Map(); // a code point -> is it a letter or digit (each distinct one tested once)
|
|
122
|
+
const wordAt = (text, k) => {
|
|
123
|
+
if (k < 0 || k >= text.length) return false;
|
|
124
|
+
const cp = text.codePointAt(k);
|
|
125
|
+
let w = WORDY.get(cp);
|
|
126
|
+
if (w === undefined) WORDY.set(cp, (w = WORD.test(String.fromCodePoint(cp))));
|
|
127
|
+
return w;
|
|
128
|
+
};
|
|
129
|
+
/** The code point that ends right before offset k (a surrogate pair read whole), as an offset where it starts; -1 at the start. */
|
|
130
|
+
const before = (text, k) => (k <= 0 ? -1 : k >= 2 && /[\udc00-\udfff]/.test(text[k - 1]) && /[\ud800-\udbff]/.test(text[k - 2]) ? k - 2 : k - 1);
|
|
131
|
+
/** Does a match [start, end) touch a letter or digit on either side (part of a longer word, "he" in "the")? */
|
|
132
|
+
const partial = (text, start, end) => wordAt(text, before(text, start)) || wordAt(text, end);
|
|
133
|
+
|
|
134
|
+
/** How many matches of `find` in `text` touch a letter or digit on either side. Same matches as findRanges, no record per match. */
|
|
33
135
|
export function partialWords(text, find, matchCase = true) {
|
|
136
|
+
return countMatches(text, find, matchCase).partial;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** {n, partial}: how many matches findRanges reports, and how many of them touch a letter or digit. One pass, no record per match. */
|
|
140
|
+
export function countMatches(text, find, matchCase = true) {
|
|
34
141
|
let n = 0;
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
if ((
|
|
39
|
-
}
|
|
40
|
-
return n;
|
|
142
|
+
let p = 0;
|
|
143
|
+
eachMatch(text, find, matchCase, (start, end) => {
|
|
144
|
+
n++;
|
|
145
|
+
if (partial(text, start, end)) p++;
|
|
146
|
+
});
|
|
147
|
+
return { n, partial: p };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** [{start, end}] sorted by start, from match starts and their length (one number for all, or one per start). */
|
|
151
|
+
export function rangesOf(starts, len) {
|
|
152
|
+
return starts.map((start, i) => ({ start, end: start + (Array.isArray(len) ? len[i] : len) })).sort((a, b) => a.start - b.start);
|
|
41
153
|
}
|
|
42
154
|
|
|
43
155
|
/** Picks the matches an op addresses: all, the nth (1-based), or the only one. Returns {starts} or {error}. */
|
|
@@ -65,6 +177,42 @@ export function replaceRange(runs, s, e, replacement) {
|
|
|
65
177
|
return dropEmpty(out, first.i);
|
|
66
178
|
}
|
|
67
179
|
|
|
180
|
+
/**
|
|
181
|
+
* replaceRange for every match at once: `starts` (non-overlapping, as findAll gives them) of `len` characters each (one number, or one
|
|
182
|
+
* length per start, as findRanges reports them) become `replacement`.
|
|
183
|
+
* The same result as replaceRange applied right to left, in one pass over the runs (that was a pass per match: a paragraph of 130,000
|
|
184
|
+
* runs matched in each held the server's thread for minutes). Throws RangeError if a match crosses an object or meets no run.
|
|
185
|
+
*/
|
|
186
|
+
export function replaceRanges(runs, starts, len, replacement) {
|
|
187
|
+
if (!starts.length) return runs;
|
|
188
|
+
const at = rangesOf(starts, len);
|
|
189
|
+
const { segs } = layout(runs);
|
|
190
|
+
const cuts = new Map(); // run index -> [{from, to, put}] in order
|
|
191
|
+
let g = 0;
|
|
192
|
+
let keep = -1;
|
|
193
|
+
for (const [k, { start: s, end: e }] of at.entries()) {
|
|
194
|
+
while (g < segs.length && segs[g].end <= s) g++;
|
|
195
|
+
let first = true;
|
|
196
|
+
for (let h = g; h < segs.length && segs[h].start < e; h++) {
|
|
197
|
+
const seg = segs[h];
|
|
198
|
+
if (!seg.text) throw new RangeError("the range crosses an image or other object");
|
|
199
|
+
(cuts.get(seg.i) || cuts.set(seg.i, []).get(seg.i)).push({ from: Math.max(s, seg.start) - seg.start, to: Math.min(e, seg.end) - seg.start, put: first ? replacement : "" });
|
|
200
|
+
if (first && k === 0) keep = seg.i;
|
|
201
|
+
first = false;
|
|
202
|
+
}
|
|
203
|
+
if (first) throw new RangeError("the range crosses an image or other object");
|
|
204
|
+
}
|
|
205
|
+
const out = runs.map((r, i) => {
|
|
206
|
+
const list = cuts.get(i);
|
|
207
|
+
if (!list) return { ...r };
|
|
208
|
+
let text = "";
|
|
209
|
+
let cur = 0;
|
|
210
|
+
for (const c of list) (text += r.text.slice(cur, c.from) + c.put), (cur = c.to);
|
|
211
|
+
return { ...r, text: text + r.text.slice(cur) };
|
|
212
|
+
});
|
|
213
|
+
return dropEmpty(out, keep);
|
|
214
|
+
}
|
|
215
|
+
|
|
68
216
|
/** Removes emptied text runs, but keeps `keep` if nothing else would carry the paragraph. */
|
|
69
217
|
function dropEmpty(runs, keep) {
|
|
70
218
|
const res = runs.filter((r, i) => !(isText(r) && isVisible(r) && r.text === "" && i !== keep));
|
|
@@ -101,9 +249,63 @@ export function splitForRange(runs, s, e) {
|
|
|
101
249
|
return { runs: out, covered };
|
|
102
250
|
}
|
|
103
251
|
|
|
252
|
+
/**
|
|
253
|
+
* splitForRange for several ranges at once: `ranges` [{start, end}] sorted and not overlapping. Returns {runs, covered: [indexes in
|
|
254
|
+
* the new runs, per range]}, the same runs as splitting for each range in turn, in one pass (a pass per range was quadratic).
|
|
255
|
+
* Null when a range is empty, or the ranges overlap or are out of order (the caller then splits one range at a time).
|
|
256
|
+
*/
|
|
257
|
+
export function splitForRanges(runs, ranges) {
|
|
258
|
+
for (let k = 0; k < ranges.length; k++) if (!(ranges[k].start < ranges[k].end) || (k && ranges[k].start < ranges[k - 1].end)) return null;
|
|
259
|
+
const out = [];
|
|
260
|
+
const covered = ranges.map(() => []);
|
|
261
|
+
let pos = 0;
|
|
262
|
+
let k = 0;
|
|
263
|
+
for (const r of runs) {
|
|
264
|
+
if (!isVisible(r)) {
|
|
265
|
+
out.push(r);
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
const len = isText(r) ? r.text.length : 1;
|
|
269
|
+
const a = pos;
|
|
270
|
+
const b = pos + len;
|
|
271
|
+
pos = b;
|
|
272
|
+
while (k < ranges.length && ranges[k].end <= a) k++;
|
|
273
|
+
let hit = false;
|
|
274
|
+
let cur = 0;
|
|
275
|
+
for (let j = k; j < ranges.length && ranges[j].start < b; j++) {
|
|
276
|
+
const { start: s, end: e } = ranges[j];
|
|
277
|
+
if (e <= a) continue;
|
|
278
|
+
if (!isText(r)) {
|
|
279
|
+
covered[j].push(out.length); // objects inside the range are covered whole
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
hit = true;
|
|
283
|
+
const from = Math.max(s, a) - a;
|
|
284
|
+
const to = Math.min(e, b) - a;
|
|
285
|
+
if (from > cur) out.push({ ...r, text: r.text.slice(cur, from) });
|
|
286
|
+
if (to > from) {
|
|
287
|
+
covered[j].push(out.length);
|
|
288
|
+
out.push({ ...r, text: r.text.slice(from, to) });
|
|
289
|
+
}
|
|
290
|
+
cur = to;
|
|
291
|
+
}
|
|
292
|
+
if (!hit) out.push(r);
|
|
293
|
+
else if (cur < len) out.push({ ...r, text: r.text.slice(cur) });
|
|
294
|
+
}
|
|
295
|
+
return { runs: out, covered };
|
|
296
|
+
}
|
|
297
|
+
|
|
104
298
|
/** Merges a run-property patch into the `d` (direct formatting) of the covered text runs. */
|
|
105
299
|
export function applySet(runs, covered, patch) {
|
|
106
|
-
|
|
300
|
+
const at = new Set(covered); // includes() per run was quadratic on a paragraph of many runs
|
|
301
|
+
return runs.map((r, i) => (at.has(i) && isText(r) ? { ...r, d: { ...(r.d || {}), ...patch } } : r));
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/** applySet for several ranges at once: `covered` from splitForRanges, `patches` one per range. */
|
|
305
|
+
export function applySets(runs, covered, patches) {
|
|
306
|
+
const at = new Map();
|
|
307
|
+
covered.forEach((list, j) => list.forEach((i) => at.set(i, { ...(at.get(i) || {}), ...patches[j] })));
|
|
308
|
+
return runs.map((r, i) => (at.has(i) && isText(r) ? { ...r, d: { ...(r.d || {}), ...at.get(i) } } : r));
|
|
107
309
|
}
|
|
108
310
|
|
|
109
311
|
/** Sets the text of a whole paragraph: the first visible text run carries it, the other visible text runs go. Objects stay where they are (callers refuse those). */
|
package/src/ops/schema.js
CHANGED
|
@@ -62,10 +62,17 @@ function check(value, schema, at, errors) {
|
|
|
62
62
|
}
|
|
63
63
|
|
|
64
64
|
const DOC_OPS = new Set(["replace_text", "insert_paragraph_after", "insert_paragraph_before", "delete_paragraph", "set_paragraph_style", "format_text"]);
|
|
65
|
+
const HTML_OPS = new Set(["set_text", "set_attribute", "remove_attribute", "insert_html", "replace_element", "delete_element"]);
|
|
66
|
+
const HTML_TARGET = ["anchor_id", "path", "element_id"];
|
|
65
67
|
|
|
66
68
|
/** Rules JSON Schema cannot say without heavy machinery: exclusive field groups and what `expect` must contain. */
|
|
67
69
|
function semantic(op, at, errors) {
|
|
68
|
-
|
|
70
|
+
const html = HTML_OPS.has(op.op) || (op.op === "replace_text" && ("path" in (op.target || {}) || "element_id" in (op.target || {})));
|
|
71
|
+
if (html) {
|
|
72
|
+
// an element of an HTML file: exactly one of anchor_id, path, element_id (replace_text names one of the last two for HTML)
|
|
73
|
+
const t = op.target || {};
|
|
74
|
+
if (HTML_TARGET.filter((k) => k in t).length !== 1 || Object.keys(t).some((k) => !HTML_TARGET.includes(k))) errors.push(`${at}.target: give exactly one of anchor_id, path or element_id`);
|
|
75
|
+
} else if (DOC_OPS.has(op.op)) {
|
|
69
76
|
const t = op.target || {};
|
|
70
77
|
const n = ["anchor_id", "para_id", "body_path", "text_hash"].filter((k) => k in t);
|
|
71
78
|
if (!n.length) errors.push(`${at}.target: give anchor_id, or para_id, or body_path with text_hash, or text_hash`);
|
package/src/ops/xlsx.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { checkExpect, colIndex, colLetters, engineFailure, fail, parseA1, refused } from "./common.js";
|
|
2
2
|
import { openEngine } from "./engine.js";
|
|
3
|
-
import {
|
|
3
|
+
import { countMatches, findRanges } from "./runs.js";
|
|
4
4
|
import { a1Range, parseWorkbook } from "../xlsx.js";
|
|
5
5
|
|
|
6
6
|
/**
|
|
@@ -105,7 +105,7 @@ export async function bind(st, op) {
|
|
|
105
105
|
const hits = await allMatches(st, op);
|
|
106
106
|
const total = hits.reduce((n, x) => n + x.n, 0);
|
|
107
107
|
// `t` is the formula when a cell has one, so a hit whose text starts with "=" would rewrite a formula
|
|
108
|
-
const extra = { partial_words: hits.reduce((n, x) => n +
|
|
108
|
+
const extra = { partial_words: hits.reduce((n, x) => n + x.partial, 0), in_formulas: hits.reduce((n, x) => n + (x.t.startsWith("=") ? x.n : 0), 0) };
|
|
109
109
|
if (total !== op.expect.count) return fail("expect_mismatch", `found ${total} occurrence(s), expected ${op.expect.count}`, { found: total, ...extra });
|
|
110
110
|
Object.assign(b, extra);
|
|
111
111
|
b.hits = hits;
|
|
@@ -135,8 +135,10 @@ function bindShape(st, op) {
|
|
|
135
135
|
if (op.op === "set_cells") {
|
|
136
136
|
// a merged range shows only its top-left cell: a value in any other cell of it is hidden from the person (an empty write is harmless)
|
|
137
137
|
const flat = Array.isArray(op.values) ? op.values : [[op.values]];
|
|
138
|
+
const written = flat.flatMap((row, i) => row.map((v, j) => ({ r: rect.r0 + i, c: rect.c0 + j, v }))).filter((x) => input(x.v) !== ""); // once, not per range
|
|
138
139
|
for (const m of mergesOf(st, target.sheet)) {
|
|
139
|
-
|
|
140
|
+
if (m.r1 < rect.r0 || m.r0 > rect.r1 || m.c1 < rect.c0 || m.c0 > rect.c1) continue; // a range outside the write: nothing of it is hit
|
|
141
|
+
const hit = written.find((x) => x.r >= m.r0 && x.r <= m.r1 && x.c >= m.c0 && x.c <= m.c1 && !(x.r === m.r0 && x.c === m.c0));
|
|
140
142
|
if (hit) return refused("merged_cell", `${colLetters(hit.c)}${hit.r + 1} is inside the merged range ${a1Range(m)}, which shows only its top-left cell ${colLetters(m.c0)}${m.r0 + 1}: write there, or pick a cell outside the range`);
|
|
141
143
|
}
|
|
142
144
|
}
|
|
@@ -199,8 +201,8 @@ async function allMatches(st, op) {
|
|
|
199
201
|
for (const row of b.rows)
|
|
200
202
|
for (const c of row.cells) {
|
|
201
203
|
const t = await cellText(st, sh, row.r, c[0]);
|
|
202
|
-
const n =
|
|
203
|
-
if (n) out.push({ sh, r: row.r, c: c[0], n, t });
|
|
204
|
+
const { n, partial } = countMatches(t, op.find, op.match_case !== false);
|
|
205
|
+
if (n) out.push({ sh, r: row.r, c: c[0], n, partial, t });
|
|
204
206
|
}
|
|
205
207
|
}
|
|
206
208
|
}
|
|
@@ -215,8 +217,11 @@ export async function exec(st, b) {
|
|
|
215
217
|
const total = b.hits.reduce((n, x) => n + x.n, 0);
|
|
216
218
|
const byFormula = [];
|
|
217
219
|
for (const x of b.hits) {
|
|
218
|
-
|
|
219
|
-
|
|
220
|
+
// the same matches the count above came from (findRanges), not a second reading of "case-insensitive" by a RegExp
|
|
221
|
+
let next = "";
|
|
222
|
+
let cur = 0;
|
|
223
|
+
for (const { start, end } of findRanges(x.t, op.find, op.match_case !== false)) (next += x.t.slice(cur, start) + op.replace), (cur = end);
|
|
224
|
+
next += x.t.slice(cur);
|
|
220
225
|
byFormula.push({ sh: x.sh, r: x.r, c: x.c, input: next });
|
|
221
226
|
}
|
|
222
227
|
for (const sh of new Set(byFormula.map((x) => x.sh))) await st.h.engine.call("xlsx.set", [sh, byFormula.filter((x) => x.sh === sh).map(({ r, c, input }) => ({ r, c, input }))]);
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { env } from "./constants.js";
|
|
2
|
+
import "./html-parser.js"; // an HTML file's ops run here, never on the server's or the CLI's thread (test/html-parser-thread.test.js)
|
|
3
|
+
import { previewAll, runOps } from "./ops/run.js";
|
|
4
|
+
|
|
5
|
+
// The engine's work for a quick change, off the server's thread (src/offload.js opsOffThread), run by src/worker-host.js within a time
|
|
6
|
+
// and memory limit: the count a quick preview and a quick plan show (`preview`: previewAll) and the change itself made on this Mac
|
|
7
|
+
// (`run`: runOps, which the local runner's apply calls through src/local.js). An outside review: a 12,538-byte document of one
|
|
8
|
+
// paragraph of 8 million characters held the server's thread for 4 s while its matches were counted. One message back: {ok: true, p} |
|
|
9
|
+
// {ok: true, r: runOps' answer}. ONIONSKIN_TEST_OPS_DELAY_MS (tests only) blocks this thread for that long first, the way a slow count
|
|
10
|
+
// would.
|
|
11
|
+
export async function work({ kind, bytes, format, op, ops, name, partial, anchors }) {
|
|
12
|
+
const delay = Number(env("TEST_OPS_DELAY_MS", 0));
|
|
13
|
+
if (delay > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, delay);
|
|
14
|
+
const b = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
15
|
+
if (kind === "preview") return { ok: true, p: await previewAll(b, format, op, { name }) };
|
|
16
|
+
return { ok: true, r: await runOps({ bytes: b, format, name, ops, partial, anchors: new Map(anchors || []) }) };
|
|
17
|
+
}
|