onionskin-cli 0.0.2 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -0
- package/INSTALL.md +5 -1
- package/NOTICE +34 -0
- package/README.md +101 -33
- package/docs/contracts/batch.schema.json +27 -4
- package/docs/contracts/ops.schema.json +117 -5
- package/package.json +4 -2
- package/skills/onionskin/SKILL.md +9 -6
- package/skills/onionskin/reference/batch.md +5 -3
- package/skills/onionskin/reference/formats.md +5 -1
- package/skills/onionskin/reference/ops.md +47 -3
- package/skills/onionskin/reference/troubleshooting.md +31 -0
- package/src/agent-detect.js +28 -2
- package/src/agent-prompt.js +12 -1
- package/src/agent-runner.js +77 -19
- package/src/anchor-work.js +154 -0
- package/src/anchor-worker.js +38 -0
- package/src/anchors.js +61 -11
- package/src/apply.js +119 -47
- package/src/batch.js +32 -8
- package/src/check-worker.js +8 -5
- package/src/cli.js +91 -25
- package/src/constants.js +34 -1
- package/src/diff.js +3 -1
- package/src/docx.js +368 -58
- package/src/edit-hash.js +19 -0
- package/src/fsutil.js +1 -1
- package/src/history-worker.js +8 -8
- package/src/history.js +135 -31
- package/src/html-active.js +384 -0
- package/src/html-envelope.js +51 -0
- package/src/html-parser.js +8 -0
- package/src/html-render-worker.js +25 -0
- package/src/html-render.js +880 -0
- package/src/html-utf8.js +65 -0
- package/src/html-write-worker.js +14 -0
- package/src/html-write.js +380 -0
- package/src/html.js +1257 -0
- package/src/local.js +68 -4
- package/src/merge-package.js +1110 -0
- package/src/merge-pptx.js +98 -38
- package/src/merge-proof.js +1019 -0
- package/src/merge-worker.js +60 -0
- package/src/merge-xlsx.js +84 -29
- package/src/merge.js +286 -83
- package/src/nfc.js +46 -0
- package/src/notes.js +122 -0
- package/src/offload.js +400 -1
- package/src/open-worker.js +27 -0
- package/src/ops/common.js +2 -1
- package/src/ops/docx.js +58 -29
- package/src/ops/html.js +394 -0
- package/src/ops/pptx.js +74 -21
- package/src/ops/run.js +23 -3
- package/src/ops/runs.js +215 -13
- package/src/ops/schema.js +8 -1
- package/src/ops/xlsx.js +12 -7
- package/src/ops-worker.js +17 -0
- package/src/package.js +26 -2
- package/src/person.js +8 -1
- package/src/pptx.js +85 -21
- package/src/quick-plan.js +13 -6
- package/src/redline-worker.js +10 -14
- package/src/reply.js +46 -19
- package/src/retain.js +223 -0
- package/src/review.js +32 -18
- package/src/rewrite-check.js +2 -2
- package/src/sanitize.js +16 -3
- package/src/server.js +88 -37
- package/src/session.js +1143 -208
- package/src/stop-gate.js +34 -0
- package/src/suggest-reply.js +42 -7
- package/src/suggest.js +14 -29
- package/src/text.js +161 -11
- package/src/update.js +288 -0
- package/src/voice-guide.js +4 -3
- package/src/voice-learn.js +4 -3
- package/src/voice-sample-worker.js +7 -6
- package/src/voice-worker.js +4 -6
- package/src/watch.js +188 -0
- package/src/worker-host.js +23 -0
- package/src/xlsx.js +355 -94
- package/src/xml-tree.js +6 -3
- package/src/xml.js +82 -34
- package/src/zip-write.js +2 -1
- package/src/zip.js +9 -5
- package/vendor/parse5/LICENSE.txt +32 -0
- package/vendor/parse5/parse5.mjs +7822 -0
- package/vendor/parse5.lock.json +23 -0
- package/web/dist/assets/index-DkNx0LSH.css +1 -0
- package/web/dist/assets/{index-BUtofdtr.js → index-DytKkMdi.js} +117 -107
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-Dfizlg0i.css +0 -1
package/src/merge.js
CHANGED
|
@@ -1,26 +1,28 @@
|
|
|
1
1
|
import { fmtNote } from "./fmt.js";
|
|
2
|
-
import { docxFmt, lcsPairs, matchParagraphs } from "./history.js";
|
|
2
|
+
import { docxFmt, lcsPairs, matchParagraphs, paraIdentity } from "./history.js";
|
|
3
3
|
import { readDocxParagraphs } from "./docx.js";
|
|
4
4
|
import { normText } from "./text.js";
|
|
5
5
|
|
|
6
6
|
// What the user did while the agent worked, carried onto the agent's version. Three versions of one DOCX: `base` (what the page
|
|
7
7
|
// loaded: the file as sent), `mine` (base plus the user's edits, saved by the page) and `theirs` (the file on disk after the agent).
|
|
8
8
|
// A paragraph only the user changed becomes an op on `theirs`, run through the same engine, expect check and package check as an
|
|
9
|
-
// agent's `apply`.
|
|
10
|
-
// (`unplaced`), so nothing typed is lost without a word.
|
|
9
|
+
// agent's `apply`. Text is compared as written: a line break, a tab or a second space is an edit. A paragraph both sides changed is
|
|
10
|
+
// never guessed at: the agent's text stays and the user's text is handed back (`unplaced`), so nothing typed is lost without a word.
|
|
11
11
|
// Formatting the user changed without changing the text (run properties by character, and the paragraph's style, alignment and
|
|
12
12
|
// list) is set on the agent's paragraph when the agent left that formatting alone; on a paragraph the agent rewrote only the
|
|
13
|
-
// paragraph's own style, alignment and list are set. A body paragraph the user moved is moved in the agent's version
|
|
14
|
-
// agent left it alone and the paragraph next to its new place is still there. Each of these is
|
|
15
|
-
// kept: if one did not come out as planned, the merge runs again without any of them and says so.
|
|
13
|
+
// paragraph's own style, alignment and list are set. A body paragraph the user moved (edited or not) is moved in the agent's version
|
|
14
|
+
// when the agent left it alone, moved no paragraph itself, and the paragraph next to its new place is still there. Each of these is
|
|
15
|
+
// proven on the result before it is kept: if one did not come out as planned, the merge runs again without any of them and says so.
|
|
16
|
+
// Then the proof of the merge (src/merge-proof.js, in src/merge-worker.js) checks the result on its own: every entry of `unplaced` names
|
|
17
|
+
// what it stands for (`of`), and a result that does not account for every edit is not returned.
|
|
16
18
|
// Ceilings: pictures and anything outside paragraphs are not carried. Not carried and reported: formatting of a kind the engine
|
|
17
19
|
// cannot set (character style, theme color, shading, spacing, indent), moves inside tables, a moved paragraph that ends a section or
|
|
18
|
-
// holds one end of a comment or bookmark, and formatting changed on a paragraph the user
|
|
19
|
-
// as it is; the report names what the user changed inside it when that is certain, and asks
|
|
20
|
-
// reads differently, which every cut and paste in the editor does).
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
// only in the part before the first changed character.
|
|
20
|
+
// holds one end of a comment or bookmark, a move while the agent moved paragraphs too, and formatting changed on a paragraph the user
|
|
21
|
+
// also moved (the agent's paragraph is moved as it is; the report names what the user changed inside it when that is certain, and asks
|
|
22
|
+
// to check the formatting whenever the user's copy reads differently, which every cut and paste in the editor does). Carried, with a
|
|
23
|
+
// line asking to check: the text of a paragraph the user added whose formatting the result does not have (the engine gives it the
|
|
24
|
+
// paragraph formatting of its neighbor), and text typed into a paragraph whose formatting inside then differs from the user's. A
|
|
25
|
+
// paragraph the user split keeps its run formatting only in the part before the first changed character.
|
|
24
26
|
|
|
25
27
|
const CAP = 2000;
|
|
26
28
|
const cut = (t) => String(t ?? "").slice(0, CAP);
|
|
@@ -33,7 +35,7 @@ const label = (p) => {
|
|
|
33
35
|
* The smallest `find` / `replace` that turns `before` into `after` and occurs exactly once in `before` (so run formatting around it
|
|
34
36
|
* is kept), or null when there is none within the op's limits (then the whole paragraph text is set).
|
|
35
37
|
*/
|
|
36
|
-
export function minimalReplace(before, after) {
|
|
38
|
+
export function minimalReplace(before, after, shared = null) {
|
|
37
39
|
let p = 0;
|
|
38
40
|
const max = Math.min(before.length, after.length);
|
|
39
41
|
while (p < max && before[p] === after[p]) p++;
|
|
@@ -41,16 +43,27 @@ export function minimalReplace(before, after) {
|
|
|
41
43
|
while (s < max - p && before[before.length - 1 - s] === after[after.length - 1 - s]) s++;
|
|
42
44
|
let a = p;
|
|
43
45
|
let b = before.length - s; // [a, b) of `before` is replaced
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
46
|
+
// only uniqueness matters: stop at the second match (counting every overlapping match of each widening was quadratic: a run of
|
|
47
|
+
// 20,000 "a" with one letter typed in the middle held the server's thread for more than 5 seconds in an outside review). Each
|
|
48
|
+
// search still spans up to the whole text when the second match is far away, so the text the searches span is budgeted: past
|
|
49
|
+
// 16 times the paragraph (plus a floor), null, and the whole paragraph text is set instead (its run formatting is not kept).
|
|
50
|
+
// `shared` ({left}) is the budget of a whole merge: the floor is per paragraph, so 3,900 paragraphs of 2,000 "a" with one letter typed
|
|
51
|
+
// in each spent 5 s of the merge's time in an outside review. Past it, every later paragraph is set whole as well.
|
|
52
|
+
let budget = 2_000_000 + 16 * before.length;
|
|
53
|
+
const once = (needle) => {
|
|
54
|
+
const i = before.indexOf(needle);
|
|
55
|
+
if (i < 0) return false;
|
|
56
|
+
const j = before.indexOf(needle, i + 1);
|
|
57
|
+
const span = j < 0 ? before.length : j + needle.length; // what the two searches spanned
|
|
58
|
+
budget -= span;
|
|
59
|
+
if (shared) shared.left -= span;
|
|
60
|
+
return j < 0;
|
|
48
61
|
};
|
|
49
62
|
// widen (left first, then right) until the find text is not empty and occurs once
|
|
50
|
-
while ((b === a ||
|
|
63
|
+
while ((b === a || !once(before.slice(a, b))) && (a > 0 || b < before.length)) {
|
|
51
64
|
if (a > 0) a--;
|
|
52
65
|
else b++;
|
|
53
|
-
if (b - a > CAP) return null;
|
|
66
|
+
if (b - a > CAP || budget < 0 || shared?.left < 0) return null;
|
|
54
67
|
}
|
|
55
68
|
if (b === a) return null; // an empty paragraph has nothing to find
|
|
56
69
|
const find = before.slice(a, b);
|
|
@@ -136,6 +149,56 @@ function paraPatch(from, to) {
|
|
|
136
149
|
return { set, no };
|
|
137
150
|
}
|
|
138
151
|
|
|
152
|
+
/**
|
|
153
|
+
* Are two paragraphs in the same place: both in the body, or in the same cell (row and column) of the same table? A table is its order
|
|
154
|
+
* among the body's tables (`tableNo`; its body index shifts with every paragraph added above it). Two cells 0 0 of two tables are two
|
|
155
|
+
* places (an outside review: a paragraph moved from the first table to the second was edited in the first and nothing was said).
|
|
156
|
+
*/
|
|
157
|
+
const sameBox = (a, b) => (a.path?.length ?? 0) === (b.path?.length ?? 0) && (a.path?.length !== 4 || (a.tableNo === b.tableNo && a.path[1] === b.path[1] && a.path[2] === b.path[2]));
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Pairs, in place, a base paragraph `mapA` leaves unpaired alone between two paired neighbors with the first unpaired paragraph of B
|
|
161
|
+
* between their partners, when the two are in the same place (sameBox) and the ids allow it (`may`). Linear.
|
|
162
|
+
*/
|
|
163
|
+
function pairSlots(A, B, mapA, mapB, may) {
|
|
164
|
+
let i0 = -1;
|
|
165
|
+
for (let i = 0; i <= A.length; i++) {
|
|
166
|
+
if (i < A.length && mapA[i] < 0) continue;
|
|
167
|
+
const j0 = i0 < 0 ? -1 : mapA[i0];
|
|
168
|
+
const j1 = i < A.length ? mapA[i] : B.length;
|
|
169
|
+
// one paragraph rewritten (a long run of rewritten paragraphs stays with the matcher: see "every one of 20,000 paragraphs rewritten")
|
|
170
|
+
const b = j0 + 1;
|
|
171
|
+
if (i - i0 === 2 && b < j1 && mapB[b] < 0 && sameBox(A[i0 + 1], B[b]) && may(i0 + 1, b)) (mapA[i0 + 1] = b), (mapB[b] = i0 + 1);
|
|
172
|
+
i0 = i;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* The indexes j of `map` (index -> paired index, -1: none) that are out of order: those outside one longest run whose paired indexes
|
|
178
|
+
* increase (a longest increasing subsequence, n log n). The proof of the merge (src/merge-proof.js) finds moves on its own.
|
|
179
|
+
*/
|
|
180
|
+
function outOfOrder(map) {
|
|
181
|
+
const js = [];
|
|
182
|
+
for (let j = 0; j < map.length; j++) if (map[j] >= 0) js.push(j);
|
|
183
|
+
const tails = []; // tails[len - 1]: the index in js of the smallest last value of a run of length len
|
|
184
|
+
const back = new Int32Array(js.length).fill(-1);
|
|
185
|
+
for (let k = 0; k < js.length; k++) {
|
|
186
|
+
const v = map[js[k]];
|
|
187
|
+
let lo = 0;
|
|
188
|
+
let hi = tails.length;
|
|
189
|
+
while (lo < hi) {
|
|
190
|
+
const mid = (lo + hi) >> 1;
|
|
191
|
+
if (map[js[tails[mid]]] < v) lo = mid + 1;
|
|
192
|
+
else hi = mid;
|
|
193
|
+
}
|
|
194
|
+
tails[lo] = k;
|
|
195
|
+
back[k] = lo > 0 ? tails[lo - 1] : -1;
|
|
196
|
+
}
|
|
197
|
+
const out = new Set(js);
|
|
198
|
+
for (let k = tails.length ? tails[tails.length - 1] : -1; k >= 0; k = back[k]) out.delete(js[k]);
|
|
199
|
+
return out;
|
|
200
|
+
}
|
|
201
|
+
|
|
139
202
|
/**
|
|
140
203
|
* Plans the merge. Arguments are paragraph lists (readDocxParagraphs with detail). Returns
|
|
141
204
|
* {items: [{kind, label, text, t, op, ...}], unplaced: [{label, text, why}]}: every item is tried on `theirs`. `t` is the index in
|
|
@@ -143,29 +206,70 @@ function paraPatch(from, to) {
|
|
|
143
206
|
* a `note` (with `check`, see `lost`) when its formatting in `mine` is not the original's.
|
|
144
207
|
*/
|
|
145
208
|
export function planDocxMerge(base, mine, theirs) {
|
|
146
|
-
|
|
147
|
-
//
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
209
|
+
// Which paragraph of the person's version, and of the agent's, is base paragraph i: one rule for both sides and for History
|
|
210
|
+
// (paraIdentity in src/history.js). A paragraph id unique in its version is the paragraph: a match by text and position never pairs
|
|
211
|
+
// it with a paragraph that carries another id (an outside review: the person retyped paragraph 1 to read as paragraph 2 and edited
|
|
212
|
+
// paragraph 2; text matching took the new paragraph 1 for paragraph 2, and the edit of paragraph 1 was lost). On the agent's side
|
|
213
|
+
// the rule is strict: a base paragraph whose id the agent's version, which has ids, no longer has was removed or rebuilt, and an
|
|
214
|
+
// edit of it is handed back. If several of the agent's paragraphs carry the id (one copied with it), only a text match that carries
|
|
215
|
+
// it counts. Without ids on either side, text and position alone decide, as before. In doubt the edit is handed back (unplaced),
|
|
216
|
+
// never applied to a paragraph the person did not edit (an outside review: an edit of the first paragraph overwrote the second one,
|
|
217
|
+
// whose text the agent had replaced with the first one's).
|
|
218
|
+
const { mapA: baseToMine, mapB: mineToBase, moved: same } = matchParagraphs(base, mine);
|
|
219
|
+
// A paragraph rewritten whole between two paragraphs the person left in place faces the first new paragraph between their copies: in
|
|
220
|
+
// the same place (body or table cell), it is that paragraph, edited (a short text, such as a table cell's, has no similarity to go
|
|
221
|
+
// by: "Item" retyped as "Mine" was a deletion and a new paragraph, and a line added under it went into another cell)
|
|
222
|
+
pairSlots(base, mine, baseToMine, mineToBase, paraIdentity(base, mine, { strict: true }).may);
|
|
223
|
+
// Which of the person's paragraphs moved: those the matcher found elsewhere with the same text, and of the others those outside a
|
|
224
|
+
// longest run that keeps the base order. A paragraph the person also edited counts like any other (an outside review: base [A, B, C,
|
|
225
|
+
// D], the person moved A after C and edited it, and the move was dropped without a word; so was the order of two paragraphs swapped
|
|
226
|
+
// and both edited).
|
|
227
|
+
const moved = new Set([...same, ...outOfOrder(Int32Array.from(mineToBase, (i, j) => (same.has(j) ? -1 : i)))]);
|
|
228
|
+
const theirSide = paraIdentity(base, theirs, { strict: true });
|
|
229
|
+
/** may paragraph n of the agent's version be base paragraph i, as far as the ids say (the rule above)? */
|
|
230
|
+
const agrees = theirSide.may;
|
|
231
|
+
// a base paragraph the agent left alone, by position: the same text in the same order, ids told apart
|
|
232
|
+
const { mapA: baseToTheirs } = lcsPairs(theirSide.keyA, theirSide.keyB);
|
|
233
|
+
const agent = matchParagraphs(base, theirs, { strict: true }); // where the agent's other paragraphs came from: an edited one, one it moved
|
|
234
|
+
// the agent changed the order of paragraphs too: then which order wins is not certain, and no move of the person's is carried
|
|
235
|
+
const agentMoved = outOfOrder(agent.mapA).size > 0;
|
|
236
|
+
const ids = (L) => {
|
|
237
|
+
const m = new Map(); // id -> the indexes of the paragraphs that carry it
|
|
238
|
+
L.forEach((p, n) => p.paraId && (m.get(p.paraId) || m.set(p.paraId, []).get(p.paraId)).push(n));
|
|
239
|
+
return m;
|
|
240
|
+
};
|
|
241
|
+
const baseIds = ids(base);
|
|
242
|
+
const theirIds = ids(theirs);
|
|
243
|
+
const ownId = (i) => (base[i].paraId && baseIds.get(base[i].paraId).length === 1 ? base[i].paraId : null);
|
|
244
|
+
/** base paragraph i where it stands in the agent's version (same text, same order), when the ids agree; else -1 */
|
|
245
|
+
const inPlace = (i) => {
|
|
246
|
+
const t = baseToTheirs[i];
|
|
247
|
+
return t >= 0 && theirs[t].path && agrees(i, t) ? t : -1;
|
|
248
|
+
};
|
|
249
|
+
/** inPlace(i), else the paragraph with its unique id when the agent moved it and left its text alone; else -1 */
|
|
250
|
+
const at = (i) => {
|
|
251
|
+
const t = inPlace(i);
|
|
252
|
+
if (t >= 0) return t;
|
|
253
|
+
const n = ownId(i) ? theirIds.get(ownId(i)) : null;
|
|
254
|
+
return n?.length === 1 && theirs[n[0]].path && theirs[n[0]].hash === base[i].hash ? n[0] : -1;
|
|
255
|
+
};
|
|
151
256
|
const locator = (n) => ({ body_path: theirs[n].path, text_hash: theirs[n].hash });
|
|
152
|
-
const
|
|
257
|
+
const work = { left: 4_000_000 + 16 * base.reduce((n, p) => n + p.text.length, 0) }; // minimalReplace over the whole plan
|
|
153
258
|
const items = [];
|
|
154
259
|
const unplaced = [];
|
|
155
|
-
|
|
260
|
+
// `of` names the paragraph an entry stands for, for the proof of the merge (src/merge-proof.js): "m" and the paragraph's ordinal in
|
|
261
|
+
// the person's version, or "b" and its ordinal in the base
|
|
262
|
+
const ref = (p) => `${base[p.ordinal] === p ? "b" : "m"}${p.ordinal}`;
|
|
263
|
+
const no = (p, why, text = p.text, of = [ref(p)]) => unplaced.push({ label: label(p), text: cut(text), why, of });
|
|
156
264
|
|
|
157
265
|
// The agent's version of a base paragraph whose text it changed, only when there is no doubt which one it is: the same unique
|
|
158
|
-
// paragraph id, or the one paragraph between the same two untouched neighbors.
|
|
159
|
-
const ids = (L) => L.reduce((m, p) => (p.paraId ? m.set(p.paraId, (m.get(p.paraId) || 0) + 1) : m), new Map());
|
|
160
|
-
let idCount = null;
|
|
266
|
+
// paragraph id, or the one paragraph between the same two untouched neighbors (and never one the ids say is another paragraph).
|
|
161
267
|
const rewritten = (i) => {
|
|
162
268
|
const n = agent.mapA[i];
|
|
163
|
-
if (n < 0 || baseToTheirs[i] >= 0 || agent.moved.has(n) || !theirs[n].path) return -1;
|
|
164
|
-
|
|
165
|
-
const
|
|
166
|
-
|
|
167
|
-
const before = i === 0 ? n === 0 : baseToTheirs[i - 1] >= 0 && baseToTheirs[i - 1] === n - 1;
|
|
168
|
-
const after = i === base.length - 1 ? n === theirs.length - 1 : baseToTheirs[i + 1] >= 0 && baseToTheirs[i + 1] === n + 1;
|
|
269
|
+
if (n < 0 || baseToTheirs[i] >= 0 || agent.moved.has(n) || !theirs[n].path || !agrees(i, n)) return -1;
|
|
270
|
+
if (theirIds.get(ownId(i))?.length === 1) return n; // the agent's version may have no ids at all (an outside review: this threw)
|
|
271
|
+
const before = i === 0 ? n === 0 : inPlace(i - 1) >= 0 && inPlace(i - 1) === n - 1;
|
|
272
|
+
const after = i === base.length - 1 ? n === theirs.length - 1 : inPlace(i + 1) >= 0 && inPlace(i + 1) === n + 1;
|
|
169
273
|
return before && after ? n : -1;
|
|
170
274
|
};
|
|
171
275
|
|
|
@@ -186,7 +290,7 @@ export function planDocxMerge(base, mine, theirs) {
|
|
|
186
290
|
const mineWords = [...("style" in para.set ? ["style"] : []), ...("align" in para.set ? ["alignment"] : []), ...("list" in para.set || "ilvl" in para.set ? ["list"] : []), ...words];
|
|
187
291
|
if (n < 0) {
|
|
188
292
|
if (typed) return; // the text was handed back with its own reason
|
|
189
|
-
return no(m, `You changed formatting here${mineWords.length ? ` (${list(mineWords)})` : ""}, but the agent changed, moved or removed this paragraph, so it was not carried over.`, "");
|
|
293
|
+
return no(m, `You changed formatting here${mineWords.length ? ` (${list(mineWords)})` : ""}, but the agent changed, moved or removed this paragraph, so it was not carried over.`, "", [ref(m), ref(b)]);
|
|
190
294
|
}
|
|
191
295
|
const a = theirs[n];
|
|
192
296
|
const carry = {};
|
|
@@ -208,8 +312,8 @@ export function planDocxMerge(base, mine, theirs) {
|
|
|
208
312
|
else notes.push(`The agent changed formatting inside this paragraph too, so the ${list(words)} you changed was not carried over.`);
|
|
209
313
|
} else if (vague) notes.push(`Formatting inside this paragraph is different in your version (${vague}). It could not be carried over.`);
|
|
210
314
|
if (cannot.length) notes.push(`Changes to ${list(cannot)} are not carried over.`);
|
|
211
|
-
if (!done.length) return no(m, notes.join(" "), "");
|
|
212
|
-
items.push({ kind: "format", label: label(m), text: "", hash: null, t: n, words: done, note: notes.join(" "), op: { op: "format_paragraph", target: locator(n), expect: { text_hash: a.hash }, ...(Object.keys(carry).length ? { para: carry } : {}), ...(runs.length ? { runs } : {}) } });
|
|
315
|
+
if (!done.length) return no(m, notes.join(" "), "", [ref(m), ref(b)]);
|
|
316
|
+
items.push({ kind: "format", label: label(m), of: [ref(m), ref(b)], text: "", hash: null, t: n, words: done, note: notes.join(" "), op: { op: "format_paragraph", target: locator(n), expect: { text_hash: a.hash }, ...(Object.keys(carry).length ? { para: carry } : {}), ...(runs.length ? { runs } : {}) } });
|
|
213
317
|
};
|
|
214
318
|
|
|
215
319
|
/**
|
|
@@ -228,11 +332,31 @@ export function planDocxMerge(base, mine, theirs) {
|
|
|
228
332
|
return { note: "This paragraph has the formatting of the agent's version: if you changed its formatting too, set it again.", check: true };
|
|
229
333
|
};
|
|
230
334
|
|
|
231
|
-
// anchors for new paragraphs: the nearest paragraph before (else after) that the agent left alone
|
|
232
|
-
//
|
|
335
|
+
// anchors for new paragraphs: the nearest paragraph before (else after) that the agent left alone, or rewrote when there is no doubt
|
|
336
|
+
// which paragraph of its version that is (an anchor past the agent's edited neighbor put the new paragraph on the wrong side of it).
|
|
337
|
+
// The user may have edited it (a split leaves the first half as an edit of the original): every op is bound to the agent's file
|
|
338
|
+
// before any op runs.
|
|
233
339
|
const still = (j) => mineToBase[j] >= 0 && !moved.has(j); // a paragraph of the base that is where it was
|
|
234
|
-
const
|
|
340
|
+
const placed = (i) => {
|
|
341
|
+
const t = inPlace(i);
|
|
342
|
+
if (t >= 0) return t;
|
|
343
|
+
const n = ownId(i) ? theirIds.get(ownId(i)) : null;
|
|
344
|
+
return n?.length === 1 && theirs[n[0]].path && agrees(i, n[0]) ? n[0] : rewritten(i);
|
|
345
|
+
};
|
|
346
|
+
const anchorOf = (j) => (still(j) ? placed(mineToBase[j]) : -1);
|
|
347
|
+
// the nearest paragraph of `mine` before and after j that passes `ok`, for every j at once (a walk from each paragraph was quadratic
|
|
348
|
+
// over a long stretch of new or moved paragraphs). -1 / mine.length: none.
|
|
349
|
+
const nearest = (ok) => {
|
|
350
|
+
const prev = new Int32Array(mine.length);
|
|
351
|
+
const next = new Int32Array(mine.length);
|
|
352
|
+
for (let j = 0, last = -1; j < mine.length; j++) (prev[j] = last), ok(j) && (last = j);
|
|
353
|
+
for (let j = mine.length - 1, last = mine.length; j >= 0; j--) (next[j] = last), ok(j) && (last = j);
|
|
354
|
+
return { prev, next };
|
|
355
|
+
};
|
|
356
|
+
let nearStill = null;
|
|
357
|
+
let nearAnchor = null;
|
|
235
358
|
const inserts = []; // {where: "after"|"before", to: index in theirs, m, move?: index in theirs of the paragraph to move}
|
|
359
|
+
let unanchored = false; // a paragraph of the user's could not be placed: it may be one of the "deleted" ones, rewritten past recognition
|
|
236
360
|
|
|
237
361
|
/**
|
|
238
362
|
* A paragraph the user moved: the agent's paragraph is moved as it is, placed by the paragraph right before it in the user's
|
|
@@ -242,20 +366,20 @@ export function planDocxMerge(base, mine, theirs) {
|
|
|
242
366
|
* report only (`lost`).
|
|
243
367
|
*/
|
|
244
368
|
const move = (m, j, i) => {
|
|
245
|
-
const n =
|
|
369
|
+
const n = inPlace(i);
|
|
246
370
|
const more = lost(base[i], m);
|
|
247
371
|
const t = theirs[n];
|
|
248
|
-
const say = (why) => void no(m, why, "");
|
|
372
|
+
const say = (why) => void no(m, why, "", [ref(m), ref(base[i])]);
|
|
249
373
|
if (!t) return say(agent.moved.has(agent.mapA[i]) ? "You moved this paragraph and so did the agent. It is where the agent put it." : "You moved this paragraph, but the agent changed or removed it, so the move was not carried over.");
|
|
374
|
+
if (agentMoved) return say("You moved this paragraph, but the agent moved paragraphs too, so the move was not carried over.");
|
|
250
375
|
if (t.path?.length !== 1 || m.path?.length !== 1) return say("You moved this paragraph in or out of a table. Moves that involve tables are not carried over.");
|
|
251
376
|
if (t.sect) return say("You moved this paragraph, but it ends a section of the document, so the move was not carried over.");
|
|
252
377
|
if (t.loose) return say("You moved this paragraph, but a comment or bookmark starts or ends inside it, so the move was not carried over.");
|
|
253
378
|
const body = (k) => (k >= 0 && k < mine.length && anchorOf(k) >= 0 && theirs[anchorOf(k)].path.length === 1 ? anchorOf(k) : -1);
|
|
254
|
-
|
|
255
|
-
|
|
379
|
+
nearStill ||= nearest(still);
|
|
380
|
+
let k = nearStill.prev[j];
|
|
256
381
|
if (body(k) >= 0) return void inserts.push({ where: "after", to: body(k), m, move: n, ...more });
|
|
257
|
-
k = j
|
|
258
|
-
while (k < mine.length && !still(k)) k++;
|
|
382
|
+
k = nearStill.next[j];
|
|
259
383
|
if (body(k) >= 0) return void inserts.push({ where: "before", to: body(k), m, move: n, ...more });
|
|
260
384
|
say("You moved this paragraph, but the agent changed the paragraphs next to its new place, so the move was not carried over.");
|
|
261
385
|
};
|
|
@@ -264,43 +388,73 @@ export function planDocxMerge(base, mine, theirs) {
|
|
|
264
388
|
const i = mineToBase[j];
|
|
265
389
|
if (i >= 0) {
|
|
266
390
|
const b = base[i];
|
|
267
|
-
|
|
268
|
-
|
|
391
|
+
// moved into another table or cell, its order among the paragraphs kept: the move is not carried (a move that changes the order
|
|
392
|
+
// says so in move()); its text and formatting are, where it stands in the agent's version
|
|
393
|
+
if (!sameBox(b, m) && !moved.has(j)) no(m, "You moved this paragraph into another table or table cell. Moves that involve tables are not carried over.", "", [ref(m), ref(b)]);
|
|
394
|
+
if (b.text === m.text) return moved.has(j) ? move(m, j, i) : format(b, m, i, false);
|
|
395
|
+
// The text changed. White space counts as written: a line break, a tab or a second space is an edit too, though the text hashes the
|
|
396
|
+
// same (an outside review: a line break typed into "A B" was dropped without a word). So does the agent's, on its side.
|
|
269
397
|
const t = at(i);
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
398
|
+
const now = t >= 0 ? theirs[t] : agent.mapA[i] >= 0 && agrees(i, agent.mapA[i]) ? theirs[agent.mapA[i]] : null;
|
|
399
|
+
// the agent's own copy of this paragraph already reads as the user's (an earlier pass carried it). Only that one paragraph counts:
|
|
400
|
+
// the same text anywhere else in the agent's version says nothing about this one (an outside review lost an edit that way)
|
|
401
|
+
if (now?.text === m.text) return moved.has(j) ? move(m, j, i) : undefined;
|
|
402
|
+
if (t < 0 || now.text !== b.text) return no(m, "The agent changed this paragraph too. Its version is in the document; this is yours.", m.text, [ref(m), ref(b)]);
|
|
403
|
+
const r = minimalReplace(b.text, m.text, work);
|
|
273
404
|
const op = { op: "replace_text", target: locator(t), expect: { text_hash: b.hash }, ...(r ? r : { text: m.text }) };
|
|
274
|
-
items.push({ kind: "changed", label: label(m), text: cut(m.text), hash: m.hash, t, op });
|
|
275
|
-
|
|
405
|
+
items.push({ kind: "changed", label: label(m), of: [ref(m), ref(b)], text: cut(m.text), hash: m.hash, raw: m.text, m, t, op });
|
|
406
|
+
format(b, m, i, true);
|
|
407
|
+
if (moved.has(j)) move(m, j, i); // moved as well: the agent's paragraph, edited first, is moved
|
|
408
|
+
return;
|
|
276
409
|
}
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
if (k >= 0) return inserts.push({ where: "after", to: anchorOf(k), m });
|
|
281
|
-
k = j
|
|
282
|
-
|
|
283
|
-
|
|
410
|
+
nearAnchor ||= nearest((x) => anchorOf(x) >= 0);
|
|
411
|
+
// placed by a paragraph in the same place: the body, or the same table cell (a new line typed in a cell was put in the body)
|
|
412
|
+
let k = nearAnchor.prev[j];
|
|
413
|
+
if (k >= 0 && sameBox(mine[k], m) && sameBox(theirs[anchorOf(k)], mine[k])) return inserts.push({ where: "after", to: anchorOf(k), m });
|
|
414
|
+
k = nearAnchor.next[j];
|
|
415
|
+
if (k < mine.length && sameBox(mine[k], m) && sameBox(theirs[anchorOf(k)], mine[k])) return inserts.push({ where: "before", to: anchorOf(k), m });
|
|
416
|
+
unanchored = true;
|
|
284
417
|
no(m, "There was no unchanged paragraph near this new one to place it by.");
|
|
285
418
|
});
|
|
286
419
|
// several new or moved paragraphs after one anchor: each lands right after the anchor, so the last goes first
|
|
287
420
|
for (const x of inserts.filter((x) => x.where === "after").reverse().concat(inserts.filter((x) => x.where === "before"))) {
|
|
288
421
|
const to = locator(x.to);
|
|
289
|
-
if (x.move === undefined) items.push({ kind: "added", label: label(x.m), text: cut(x.m.text), hash: x.m.hash, t: x.to, where: x.where, op: { op: x.where === "after" ? "insert_paragraph_after" : "insert_paragraph_before", target: to, expect: { text_hash: to.text_hash }, text: x.m.text } });
|
|
290
|
-
else items.push({ kind: "moved", label: label(x.m), text: "", hash: x.m.hash, t: x.move, to: x.to, where: x.where, ...(x.note ? { note: x.note, check: x.check } : {}), op: { op: "move_paragraph", target: locator(x.move), expect: { text_hash: x.
|
|
422
|
+
if (x.move === undefined) items.push({ kind: "added", label: label(x.m), of: [ref(x.m)], text: cut(x.m.text), hash: x.m.hash, m: x.m, t: x.to, where: x.where, op: { op: x.where === "after" ? "insert_paragraph_after" : "insert_paragraph_before", target: to, expect: { text_hash: to.text_hash }, text: x.m.text } });
|
|
423
|
+
else items.push({ kind: "moved", label: label(x.m), of: [ref(x.m), ref(base[mineToBase[x.m.ordinal]])], text: "", hash: x.m.hash, t: x.move, to: x.to, where: x.where, ...(x.note ? { note: x.note, check: x.check } : {}), op: { op: "move_paragraph", target: locator(x.move), expect: { text_hash: theirs[x.move].hash }, to, where: x.where } });
|
|
291
424
|
}
|
|
425
|
+
// A base paragraph with no match in the user's version is deleted only when every paragraph of the user's could be placed. When one could
|
|
426
|
+
// not (the matcher's similarity budget ran out on a long run of edited paragraphs, so none of them matched), the unmatched base paragraphs
|
|
427
|
+
// may be those very paragraphs, rewritten: deleting them removed the whole document (a merge of 49,999 edited paragraphs left one).
|
|
428
|
+
const kept = [];
|
|
292
429
|
base.forEach((b, i) => {
|
|
293
|
-
if (baseToMine[i] >= 0
|
|
294
|
-
const t = at(i);
|
|
295
|
-
if (t >= 0)
|
|
430
|
+
if (baseToMine[i] >= 0) return;
|
|
431
|
+
const t = at(i) >= 0 && theirs[at(i)].text === b.text ? at(i) : -1; // the agent's white space counts too
|
|
432
|
+
if (t >= 0 && unanchored) kept.push(ref(b));
|
|
433
|
+
else if (t >= 0) items.push({ kind: "removed", label: label(b), of: [ref(b)], text: "", hash: null, t, op: { op: "delete_paragraph", target: locator(t), expect: { text_hash: b.hash } } });
|
|
296
434
|
else no(b, "You deleted this paragraph, but the agent changed it, so it was kept.", "");
|
|
297
435
|
});
|
|
436
|
+
if (kept.length) unplaced.push({ label: "Deleted paragraphs", text: "", why: `${kept.length === 1 ? "A paragraph you deleted was" : `${kept.length} paragraphs you deleted or rewrote were`} kept, because Onionskin could not place your other edits next to ${kept.length === 1 ? "it" : "them"}.`, of: kept });
|
|
437
|
+
// A table cell holds at least one paragraph: deleting all of a cell's paragraphs leaves a cell Word reports as damaged (found by the
|
|
438
|
+
// proof of the merge: a table row the person deleted came back as a row of empty cells). Those deletions are a row or a cell the
|
|
439
|
+
// person removed, which is not carried: they are handed back and the paragraphs stay.
|
|
440
|
+
const cellOf = (p) => (p.path?.length === 4 ? `${p.tableNo} ${p.path[1]} ${p.path[2]}` : null);
|
|
441
|
+
const left = new Map(); // cell of the agent's version -> paragraphs it keeps after the plan
|
|
442
|
+
for (const p of theirs) if (cellOf(p)) left.set(cellOf(p), (left.get(cellOf(p)) || 0) + 1);
|
|
443
|
+
for (const it of items) if (it.kind === "removed" && cellOf(theirs[it.t])) left.set(cellOf(theirs[it.t]), left.get(cellOf(theirs[it.t])) - 1);
|
|
444
|
+
for (const it of items) if (it.kind === "added" && cellOf(theirs[it.t])) left.set(cellOf(theirs[it.t]), left.get(cellOf(theirs[it.t])) + 1);
|
|
445
|
+
for (let k = items.length - 1; k >= 0; k--) {
|
|
446
|
+
const it = items[k];
|
|
447
|
+
if (it.kind !== "removed" || !cellOf(theirs[it.t]) || left.get(cellOf(theirs[it.t])) > 0) continue;
|
|
448
|
+
items.splice(k, 1);
|
|
449
|
+
unplaced.push({ label: it.label, text: "", why: "You deleted this paragraph with the table row or cell it was in. Rows and cells are not carried over, so it was kept.", of: it.of });
|
|
450
|
+
}
|
|
298
451
|
// replacements first, then formatting, then new and moved paragraphs, then deletions (an anchor is never a paragraph this plan deletes)
|
|
299
452
|
const order = { changed: 0, format: 1, added: 2, moved: 2, removed: 3 };
|
|
300
453
|
items.sort((x, y) => order[x.kind] - order[y.kind]);
|
|
301
454
|
return { items, unplaced };
|
|
302
455
|
}
|
|
303
456
|
|
|
457
|
+
const turn = () => new Promise((resolve) => setImmediate(resolve));
|
|
304
458
|
const SOFT = new Set(["format", "moved"]); // carried only when the result proves them; the merge can always do without
|
|
305
459
|
const PROTECTED = "a protected paragraph (it holds a field, a content control or something else that cannot be rewritten safely)";
|
|
306
460
|
/** Why one item is not in the document, in words for the user. `r` is the op's result when it ran. */
|
|
@@ -313,24 +467,42 @@ function reason(it, r) {
|
|
|
313
467
|
|
|
314
468
|
/** The order of paragraphs (as text hashes, with the index in `theirs` each came from) the applied ops must give. */
|
|
315
469
|
function expected(theirs, items, results) {
|
|
316
|
-
|
|
317
|
-
const
|
|
470
|
+
// a linked list of the agent's paragraphs, each found by its index in `theirs` (a search and a splice per item was quadratic)
|
|
471
|
+
const head = { next: null };
|
|
472
|
+
const tail = { prev: head };
|
|
473
|
+
head.next = tail;
|
|
474
|
+
const put = (a, x) => ((x.prev = a), (x.next = a.next), (a.next.prev = x), (a.next = x), (x.live = true)); // x right after a
|
|
475
|
+
const take = (x) => ((x.prev.next = x.next), (x.next.prev = x.prev), (x.live = false));
|
|
476
|
+
const nodes = theirs.map((p, n) => {
|
|
477
|
+
const x = { n, hash: p.hash };
|
|
478
|
+
put(tail.prev, x);
|
|
479
|
+
return x;
|
|
480
|
+
});
|
|
481
|
+
const live = (n) => (nodes[n]?.live ? nodes[n] : null);
|
|
318
482
|
items.forEach((it, k) => {
|
|
319
483
|
if (results[k]?.status !== "applied") return;
|
|
320
|
-
const
|
|
321
|
-
if (
|
|
322
|
-
if (it.kind === "changed")
|
|
323
|
-
else if (it.kind === "removed")
|
|
324
|
-
else if (it.kind === "added")
|
|
484
|
+
const x = live(it.t);
|
|
485
|
+
if (!x) return;
|
|
486
|
+
if (it.kind === "changed") x.hash = it.hash;
|
|
487
|
+
else if (it.kind === "removed") take(x);
|
|
488
|
+
else if (it.kind === "added") put(it.where === "after" ? x : x.prev, { n: -1, hash: it.hash, it });
|
|
325
489
|
else if (it.kind === "moved") {
|
|
326
|
-
|
|
327
|
-
const to =
|
|
328
|
-
|
|
490
|
+
take(x);
|
|
491
|
+
const to = live(it.to); // always there (an anchor is never a paragraph the plan removes); if not, where a splice at -1 put it
|
|
492
|
+
put(to ? (it.where === "after" ? to : to.prev) : it.where === "after" || tail.prev === head ? head : tail.prev.prev, x);
|
|
329
493
|
}
|
|
330
494
|
});
|
|
495
|
+
const seq = [];
|
|
496
|
+
for (let x = head.next; x !== tail; x = x.next) seq.push({ n: x.n, hash: x.hash, it: x.it });
|
|
331
497
|
return seq;
|
|
332
498
|
}
|
|
333
499
|
|
|
500
|
+
/** Do two readings of the same text look the same: paragraph style, alignment, list, other paragraph settings and runs by character? */
|
|
501
|
+
function sameLook(a, b) {
|
|
502
|
+
const runs = runPatches(a.spans, b.spans);
|
|
503
|
+
return styleKey(a.styleName || a.style) === styleKey(b.styleName || b.style) && a.align === b.align && listSig(a) === listSig(b) && a.ppr === b.ppr && !!runs && !runs.other && !runs.ranges.length;
|
|
504
|
+
}
|
|
505
|
+
|
|
334
506
|
/** Is the paragraph `o` of the result the agent's paragraph `t` with exactly the formatting this item set, and nothing else different? */
|
|
335
507
|
function formatted(it, t, o) {
|
|
336
508
|
const para = it.op.para || {};
|
|
@@ -351,18 +523,26 @@ function formatted(it, t, o) {
|
|
|
351
523
|
*/
|
|
352
524
|
export async function mergeDocx({ base, mine, theirs, name, runOps }) {
|
|
353
525
|
const read = (bytes) => readDocxParagraphs(bytes, { detail: true });
|
|
526
|
+
// a turn of the event loop between the readings and the plan: at 50,000 paragraphs each takes 0.2 to 0.3 s, all four in a row 0.9 s
|
|
354
527
|
const agents = read(theirs);
|
|
355
|
-
|
|
528
|
+
await turn();
|
|
529
|
+
const before = read(base);
|
|
530
|
+
await turn();
|
|
531
|
+
const after = read(mine);
|
|
532
|
+
await turn();
|
|
533
|
+
const plan = planDocxMerge(before, after, agents);
|
|
534
|
+
await turn();
|
|
356
535
|
const unplaced = [...plan.unplaced];
|
|
357
|
-
|
|
536
|
+
const readings = { base: before, mine: after, theirs: agents }; // for the proof of the merge (src/merge-worker.js), read once
|
|
537
|
+
if (!plan.items.length) return { bytes: null, merged: 0, unplaced, read: readings };
|
|
358
538
|
let r = await attempt(plan.items, { theirs, agents, name, runOps, read });
|
|
359
539
|
if (r.redo) {
|
|
360
540
|
// a move or a formatting change did not come out as planned: nothing unproven is written, the rest is carried without them
|
|
361
|
-
for (const it of plan.items.filter((x) => SOFT.has(x.kind))) unplaced.push({ label: it.label, text: "", why: reason(it) });
|
|
541
|
+
for (const it of plan.items.filter((x) => SOFT.has(x.kind))) unplaced.push({ label: it.label, text: "", why: reason(it), of: it.of });
|
|
362
542
|
const rest = plan.items.filter((x) => !SOFT.has(x.kind));
|
|
363
543
|
r = rest.length ? await attempt(rest, { theirs, agents, name, runOps, read }) : { bytes: null, merged: 0, unplaced: [] };
|
|
364
544
|
}
|
|
365
|
-
return { bytes: r.bytes, merged: r.merged, unplaced: [...unplaced, ...r.unplaced] };
|
|
545
|
+
return { bytes: r.bytes, merged: r.merged, unplaced: [...unplaced, ...r.unplaced], read: readings };
|
|
366
546
|
}
|
|
367
547
|
|
|
368
548
|
/** One run of `items` on the agent's bytes, checked on the result. {redo: true} asks for a run without moves and formatting. */
|
|
@@ -370,13 +550,14 @@ async function attempt(items, { theirs, agents, name, runOps, read }) {
|
|
|
370
550
|
const unplaced = [];
|
|
371
551
|
const soft = items.some((x) => SOFT.has(x.kind));
|
|
372
552
|
const { results, bytes, blocked } = await runOps({ bytes: theirs, format: "docx", name, ops: items.map((x) => x.op), partial: true });
|
|
373
|
-
const fail = (it, why) => unplaced.push({ label: it.label, text: it.text, why });
|
|
553
|
+
const fail = (it, why) => unplaced.push({ label: it.label, text: it.text, why, of: it.of });
|
|
374
554
|
if (!bytes) {
|
|
375
555
|
if (blocked && soft) return { redo: true };
|
|
376
556
|
for (const [n, it] of items.entries()) fail(it, blocked ? "Carrying your edits over would have damaged the file, so none were applied." : reason(it, results[n]));
|
|
377
557
|
return { bytes: null, merged: 0, unplaced };
|
|
378
558
|
}
|
|
379
559
|
// the proof is the result, not the report. First: nothing of the agent's version is gone except what an op named.
|
|
560
|
+
await turn();
|
|
380
561
|
const out = read(bytes);
|
|
381
562
|
const count = (list) => list.reduce((m, h) => m.set(h, (m.get(h) || 0) + 1), new Map());
|
|
382
563
|
const named = count(items.filter((x, n) => (x.kind === "changed" || x.kind === "removed") && results[n]?.status === "applied").map((x) => x.op.expect.text_hash));
|
|
@@ -396,14 +577,36 @@ async function attempt(items, { theirs, agents, name, runOps, read }) {
|
|
|
396
577
|
const where = new Map(seq.map((x, k) => [x.n, k]));
|
|
397
578
|
if (items.some((it, n) => it.kind === "format" && results[n]?.status === "applied" && !formatted(it, agents[it.t], out[where.get(it.t)]))) return { redo: true };
|
|
398
579
|
}
|
|
399
|
-
// then: every carried text is in the merged file
|
|
580
|
+
// then: every carried text is in the merged file, and a changed paragraph reads as typed, white space as written, in its own place (the
|
|
581
|
+
// same text elsewhere proves nothing about it) whenever the result has the paragraphs the plan expects
|
|
400
582
|
const have = new Set(out.map((p) => p.hash));
|
|
583
|
+
const seq = expected(agents, items, results);
|
|
584
|
+
const place = seq.length === out.length ? new Map(seq.map((x, k) => [x.n, k])) : null;
|
|
585
|
+
if (place && items.some((it, n) => it.kind === "changed" && results[n]?.status === "applied" && out[place.get(it.t)]?.text !== it.raw)) {
|
|
586
|
+
// a paragraph of the agent's now reads as neither version: nothing is written
|
|
587
|
+
if (soft) return { redo: true };
|
|
588
|
+
for (const it of items) fail(it, "Carrying your edits over would have changed the agent's text, so none were applied.");
|
|
589
|
+
return { bytes: null, merged: 0, unplaced };
|
|
590
|
+
}
|
|
591
|
+
// The look the person gave a paragraph they added, or the inside of one they typed in, that the result does not have is said (a check
|
|
592
|
+
// line), never dropped without a word: the engine gives a new paragraph the paragraph formatting of the one it is placed by, and typed
|
|
593
|
+
// text the look of the run it lands in. The proof of the merge (src/merge-proof.js) refuses a result that differs without such a line.
|
|
594
|
+
if (place) {
|
|
595
|
+
const added = new Map();
|
|
596
|
+
seq.forEach((x, k) => x.it && added.set(x.it, k));
|
|
597
|
+
items.forEach((it, n) => {
|
|
598
|
+
if (results[n]?.status !== "applied" || !it.m || (it.kind !== "added" && it.kind !== "changed")) return;
|
|
599
|
+
const o = out[it.kind === "added" ? added.get(it) : place.get(it.t)];
|
|
600
|
+
if (!o || o.text !== it.m.text || sameLook(it.m, o)) return;
|
|
601
|
+
unplaced.push({ label: it.label, text: "", why: it.kind === "added" ? "Your new paragraph was carried over with the formatting of the paragraph next to it: if you formatted it, set it again." : "Your text was carried over, but the formatting inside this paragraph is not as you had it: check it.", of: it.of, check: true });
|
|
602
|
+
});
|
|
603
|
+
}
|
|
401
604
|
let merged = 0;
|
|
402
605
|
items.forEach((it, n) => {
|
|
403
606
|
if (results[n]?.status !== "applied") return fail(it, reason(it, results[n]));
|
|
404
607
|
if (it.hash && !have.has(it.hash)) return fail(it, "This edit did not land as typed.");
|
|
405
608
|
merged++;
|
|
406
|
-
if (it.note && it.kind === "moved") unplaced.push({ label: it.label, text: "", why: `Your move was carried over. ${it.note}`, ...(it.check ? { check: true } : {}) });
|
|
609
|
+
if (it.note && it.kind === "moved") unplaced.push({ label: it.label, text: "", why: `Your move was carried over. ${it.note}`, of: it.of, ...(it.check ? { check: true } : {}) });
|
|
407
610
|
else if (it.note) fail(it, `Your ${list(it.words)} change was carried over. ${it.note}`);
|
|
408
611
|
});
|
|
409
612
|
return { bytes, merged, unplaced };
|
package/src/nfc.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// Unicode normalization in time linear in the text. Pure and without imports: the server, the CLI and the page (web/src) normalize
|
|
2
|
+
// document text with this one function, so the same text hashes the same everywhere.
|
|
3
|
+
//
|
|
4
|
+
// String.prototype.normalize sorts each run of combining marks by combining class, in time quadratic in the run's length: one Word
|
|
5
|
+
// paragraph of "a" followed by 120,000 marks held the server's thread for 4 to 6 seconds (an outside review). Unicode's Stream-Safe
|
|
6
|
+
// Text Format (UAX #15) bounds a run at 30 non-starters, which no real text needs more of. A run of more than MAX_MARKS characters of
|
|
7
|
+
// general category M (every non-starter is one) is left as it is, and the text between such runs is normalized: one linear scan, no
|
|
8
|
+
// sort longer than MAX_MARKS. Text without such a run (every real script: Vietnamese, Arabic with harakat, Hebrew with niqqud,
|
|
9
|
+
// Devanagari, emoji sequences, Zalgo text within the limit) is normalized whole, exactly as before, so hashes stored by earlier
|
|
10
|
+
// versions of ordinary text do not change. The result depends on the text alone, and normalizing it again gives it back.
|
|
11
|
+
// Ceiling: an over-long run is not reordered or composed (its base letter stays apart from its first mark).
|
|
12
|
+
// Compatibility forms (NFKC, NFKD) too: U+FF9E and U+FF9F (halfwidth katakana voiced and semi-voiced sound marks) are not of category M
|
|
13
|
+
// but decompose to the combining marks U+3099 and U+309A, so "゙̕" repeated built the same unbounded combining sequence out of what
|
|
14
|
+
// the run pattern did not see (2.4 s at 40,000 pairs, four times the 20,000's). They count as marks in a run. test/nfc.test.js scans
|
|
15
|
+
// every code point for another character whose compatibility decomposition starts with a mark: these two (and the Thai and Lao
|
|
16
|
+
// sara am, whose first character has combining class 0, so nothing to sort) are all there are.
|
|
17
|
+
|
|
18
|
+
export const MAX_MARKS = 30;
|
|
19
|
+
// at most CHUNK marks a match, joined into runs by the loop below: an unbounded match of 7.9 million marks overflowed the regex stack
|
|
20
|
+
// ("a" + "\u0315\u0300" x 3,949,999: Send answered 422 "Maximum call stack size exceeded")
|
|
21
|
+
const CHUNK = 4096;
|
|
22
|
+
const RUNS = new RegExp(`[\\p{M}\\uFF9E\\uFF9F]{1,${CHUNK}}`, "gu");
|
|
23
|
+
|
|
24
|
+
/** s.normalize(form), linear in the length of s (see above). Null and undefined read as "". */
|
|
25
|
+
export function nfc(s, form = "NFC") {
|
|
26
|
+
s = String(s ?? "");
|
|
27
|
+
let out = null;
|
|
28
|
+
let from = 0;
|
|
29
|
+
let start = -1; // the run being read: [start, end)
|
|
30
|
+
let end = -1;
|
|
31
|
+
const close = () => {
|
|
32
|
+
// UTF-16 units first (never fewer than the marks), then the marks themselves (a mark outside the BMP is two units)
|
|
33
|
+
if (start < 0 || end - start <= MAX_MARKS || (end - start <= 2 * MAX_MARKS && [...s.slice(start, end)].length <= MAX_MARKS)) return;
|
|
34
|
+
(out ||= []).push(s.slice(from, start).normalize(form), s.slice(start, end));
|
|
35
|
+
from = end;
|
|
36
|
+
};
|
|
37
|
+
RUNS.lastIndex = 0;
|
|
38
|
+
for (let m; (m = RUNS.exec(s)); ) {
|
|
39
|
+
if (m.index !== end) (close(), (start = m.index));
|
|
40
|
+
end = m.index + m[0].length;
|
|
41
|
+
}
|
|
42
|
+
close();
|
|
43
|
+
if (!out) return s.normalize(form);
|
|
44
|
+
out.push(s.slice(from).normalize(form));
|
|
45
|
+
return out.join("");
|
|
46
|
+
}
|