onionskin-cli 0.0.3 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CHANGELOG.md +198 -0
  2. package/INSTALL.md +5 -1
  3. package/NOTICE +34 -0
  4. package/README.md +102 -34
  5. package/docs/contracts/batch.schema.json +27 -4
  6. package/docs/contracts/ops.schema.json +117 -5
  7. package/package.json +4 -2
  8. package/scripts/checker/check.py +24 -5
  9. package/scripts/checker/requirements.txt +482 -5
  10. package/scripts/requirements.txt +482 -5
  11. package/scripts/rewriter/apple/main.swift +1 -1
  12. package/scripts/rewriter/apple/onionskin-fm +0 -0
  13. package/scripts/rewriter/apple/onionskin-fm.json +1 -1
  14. package/skills/onionskin/SKILL.md +12 -9
  15. package/skills/onionskin/reference/batch.md +5 -3
  16. package/skills/onionskin/reference/formats.md +5 -1
  17. package/skills/onionskin/reference/ops.md +48 -4
  18. package/skills/onionskin/reference/troubleshooting.md +31 -0
  19. package/src/agent-detect.js +28 -2
  20. package/src/agent-prompt.js +12 -1
  21. package/src/agent-runner.js +110 -20
  22. package/src/anchor-work.js +154 -0
  23. package/src/anchor-worker.js +38 -0
  24. package/src/anchors.js +61 -11
  25. package/src/apply.js +119 -47
  26. package/src/batch.js +32 -8
  27. package/src/check-worker.js +8 -5
  28. package/src/checker.js +238 -33
  29. package/src/cli.js +248 -42
  30. package/src/constants.js +34 -1
  31. package/src/diff.js +3 -1
  32. package/src/doctor.js +12 -3
  33. package/src/docx.js +464 -87
  34. package/src/edit-hash.js +19 -0
  35. package/src/fsutil.js +14 -6
  36. package/src/history-worker.js +8 -8
  37. package/src/history.js +144 -31
  38. package/src/html-active.js +384 -0
  39. package/src/html-envelope.js +51 -0
  40. package/src/html-parser.js +8 -0
  41. package/src/html-render-worker.js +25 -0
  42. package/src/html-render.js +939 -0
  43. package/src/html-utf8.js +65 -0
  44. package/src/html-write-worker.js +14 -0
  45. package/src/html-write.js +380 -0
  46. package/src/html.js +1259 -0
  47. package/src/local.js +68 -4
  48. package/src/merge-package.js +1166 -0
  49. package/src/merge-pptx.js +98 -38
  50. package/src/merge-proof.js +1019 -0
  51. package/src/merge-worker.js +60 -0
  52. package/src/merge-xlsx.js +84 -29
  53. package/src/merge.js +286 -83
  54. package/src/nfc.js +46 -0
  55. package/src/notes.js +122 -0
  56. package/src/offload.js +419 -1
  57. package/src/open-worker.js +27 -0
  58. package/src/ops/common.js +2 -1
  59. package/src/ops/docx-table.js +5 -3
  60. package/src/ops/docx.js +58 -29
  61. package/src/ops/html.js +394 -0
  62. package/src/ops/pptx.js +82 -23
  63. package/src/ops/run.js +23 -3
  64. package/src/ops/runs.js +215 -13
  65. package/src/ops/schema.js +8 -1
  66. package/src/ops/xlsx.js +121 -10
  67. package/src/ops-worker.js +17 -0
  68. package/src/package.js +43 -4
  69. package/src/person.js +2 -1
  70. package/src/pptx.js +323 -23
  71. package/src/question.js +10 -3
  72. package/src/quick-plan.js +13 -6
  73. package/src/quick.js +1 -1
  74. package/src/redline-check.js +12 -2
  75. package/src/redline-worker.js +10 -14
  76. package/src/reply.js +46 -19
  77. package/src/retain.js +223 -0
  78. package/src/review.js +32 -18
  79. package/src/rewrite-check.js +2 -2
  80. package/src/rewriter.js +38 -8
  81. package/src/sanitize.js +16 -3
  82. package/src/server-entry.js +4 -3
  83. package/src/server.js +175 -62
  84. package/src/session.js +1151 -210
  85. package/src/setup.js +20 -2
  86. package/src/stop-gate.js +34 -0
  87. package/src/suggest-reply.js +42 -7
  88. package/src/suggest.js +14 -29
  89. package/src/text.js +161 -11
  90. package/src/update.js +517 -0
  91. package/src/voice-guide.js +15 -9
  92. package/src/voice-learn.js +15 -4
  93. package/src/voice-sample-worker.js +7 -6
  94. package/src/voice-store.js +11 -4
  95. package/src/voice-worker.js +4 -6
  96. package/src/voice.js +2 -1
  97. package/src/worker-host.js +23 -0
  98. package/src/xlsx.js +378 -97
  99. package/src/xml-tree.js +6 -3
  100. package/src/xml.js +82 -34
  101. package/src/zip-write.js +10 -5
  102. package/src/zip.js +73 -18
  103. package/vendor/parse5/LICENSE.txt +32 -0
  104. package/vendor/parse5/parse5.mjs +7822 -0
  105. package/vendor/parse5.lock.json +23 -0
  106. package/web/dist/assets/{index-DS-WST1E.js → index-CjsTf5gj.js} +58 -52
  107. package/web/dist/assets/index-DnbLtHhG.css +1 -0
  108. package/web/dist/index.html +2 -2
  109. package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/merge.js CHANGED
@@ -1,26 +1,28 @@
1
1
  import { fmtNote } from "./fmt.js";
2
- import { docxFmt, lcsPairs, matchParagraphs } from "./history.js";
2
+ import { docxFmt, lcsPairs, matchParagraphs, paraIdentity } from "./history.js";
3
3
  import { readDocxParagraphs } from "./docx.js";
4
4
  import { normText } from "./text.js";
5
5
 
6
6
  // What the user did while the agent worked, carried onto the agent's version. Three versions of one DOCX: `base` (what the page
7
7
  // loaded: the file as sent), `mine` (base plus the user's edits, saved by the page) and `theirs` (the file on disk after the agent).
8
8
  // A paragraph only the user changed becomes an op on `theirs`, run through the same engine, expect check and package check as an
9
- // agent's `apply`. A paragraph both sides changed is never guessed at: the agent's text stays and the user's text is handed back
10
- // (`unplaced`), so nothing typed is lost without a word.
9
+ // agent's `apply`. Text is compared as written: a line break, a tab or a second space is an edit. A paragraph both sides changed is
10
+ // never guessed at: the agent's text stays and the user's text is handed back (`unplaced`), so nothing typed is lost without a word.
11
11
  // Formatting the user changed without changing the text (run properties by character, and the paragraph's style, alignment and
12
12
  // list) is set on the agent's paragraph when the agent left that formatting alone; on a paragraph the agent rewrote only the
13
- // paragraph's own style, alignment and list are set. A body paragraph the user moved is moved in the agent's version when the
14
- // agent left it alone and the paragraph next to its new place is still there. Each of these is proven on the result before it is
15
- // kept: if one did not come out as planned, the merge runs again without any of them and says so.
13
+ // paragraph's own style, alignment and list are set. A body paragraph the user moved (edited or not) is moved in the agent's version
14
+ // when the agent left it alone, moved no paragraph itself, and the paragraph next to its new place is still there. Each of these is
15
+ // proven on the result before it is kept: if one did not come out as planned, the merge runs again without any of them and says so.
16
+ // Then the proof of the merge (src/merge-proof.js, in src/merge-worker.js) checks the result on its own: every entry of `unplaced` names
17
+ // what it stands for (`of`), and a result that does not account for every edit is not returned.
16
18
  // Ceilings: pictures and anything outside paragraphs are not carried. Not carried and reported: formatting of a kind the engine
17
19
  // cannot set (character style, theme color, shading, spacing, indent), moves inside tables, a moved paragraph that ends a section or
18
- // holds one end of a comment or bookmark, and formatting changed on a paragraph the user also moved (the agent's paragraph is moved
19
- // as it is; the report names what the user changed inside it when that is certain, and asks to check the formatting whenever the user's copy
20
- // reads differently, which every cut and paste in the editor does). Not carried and not reported: run formatting changed in a
21
- // paragraph whose text the user also changed, the formatting of a paragraph the user added (it takes its neighbor's), and a move
22
- // of a paragraph the user also edited (its text is carried in the old place). A paragraph the user split keeps its run formatting
23
- // only in the part before the first changed character.
20
+ // holds one end of a comment or bookmark, a move while the agent moved paragraphs too, and formatting changed on a paragraph the user
21
+ // also moved (the agent's paragraph is moved as it is; the report names what the user changed inside it when that is certain, and asks
22
+ // to check the formatting whenever the user's copy reads differently, which every cut and paste in the editor does). Carried, with a
23
+ // line asking to check: the text of a paragraph the user added whose formatting the result does not have (the engine gives it the
24
+ // paragraph formatting of its neighbor), and text typed into a paragraph whose formatting inside then differs from the user's. A
25
+ // paragraph the user split keeps its run formatting only in the part before the first changed character.
24
26
 
25
27
  const CAP = 2000;
26
28
  const cut = (t) => String(t ?? "").slice(0, CAP);
@@ -33,7 +35,7 @@ const label = (p) => {
33
35
  * The smallest `find` / `replace` that turns `before` into `after` and occurs exactly once in `before` (so run formatting around it
34
36
  * is kept), or null when there is none within the op's limits (then the whole paragraph text is set).
35
37
  */
36
- export function minimalReplace(before, after) {
38
+ export function minimalReplace(before, after, shared = null) {
37
39
  let p = 0;
38
40
  const max = Math.min(before.length, after.length);
39
41
  while (p < max && before[p] === after[p]) p++;
@@ -41,16 +43,27 @@ export function minimalReplace(before, after) {
41
43
  while (s < max - p && before[before.length - 1 - s] === after[after.length - 1 - s]) s++;
42
44
  let a = p;
43
45
  let b = before.length - s; // [a, b) of `before` is replaced
44
- const count = (needle) => {
45
- let n = 0;
46
- for (let i = before.indexOf(needle); i >= 0; i = before.indexOf(needle, i + 1)) n++;
47
- return n;
46
+ // only uniqueness matters: stop at the second match (counting every overlapping match of each widening was quadratic: a run of
47
+ // 20,000 "a" with one letter typed in the middle held the server's thread for more than 5 seconds in an outside review). Each
48
+ // search still spans up to the whole text when the second match is far away, so the text the searches span is budgeted: past
49
+ // 16 times the paragraph (plus a floor), null, and the whole paragraph text is set instead (its run formatting is not kept).
50
+ // `shared` ({left}) is the budget of a whole merge: the floor is per paragraph, so 3,900 paragraphs of 2,000 "a" with one letter typed
51
+ // in each spent 5 s of the merge's time in an outside review. Past it, every later paragraph is set whole as well.
52
+ let budget = 2_000_000 + 16 * before.length;
53
+ const once = (needle) => {
54
+ const i = before.indexOf(needle);
55
+ if (i < 0) return false;
56
+ const j = before.indexOf(needle, i + 1);
57
+ const span = j < 0 ? before.length : j + needle.length; // what the two searches spanned
58
+ budget -= span;
59
+ if (shared) shared.left -= span;
60
+ return j < 0;
48
61
  };
49
62
  // widen (left first, then right) until the find text is not empty and occurs once
50
- while ((b === a || count(before.slice(a, b)) !== 1) && (a > 0 || b < before.length)) {
63
+ while ((b === a || !once(before.slice(a, b))) && (a > 0 || b < before.length)) {
51
64
  if (a > 0) a--;
52
65
  else b++;
53
- if (b - a > CAP) return null;
66
+ if (b - a > CAP || budget < 0 || shared?.left < 0) return null;
54
67
  }
55
68
  if (b === a) return null; // an empty paragraph has nothing to find
56
69
  const find = before.slice(a, b);
@@ -136,6 +149,56 @@ function paraPatch(from, to) {
136
149
  return { set, no };
137
150
  }
138
151
 
152
+ /**
153
+ * Are two paragraphs in the same place: both in the body, or in the same cell (row and column) of the same table? A table is its order
154
+ * among the body's tables (`tableNo`; its body index shifts with every paragraph added above it). Two cells 0 0 of two tables are two
155
+ * places (an outside review: a paragraph moved from the first table to the second was edited in the first and nothing was said).
156
+ */
157
+ const sameBox = (a, b) => (a.path?.length ?? 0) === (b.path?.length ?? 0) && (a.path?.length !== 4 || (a.tableNo === b.tableNo && a.path[1] === b.path[1] && a.path[2] === b.path[2]));
158
+
159
+ /**
160
+ * Pairs, in place, a base paragraph `mapA` leaves unpaired alone between two paired neighbors with the first unpaired paragraph of B
161
+ * between their partners, when the two are in the same place (sameBox) and the ids allow it (`may`). Linear.
162
+ */
163
+ function pairSlots(A, B, mapA, mapB, may) {
164
+ let i0 = -1;
165
+ for (let i = 0; i <= A.length; i++) {
166
+ if (i < A.length && mapA[i] < 0) continue;
167
+ const j0 = i0 < 0 ? -1 : mapA[i0];
168
+ const j1 = i < A.length ? mapA[i] : B.length;
169
+ // one paragraph rewritten (a long run of rewritten paragraphs stays with the matcher: see "every one of 20,000 paragraphs rewritten")
170
+ const b = j0 + 1;
171
+ if (i - i0 === 2 && b < j1 && mapB[b] < 0 && sameBox(A[i0 + 1], B[b]) && may(i0 + 1, b)) (mapA[i0 + 1] = b), (mapB[b] = i0 + 1);
172
+ i0 = i;
173
+ }
174
+ }
175
+
176
+ /**
177
+ * The indexes j of `map` (index -> paired index, -1: none) that are out of order: those outside one longest run whose paired indexes
178
+ * increase (a longest increasing subsequence, n log n). The proof of the merge (src/merge-proof.js) finds moves on its own.
179
+ */
180
+ function outOfOrder(map) {
181
+ const js = [];
182
+ for (let j = 0; j < map.length; j++) if (map[j] >= 0) js.push(j);
183
+ const tails = []; // tails[len - 1]: the index in js of the smallest last value of a run of length len
184
+ const back = new Int32Array(js.length).fill(-1);
185
+ for (let k = 0; k < js.length; k++) {
186
+ const v = map[js[k]];
187
+ let lo = 0;
188
+ let hi = tails.length;
189
+ while (lo < hi) {
190
+ const mid = (lo + hi) >> 1;
191
+ if (map[js[tails[mid]]] < v) lo = mid + 1;
192
+ else hi = mid;
193
+ }
194
+ tails[lo] = k;
195
+ back[k] = lo > 0 ? tails[lo - 1] : -1;
196
+ }
197
+ const out = new Set(js);
198
+ for (let k = tails.length ? tails[tails.length - 1] : -1; k >= 0; k = back[k]) out.delete(js[k]);
199
+ return out;
200
+ }
201
+
139
202
  /**
140
203
  * Plans the merge. Arguments are paragraph lists (readDocxParagraphs with detail). Returns
141
204
  * {items: [{kind, label, text, t, op, ...}], unplaced: [{label, text, why}]}: every item is tried on `theirs`. `t` is the index in
@@ -143,29 +206,70 @@ function paraPatch(from, to) {
143
206
  * a `note` (with `check`, see `lost`) when its formatting in `mine` is not the original's.
144
207
  */
145
208
  export function planDocxMerge(base, mine, theirs) {
146
- const { mapA: baseToMine, mapB: mineToBase, moved } = matchParagraphs(base, mine);
147
- // a base paragraph the agent left alone, by position: the same text in the same order
148
- const { mapA: baseToTheirs } = lcsPairs(base.map((p) => p.hash), theirs.map((p) => p.hash));
149
- const agent = matchParagraphs(base, theirs); // where the agent's other paragraphs came from: an edited one, one it moved
150
- const at = (i) => (theirs[baseToTheirs[i]]?.path ? baseToTheirs[i] : -1);
209
+ // Which paragraph of the person's version, and of the agent's, is base paragraph i: one rule for both sides and for History
210
+ // (paraIdentity in src/history.js). A paragraph id unique in its version is the paragraph: a match by text and position never pairs
211
+ // it with a paragraph that carries another id (an outside review: the person retyped paragraph 1 to read as paragraph 2 and edited
212
+ // paragraph 2; text matching took the new paragraph 1 for paragraph 2, and the edit of paragraph 1 was lost). On the agent's side
213
+ // the rule is strict: a base paragraph whose id the agent's version, which has ids, no longer has was removed or rebuilt, and an
214
+ // edit of it is handed back. If several of the agent's paragraphs carry the id (one copied with it), only a text match that carries
215
+ // it counts. Without ids on either side, text and position alone decide, as before. In doubt the edit is handed back (unplaced),
216
+ // never applied to a paragraph the person did not edit (an outside review: an edit of the first paragraph overwrote the second one,
217
+ // whose text the agent had replaced with the first one's).
218
+ const { mapA: baseToMine, mapB: mineToBase, moved: same } = matchParagraphs(base, mine);
219
+ // A paragraph rewritten whole between two paragraphs the person left in place faces the first new paragraph between their copies: in
220
+ // the same place (body or table cell), it is that paragraph, edited (a short text, such as a table cell's, has no similarity to go
221
+ // by: "Item" retyped as "Mine" was a deletion and a new paragraph, and a line added under it went into another cell)
222
+ pairSlots(base, mine, baseToMine, mineToBase, paraIdentity(base, mine, { strict: true }).may);
223
+ // Which of the person's paragraphs moved: those the matcher found elsewhere with the same text, and of the others those outside a
224
+ // longest run that keeps the base order. A paragraph the person also edited counts like any other (an outside review: base [A, B, C,
225
+ // D], the person moved A after C and edited it, and the move was dropped without a word; so was the order of two paragraphs swapped
226
+ // and both edited).
227
+ const moved = new Set([...same, ...outOfOrder(Int32Array.from(mineToBase, (i, j) => (same.has(j) ? -1 : i)))]);
228
+ const theirSide = paraIdentity(base, theirs, { strict: true });
229
+ /** may paragraph n of the agent's version be base paragraph i, as far as the ids say (the rule above)? */
230
+ const agrees = theirSide.may;
231
+ // a base paragraph the agent left alone, by position: the same text in the same order, ids told apart
232
+ const { mapA: baseToTheirs } = lcsPairs(theirSide.keyA, theirSide.keyB);
233
+ const agent = matchParagraphs(base, theirs, { strict: true }); // where the agent's other paragraphs came from: an edited one, one it moved
234
+ // the agent changed the order of paragraphs too: then which order wins is not certain, and no move of the person's is carried
235
+ const agentMoved = outOfOrder(agent.mapA).size > 0;
236
+ const ids = (L) => {
237
+ const m = new Map(); // id -> the indexes of the paragraphs that carry it
238
+ L.forEach((p, n) => p.paraId && (m.get(p.paraId) || m.set(p.paraId, []).get(p.paraId)).push(n));
239
+ return m;
240
+ };
241
+ const baseIds = ids(base);
242
+ const theirIds = ids(theirs);
243
+ const ownId = (i) => (base[i].paraId && baseIds.get(base[i].paraId).length === 1 ? base[i].paraId : null);
244
+ /** base paragraph i where it stands in the agent's version (same text, same order), when the ids agree; else -1 */
245
+ const inPlace = (i) => {
246
+ const t = baseToTheirs[i];
247
+ return t >= 0 && theirs[t].path && agrees(i, t) ? t : -1;
248
+ };
249
+ /** inPlace(i), else the paragraph with its unique id when the agent moved it and left its text alone; else -1 */
250
+ const at = (i) => {
251
+ const t = inPlace(i);
252
+ if (t >= 0) return t;
253
+ const n = ownId(i) ? theirIds.get(ownId(i)) : null;
254
+ return n?.length === 1 && theirs[n[0]].path && theirs[n[0]].hash === base[i].hash ? n[0] : -1;
255
+ };
151
256
  const locator = (n) => ({ body_path: theirs[n].path, text_hash: theirs[n].hash });
152
- const theirsHas = new Set(theirs.map((p) => p.hash));
257
+ const work = { left: 4_000_000 + 16 * base.reduce((n, p) => n + p.text.length, 0) }; // minimalReplace over the whole plan
153
258
  const items = [];
154
259
  const unplaced = [];
155
- const no = (p, why, text = p.text) => unplaced.push({ label: label(p), text: cut(text), why });
260
+ // `of` names the paragraph an entry stands for, for the proof of the merge (src/merge-proof.js): "m" and the paragraph's ordinal in
261
+ // the person's version, or "b" and its ordinal in the base
262
+ const ref = (p) => `${base[p.ordinal] === p ? "b" : "m"}${p.ordinal}`;
263
+ const no = (p, why, text = p.text, of = [ref(p)]) => unplaced.push({ label: label(p), text: cut(text), why, of });
156
264
 
157
265
  // The agent's version of a base paragraph whose text it changed, only when there is no doubt which one it is: the same unique
158
- // paragraph id, or the one paragraph between the same two untouched neighbors.
159
- const ids = (L) => L.reduce((m, p) => (p.paraId ? m.set(p.paraId, (m.get(p.paraId) || 0) + 1) : m), new Map());
160
- let idCount = null;
266
+ // paragraph id, or the one paragraph between the same two untouched neighbors (and never one the ids say is another paragraph).
161
267
  const rewritten = (i) => {
162
268
  const n = agent.mapA[i];
163
- if (n < 0 || baseToTheirs[i] >= 0 || agent.moved.has(n) || !theirs[n].path) return -1;
164
- idCount ||= [ids(base), ids(theirs)];
165
- const id = base[i].paraId;
166
- if (id && theirs[n].paraId === id && idCount[0].get(id) === 1 && idCount[1].get(id) === 1) return n;
167
- const before = i === 0 ? n === 0 : baseToTheirs[i - 1] >= 0 && baseToTheirs[i - 1] === n - 1;
168
- const after = i === base.length - 1 ? n === theirs.length - 1 : baseToTheirs[i + 1] >= 0 && baseToTheirs[i + 1] === n + 1;
269
+ if (n < 0 || baseToTheirs[i] >= 0 || agent.moved.has(n) || !theirs[n].path || !agrees(i, n)) return -1;
270
+ if (theirIds.get(ownId(i))?.length === 1) return n; // the agent's version may have no ids at all (an outside review: this threw)
271
+ const before = i === 0 ? n === 0 : inPlace(i - 1) >= 0 && inPlace(i - 1) === n - 1;
272
+ const after = i === base.length - 1 ? n === theirs.length - 1 : inPlace(i + 1) >= 0 && inPlace(i + 1) === n + 1;
169
273
  return before && after ? n : -1;
170
274
  };
171
275
 
@@ -186,7 +290,7 @@ export function planDocxMerge(base, mine, theirs) {
186
290
  const mineWords = [...("style" in para.set ? ["style"] : []), ...("align" in para.set ? ["alignment"] : []), ...("list" in para.set || "ilvl" in para.set ? ["list"] : []), ...words];
187
291
  if (n < 0) {
188
292
  if (typed) return; // the text was handed back with its own reason
189
- return no(m, `You changed formatting here${mineWords.length ? ` (${list(mineWords)})` : ""}, but the agent changed, moved or removed this paragraph, so it was not carried over.`, "");
293
+ return no(m, `You changed formatting here${mineWords.length ? ` (${list(mineWords)})` : ""}, but the agent changed, moved or removed this paragraph, so it was not carried over.`, "", [ref(m), ref(b)]);
190
294
  }
191
295
  const a = theirs[n];
192
296
  const carry = {};
@@ -208,8 +312,8 @@ export function planDocxMerge(base, mine, theirs) {
208
312
  else notes.push(`The agent changed formatting inside this paragraph too, so the ${list(words)} you changed was not carried over.`);
209
313
  } else if (vague) notes.push(`Formatting inside this paragraph is different in your version (${vague}). It could not be carried over.`);
210
314
  if (cannot.length) notes.push(`Changes to ${list(cannot)} are not carried over.`);
211
- if (!done.length) return no(m, notes.join(" "), "");
212
- items.push({ kind: "format", label: label(m), text: "", hash: null, t: n, words: done, note: notes.join(" "), op: { op: "format_paragraph", target: locator(n), expect: { text_hash: a.hash }, ...(Object.keys(carry).length ? { para: carry } : {}), ...(runs.length ? { runs } : {}) } });
315
+ if (!done.length) return no(m, notes.join(" "), "", [ref(m), ref(b)]);
316
+ items.push({ kind: "format", label: label(m), of: [ref(m), ref(b)], text: "", hash: null, t: n, words: done, note: notes.join(" "), op: { op: "format_paragraph", target: locator(n), expect: { text_hash: a.hash }, ...(Object.keys(carry).length ? { para: carry } : {}), ...(runs.length ? { runs } : {}) } });
213
317
  };
214
318
 
215
319
  /**
@@ -228,11 +332,31 @@ export function planDocxMerge(base, mine, theirs) {
228
332
  return { note: "This paragraph has the formatting of the agent's version: if you changed its formatting too, set it again.", check: true };
229
333
  };
230
334
 
231
- // anchors for new paragraphs: the nearest paragraph before (else after) that the agent left alone. The user may have edited it
232
- // (a split leaves the first half as an edit of the original): every op is bound to the agent's file before any op runs.
335
+ // anchors for new paragraphs: the nearest paragraph before (else after) that the agent left alone, or rewrote when there is no doubt
336
+ // which paragraph of its version that is (an anchor past the agent's edited neighbor put the new paragraph on the wrong side of it).
337
+ // The user may have edited it (a split leaves the first half as an edit of the original): every op is bound to the agent's file
338
+ // before any op runs.
233
339
  const still = (j) => mineToBase[j] >= 0 && !moved.has(j); // a paragraph of the base that is where it was
234
- const anchorOf = (j) => (still(j) ? at(mineToBase[j]) : -1);
340
+ const placed = (i) => {
341
+ const t = inPlace(i);
342
+ if (t >= 0) return t;
343
+ const n = ownId(i) ? theirIds.get(ownId(i)) : null;
344
+ return n?.length === 1 && theirs[n[0]].path && agrees(i, n[0]) ? n[0] : rewritten(i);
345
+ };
346
+ const anchorOf = (j) => (still(j) ? placed(mineToBase[j]) : -1);
347
+ // the nearest paragraph of `mine` before and after j that passes `ok`, for every j at once (a walk from each paragraph was quadratic
348
+ // over a long stretch of new or moved paragraphs). -1 / mine.length: none.
349
+ const nearest = (ok) => {
350
+ const prev = new Int32Array(mine.length);
351
+ const next = new Int32Array(mine.length);
352
+ for (let j = 0, last = -1; j < mine.length; j++) (prev[j] = last), ok(j) && (last = j);
353
+ for (let j = mine.length - 1, last = mine.length; j >= 0; j--) (next[j] = last), ok(j) && (last = j);
354
+ return { prev, next };
355
+ };
356
+ let nearStill = null;
357
+ let nearAnchor = null;
235
358
  const inserts = []; // {where: "after"|"before", to: index in theirs, m, move?: index in theirs of the paragraph to move}
359
+ let unanchored = false; // a paragraph of the user's could not be placed: it may be one of the "deleted" ones, rewritten past recognition
236
360
 
237
361
  /**
238
362
  * A paragraph the user moved: the agent's paragraph is moved as it is, placed by the paragraph right before it in the user's
@@ -242,20 +366,20 @@ export function planDocxMerge(base, mine, theirs) {
242
366
  * report only (`lost`).
243
367
  */
244
368
  const move = (m, j, i) => {
245
- const n = baseToTheirs[i];
369
+ const n = inPlace(i);
246
370
  const more = lost(base[i], m);
247
371
  const t = theirs[n];
248
- const say = (why) => void no(m, why, "");
372
+ const say = (why) => void no(m, why, "", [ref(m), ref(base[i])]);
249
373
  if (!t) return say(agent.moved.has(agent.mapA[i]) ? "You moved this paragraph and so did the agent. It is where the agent put it." : "You moved this paragraph, but the agent changed or removed it, so the move was not carried over.");
374
+ if (agentMoved) return say("You moved this paragraph, but the agent moved paragraphs too, so the move was not carried over.");
250
375
  if (t.path?.length !== 1 || m.path?.length !== 1) return say("You moved this paragraph in or out of a table. Moves that involve tables are not carried over.");
251
376
  if (t.sect) return say("You moved this paragraph, but it ends a section of the document, so the move was not carried over.");
252
377
  if (t.loose) return say("You moved this paragraph, but a comment or bookmark starts or ends inside it, so the move was not carried over.");
253
378
  const body = (k) => (k >= 0 && k < mine.length && anchorOf(k) >= 0 && theirs[anchorOf(k)].path.length === 1 ? anchorOf(k) : -1);
254
- let k = j - 1;
255
- while (k >= 0 && !still(k)) k--;
379
+ nearStill ||= nearest(still);
380
+ let k = nearStill.prev[j];
256
381
  if (body(k) >= 0) return void inserts.push({ where: "after", to: body(k), m, move: n, ...more });
257
- k = j + 1;
258
- while (k < mine.length && !still(k)) k++;
382
+ k = nearStill.next[j];
259
383
  if (body(k) >= 0) return void inserts.push({ where: "before", to: body(k), m, move: n, ...more });
260
384
  say("You moved this paragraph, but the agent changed the paragraphs next to its new place, so the move was not carried over.");
261
385
  };
@@ -264,43 +388,73 @@ export function planDocxMerge(base, mine, theirs) {
264
388
  const i = mineToBase[j];
265
389
  if (i >= 0) {
266
390
  const b = base[i];
267
- if (moved.has(j)) return move(m, j, i); // the same text in another place
268
- if (b.hash === m.hash) return normText(m.text) ? format(b, m, i, false) : undefined;
391
+ // moved into another table or cell, its order among the paragraphs kept: the move is not carried (a move that changes the order
392
+ // says so in move()); its text and formatting are, where it stands in the agent's version
393
+ if (!sameBox(b, m) && !moved.has(j)) no(m, "You moved this paragraph into another table or table cell. Moves that involve tables are not carried over.", "", [ref(m), ref(b)]);
394
+ if (b.text === m.text) return moved.has(j) ? move(m, j, i) : format(b, m, i, false);
395
+ // The text changed. White space counts as written: a line break, a tab or a second space is an edit too, though the text hashes the
396
+ // same (an outside review: a line break typed into "A B" was dropped without a word). So does the agent's, on its side.
269
397
  const t = at(i);
270
- if (t < 0 && theirsHas.has(m.hash)) return; // the same text is already there (an earlier pass carried it)
271
- if (t < 0) return no(m, "The agent changed this paragraph too. Its version is in the document; this is yours.");
272
- const r = minimalReplace(b.text, m.text);
398
+ const now = t >= 0 ? theirs[t] : agent.mapA[i] >= 0 && agrees(i, agent.mapA[i]) ? theirs[agent.mapA[i]] : null;
399
+ // the agent's own copy of this paragraph already reads as the user's (an earlier pass carried it). Only that one paragraph counts:
400
+ // the same text anywhere else in the agent's version says nothing about this one (an outside review lost an edit that way)
401
+ if (now?.text === m.text) return moved.has(j) ? move(m, j, i) : undefined;
402
+ if (t < 0 || now.text !== b.text) return no(m, "The agent changed this paragraph too. Its version is in the document; this is yours.", m.text, [ref(m), ref(b)]);
403
+ const r = minimalReplace(b.text, m.text, work);
273
404
  const op = { op: "replace_text", target: locator(t), expect: { text_hash: b.hash }, ...(r ? r : { text: m.text }) };
274
- items.push({ kind: "changed", label: label(m), text: cut(m.text), hash: m.hash, t, op });
275
- return format(b, m, i, true);
405
+ items.push({ kind: "changed", label: label(m), of: [ref(m), ref(b)], text: cut(m.text), hash: m.hash, raw: m.text, m, t, op });
406
+ format(b, m, i, true);
407
+ if (moved.has(j)) move(m, j, i); // moved as well: the agent's paragraph, edited first, is moved
408
+ return;
276
409
  }
277
- if (!normText(m.text)) return; // an empty line that was added: nothing to lose
278
- let k = j - 1;
279
- while (k >= 0 && anchorOf(k) < 0) k--;
280
- if (k >= 0) return inserts.push({ where: "after", to: anchorOf(k), m });
281
- k = j + 1;
282
- while (k < mine.length && anchorOf(k) < 0) k++;
283
- if (k < mine.length) return inserts.push({ where: "before", to: anchorOf(k), m });
410
+ nearAnchor ||= nearest((x) => anchorOf(x) >= 0);
411
+ // placed by a paragraph in the same place: the body, or the same table cell (a new line typed in a cell was put in the body)
412
+ let k = nearAnchor.prev[j];
413
+ if (k >= 0 && sameBox(mine[k], m) && sameBox(theirs[anchorOf(k)], mine[k])) return inserts.push({ where: "after", to: anchorOf(k), m });
414
+ k = nearAnchor.next[j];
415
+ if (k < mine.length && sameBox(mine[k], m) && sameBox(theirs[anchorOf(k)], mine[k])) return inserts.push({ where: "before", to: anchorOf(k), m });
416
+ unanchored = true;
284
417
  no(m, "There was no unchanged paragraph near this new one to place it by.");
285
418
  });
286
419
  // several new or moved paragraphs after one anchor: each lands right after the anchor, so the last goes first
287
420
  for (const x of inserts.filter((x) => x.where === "after").reverse().concat(inserts.filter((x) => x.where === "before"))) {
288
421
  const to = locator(x.to);
289
- if (x.move === undefined) items.push({ kind: "added", label: label(x.m), text: cut(x.m.text), hash: x.m.hash, t: x.to, where: x.where, op: { op: x.where === "after" ? "insert_paragraph_after" : "insert_paragraph_before", target: to, expect: { text_hash: to.text_hash }, text: x.m.text } });
290
- else items.push({ kind: "moved", label: label(x.m), text: "", hash: x.m.hash, t: x.move, to: x.to, where: x.where, ...(x.note ? { note: x.note, check: x.check } : {}), op: { op: "move_paragraph", target: locator(x.move), expect: { text_hash: x.m.hash }, to, where: x.where } });
422
+ if (x.move === undefined) items.push({ kind: "added", label: label(x.m), of: [ref(x.m)], text: cut(x.m.text), hash: x.m.hash, m: x.m, t: x.to, where: x.where, op: { op: x.where === "after" ? "insert_paragraph_after" : "insert_paragraph_before", target: to, expect: { text_hash: to.text_hash }, text: x.m.text } });
423
+ else items.push({ kind: "moved", label: label(x.m), of: [ref(x.m), ref(base[mineToBase[x.m.ordinal]])], text: "", hash: x.m.hash, t: x.move, to: x.to, where: x.where, ...(x.note ? { note: x.note, check: x.check } : {}), op: { op: "move_paragraph", target: locator(x.move), expect: { text_hash: theirs[x.move].hash }, to, where: x.where } });
291
424
  }
425
+ // A base paragraph with no match in the user's version is deleted only when every paragraph of the user's could be placed. When one could
426
+ // not (the matcher's similarity budget ran out on a long run of edited paragraphs, so none of them matched), the unmatched base paragraphs
427
+ // may be those very paragraphs, rewritten: deleting them removed the whole document (a merge of 49,999 edited paragraphs left one).
428
+ const kept = [];
292
429
  base.forEach((b, i) => {
293
- if (baseToMine[i] >= 0 || !normText(b.text)) return;
294
- const t = at(i);
295
- if (t >= 0) items.push({ kind: "removed", label: label(b), text: "", hash: null, t, op: { op: "delete_paragraph", target: locator(t), expect: { text_hash: b.hash } } });
430
+ if (baseToMine[i] >= 0) return;
431
+ const t = at(i) >= 0 && theirs[at(i)].text === b.text ? at(i) : -1; // the agent's white space counts too
432
+ if (t >= 0 && unanchored) kept.push(ref(b));
433
+ else if (t >= 0) items.push({ kind: "removed", label: label(b), of: [ref(b)], text: "", hash: null, t, op: { op: "delete_paragraph", target: locator(t), expect: { text_hash: b.hash } } });
296
434
  else no(b, "You deleted this paragraph, but the agent changed it, so it was kept.", "");
297
435
  });
436
+ if (kept.length) unplaced.push({ label: "Deleted paragraphs", text: "", why: `${kept.length === 1 ? "A paragraph you deleted was" : `${kept.length} paragraphs you deleted or rewrote were`} kept, because Onionskin could not place your other edits next to ${kept.length === 1 ? "it" : "them"}.`, of: kept });
437
+ // A table cell holds at least one paragraph: deleting all of a cell's paragraphs leaves a cell Word reports as damaged (found by the
438
+ // proof of the merge: a table row the person deleted came back as a row of empty cells). Those deletions are a row or a cell the
439
+ // person removed, which is not carried: they are handed back and the paragraphs stay.
440
+ const cellOf = (p) => (p.path?.length === 4 ? `${p.tableNo} ${p.path[1]} ${p.path[2]}` : null);
441
+ const left = new Map(); // cell of the agent's version -> paragraphs it keeps after the plan
442
+ for (const p of theirs) if (cellOf(p)) left.set(cellOf(p), (left.get(cellOf(p)) || 0) + 1);
443
+ for (const it of items) if (it.kind === "removed" && cellOf(theirs[it.t])) left.set(cellOf(theirs[it.t]), left.get(cellOf(theirs[it.t])) - 1);
444
+ for (const it of items) if (it.kind === "added" && cellOf(theirs[it.t])) left.set(cellOf(theirs[it.t]), left.get(cellOf(theirs[it.t])) + 1);
445
+ for (let k = items.length - 1; k >= 0; k--) {
446
+ const it = items[k];
447
+ if (it.kind !== "removed" || !cellOf(theirs[it.t]) || left.get(cellOf(theirs[it.t])) > 0) continue;
448
+ items.splice(k, 1);
449
+ unplaced.push({ label: it.label, text: "", why: "You deleted this paragraph with the table row or cell it was in. Rows and cells are not carried over, so it was kept.", of: it.of });
450
+ }
298
451
  // replacements first, then formatting, then new and moved paragraphs, then deletions (an anchor is never a paragraph this plan deletes)
299
452
  const order = { changed: 0, format: 1, added: 2, moved: 2, removed: 3 };
300
453
  items.sort((x, y) => order[x.kind] - order[y.kind]);
301
454
  return { items, unplaced };
302
455
  }
303
456
 
457
+ const turn = () => new Promise((resolve) => setImmediate(resolve));
304
458
  const SOFT = new Set(["format", "moved"]); // carried only when the result proves them; the merge can always do without
305
459
  const PROTECTED = "a protected paragraph (it holds a field, a content control or something else that cannot be rewritten safely)";
306
460
  /** Why one item is not in the document, in words for the user. `r` is the op's result when it ran. */
@@ -313,24 +467,42 @@ function reason(it, r) {
313
467
 
314
468
  /** The order of paragraphs (as text hashes, with the index in `theirs` each came from) the applied ops must give. */
315
469
  function expected(theirs, items, results) {
316
- const seq = theirs.map((p, n) => ({ n, hash: p.hash }));
317
- const find = (n) => seq.findIndex((x) => x.n === n);
470
+ // a linked list of the agent's paragraphs, each found by its index in `theirs` (a search and a splice per item was quadratic)
471
+ const head = { next: null };
472
+ const tail = { prev: head };
473
+ head.next = tail;
474
+ const put = (a, x) => ((x.prev = a), (x.next = a.next), (a.next.prev = x), (a.next = x), (x.live = true)); // x right after a
475
+ const take = (x) => ((x.prev.next = x.next), (x.next.prev = x.prev), (x.live = false));
476
+ const nodes = theirs.map((p, n) => {
477
+ const x = { n, hash: p.hash };
478
+ put(tail.prev, x);
479
+ return x;
480
+ });
481
+ const live = (n) => (nodes[n]?.live ? nodes[n] : null);
318
482
  items.forEach((it, k) => {
319
483
  if (results[k]?.status !== "applied") return;
320
- const i = find(it.t);
321
- if (i < 0) return;
322
- if (it.kind === "changed") seq[i] = { n: it.t, hash: it.hash };
323
- else if (it.kind === "removed") seq.splice(i, 1);
324
- else if (it.kind === "added") seq.splice(it.where === "after" ? i + 1 : i, 0, { n: -1, hash: it.hash });
484
+ const x = live(it.t);
485
+ if (!x) return;
486
+ if (it.kind === "changed") x.hash = it.hash;
487
+ else if (it.kind === "removed") take(x);
488
+ else if (it.kind === "added") put(it.where === "after" ? x : x.prev, { n: -1, hash: it.hash, it });
325
489
  else if (it.kind === "moved") {
326
- const [x] = seq.splice(i, 1);
327
- const to = find(it.to);
328
- seq.splice(it.where === "after" ? to + 1 : to, 0, x);
490
+ take(x);
491
+ const to = live(it.to); // always there (an anchor is never a paragraph the plan removes); if not, where a splice at -1 put it
492
+ put(to ? (it.where === "after" ? to : to.prev) : it.where === "after" || tail.prev === head ? head : tail.prev.prev, x);
329
493
  }
330
494
  });
495
+ const seq = [];
496
+ for (let x = head.next; x !== tail; x = x.next) seq.push({ n: x.n, hash: x.hash, it: x.it });
331
497
  return seq;
332
498
  }
333
499
 
500
+ /** Do two readings of the same text look the same: paragraph style, alignment, list, other paragraph settings and runs by character? */
501
+ function sameLook(a, b) {
502
+ const runs = runPatches(a.spans, b.spans);
503
+ return styleKey(a.styleName || a.style) === styleKey(b.styleName || b.style) && a.align === b.align && listSig(a) === listSig(b) && a.ppr === b.ppr && !!runs && !runs.other && !runs.ranges.length;
504
+ }
505
+
334
506
  /** Is the paragraph `o` of the result the agent's paragraph `t` with exactly the formatting this item set, and nothing else different? */
335
507
  function formatted(it, t, o) {
336
508
  const para = it.op.para || {};
@@ -351,18 +523,26 @@ function formatted(it, t, o) {
351
523
  */
352
524
  export async function mergeDocx({ base, mine, theirs, name, runOps }) {
353
525
  const read = (bytes) => readDocxParagraphs(bytes, { detail: true });
526
+ // a turn of the event loop between the readings and the plan: at 50,000 paragraphs each takes 0.2 to 0.3 s, all four in a row 0.9 s
354
527
  const agents = read(theirs);
355
- const plan = planDocxMerge(read(base), read(mine), agents);
528
+ await turn();
529
+ const before = read(base);
530
+ await turn();
531
+ const after = read(mine);
532
+ await turn();
533
+ const plan = planDocxMerge(before, after, agents);
534
+ await turn();
356
535
  const unplaced = [...plan.unplaced];
357
- if (!plan.items.length) return { bytes: null, merged: 0, unplaced };
536
+ const readings = { base: before, mine: after, theirs: agents }; // for the proof of the merge (src/merge-worker.js), read once
537
+ if (!plan.items.length) return { bytes: null, merged: 0, unplaced, read: readings };
358
538
  let r = await attempt(plan.items, { theirs, agents, name, runOps, read });
359
539
  if (r.redo) {
360
540
  // a move or a formatting change did not come out as planned: nothing unproven is written, the rest is carried without them
361
- for (const it of plan.items.filter((x) => SOFT.has(x.kind))) unplaced.push({ label: it.label, text: "", why: reason(it) });
541
+ for (const it of plan.items.filter((x) => SOFT.has(x.kind))) unplaced.push({ label: it.label, text: "", why: reason(it), of: it.of });
362
542
  const rest = plan.items.filter((x) => !SOFT.has(x.kind));
363
543
  r = rest.length ? await attempt(rest, { theirs, agents, name, runOps, read }) : { bytes: null, merged: 0, unplaced: [] };
364
544
  }
365
- return { bytes: r.bytes, merged: r.merged, unplaced: [...unplaced, ...r.unplaced] };
545
+ return { bytes: r.bytes, merged: r.merged, unplaced: [...unplaced, ...r.unplaced], read: readings };
366
546
  }
367
547
 
368
548
  /** One run of `items` on the agent's bytes, checked on the result. {redo: true} asks for a run without moves and formatting. */
@@ -370,13 +550,14 @@ async function attempt(items, { theirs, agents, name, runOps, read }) {
370
550
  const unplaced = [];
371
551
  const soft = items.some((x) => SOFT.has(x.kind));
372
552
  const { results, bytes, blocked } = await runOps({ bytes: theirs, format: "docx", name, ops: items.map((x) => x.op), partial: true });
373
- const fail = (it, why) => unplaced.push({ label: it.label, text: it.text, why });
553
+ const fail = (it, why) => unplaced.push({ label: it.label, text: it.text, why, of: it.of });
374
554
  if (!bytes) {
375
555
  if (blocked && soft) return { redo: true };
376
556
  for (const [n, it] of items.entries()) fail(it, blocked ? "Carrying your edits over would have damaged the file, so none were applied." : reason(it, results[n]));
377
557
  return { bytes: null, merged: 0, unplaced };
378
558
  }
379
559
  // the proof is the result, not the report. First: nothing of the agent's version is gone except what an op named.
560
+ await turn();
380
561
  const out = read(bytes);
381
562
  const count = (list) => list.reduce((m, h) => m.set(h, (m.get(h) || 0) + 1), new Map());
382
563
  const named = count(items.filter((x, n) => (x.kind === "changed" || x.kind === "removed") && results[n]?.status === "applied").map((x) => x.op.expect.text_hash));
@@ -396,14 +577,36 @@ async function attempt(items, { theirs, agents, name, runOps, read }) {
396
577
  const where = new Map(seq.map((x, k) => [x.n, k]));
397
578
  if (items.some((it, n) => it.kind === "format" && results[n]?.status === "applied" && !formatted(it, agents[it.t], out[where.get(it.t)]))) return { redo: true };
398
579
  }
399
- // then: every carried text is in the merged file
580
+ // then: every carried text is in the merged file, and a changed paragraph reads as typed, white space as written, in its own place (the
581
+ // same text elsewhere proves nothing about it) whenever the result has the paragraphs the plan expects
400
582
  const have = new Set(out.map((p) => p.hash));
583
+ const seq = expected(agents, items, results);
584
+ const place = seq.length === out.length ? new Map(seq.map((x, k) => [x.n, k])) : null;
585
+ if (place && items.some((it, n) => it.kind === "changed" && results[n]?.status === "applied" && out[place.get(it.t)]?.text !== it.raw)) {
586
+ // a paragraph of the agent's now reads as neither version: nothing is written
587
+ if (soft) return { redo: true };
588
+ for (const it of items) fail(it, "Carrying your edits over would have changed the agent's text, so none were applied.");
589
+ return { bytes: null, merged: 0, unplaced };
590
+ }
591
+ // The look the person gave a paragraph they added, or the inside of one they typed in, that the result does not have is said (a check
592
+ // line), never dropped without a word: the engine gives a new paragraph the paragraph formatting of the one it is placed by, and typed
593
+ // text the look of the run it lands in. The proof of the merge (src/merge-proof.js) refuses a result that differs without such a line.
594
+ if (place) {
595
+ const added = new Map();
596
+ seq.forEach((x, k) => x.it && added.set(x.it, k));
597
+ items.forEach((it, n) => {
598
+ if (results[n]?.status !== "applied" || !it.m || (it.kind !== "added" && it.kind !== "changed")) return;
599
+ const o = out[it.kind === "added" ? added.get(it) : place.get(it.t)];
600
+ if (!o || o.text !== it.m.text || sameLook(it.m, o)) return;
601
+ unplaced.push({ label: it.label, text: "", why: it.kind === "added" ? "Your new paragraph was carried over with the formatting of the paragraph next to it: if you formatted it, set it again." : "Your text was carried over, but the formatting inside this paragraph is not as you had it: check it.", of: it.of, check: true });
602
+ });
603
+ }
401
604
  let merged = 0;
402
605
  items.forEach((it, n) => {
403
606
  if (results[n]?.status !== "applied") return fail(it, reason(it, results[n]));
404
607
  if (it.hash && !have.has(it.hash)) return fail(it, "This edit did not land as typed.");
405
608
  merged++;
406
- if (it.note && it.kind === "moved") unplaced.push({ label: it.label, text: "", why: `Your move was carried over. ${it.note}`, ...(it.check ? { check: true } : {}) });
609
+ if (it.note && it.kind === "moved") unplaced.push({ label: it.label, text: "", why: `Your move was carried over. ${it.note}`, of: it.of, ...(it.check ? { check: true } : {}) });
407
610
  else if (it.note) fail(it, `Your ${list(it.words)} change was carried over. ${it.note}`);
408
611
  });
409
612
  return { bytes, merged, unplaced };
package/src/nfc.js ADDED
@@ -0,0 +1,46 @@
1
+ // Unicode normalization in time linear in the text. Pure and without imports: the server, the CLI and the page (web/src) normalize
2
+ // document text with this one function, so the same text hashes the same everywhere.
3
+ //
4
+ // String.prototype.normalize sorts each run of combining marks by combining class, in time quadratic in the run's length: one Word
5
+ // paragraph of "a" followed by 120,000 marks held the server's thread for 4 to 6 seconds (an outside review). Unicode's Stream-Safe
6
+ // Text Format (UAX #15) bounds a run at 30 non-starters, which no real text needs more of. A run of more than MAX_MARKS characters of
7
+ // general category M (every non-starter is one) is left as it is, and the text between such runs is normalized: one linear scan, no
8
+ // sort longer than MAX_MARKS. Text without such a run (every real script: Vietnamese, Arabic with harakat, Hebrew with niqqud,
9
+ // Devanagari, emoji sequences, Zalgo text within the limit) is normalized whole, exactly as before, so hashes stored by earlier
10
+ // versions of ordinary text do not change. The result depends on the text alone, and normalizing it again gives it back.
11
+ // Ceiling: an over-long run is not reordered or composed (its base letter stays apart from its first mark).
12
+ // Compatibility forms (NFKC, NFKD) too: U+FF9E and U+FF9F (halfwidth katakana voiced and semi-voiced sound marks) are not of category M
13
+ // but decompose to the combining marks U+3099 and U+309A, so "゙̕" repeated built the same unbounded combining sequence out of what
14
+ // the run pattern did not see (2.4 s at 40,000 pairs, four times the 20,000's). They count as marks in a run. test/nfc.test.js scans
15
+ // every code point for another character whose compatibility decomposition starts with a mark: these two (and the Thai and Lao
16
+ // sara am, whose first character has combining class 0, so nothing to sort) are all there are.
17
+
18
+ export const MAX_MARKS = 30;
19
+ // at most CHUNK marks a match, joined into runs by the loop below: an unbounded match of 7.9 million marks overflowed the regex stack
20
+ // ("a" + "\u0315\u0300" x 3,949,999: Send answered 422 "Maximum call stack size exceeded")
21
+ const CHUNK = 4096;
22
+ const RUNS = new RegExp(`[\\p{M}\\uFF9E\\uFF9F]{1,${CHUNK}}`, "gu");
23
+
24
+ /** s.normalize(form), linear in the length of s (see above). Null and undefined read as "". */
25
+ export function nfc(s, form = "NFC") {
26
+ s = String(s ?? "");
27
+ let out = null;
28
+ let from = 0;
29
+ let start = -1; // the run being read: [start, end)
30
+ let end = -1;
31
+ const close = () => {
32
+ // UTF-16 units first (never fewer than the marks), then the marks themselves (a mark outside the BMP is two units)
33
+ if (start < 0 || end - start <= MAX_MARKS || (end - start <= 2 * MAX_MARKS && [...s.slice(start, end)].length <= MAX_MARKS)) return;
34
+ (out ||= []).push(s.slice(from, start).normalize(form), s.slice(start, end));
35
+ from = end;
36
+ };
37
+ RUNS.lastIndex = 0;
38
+ for (let m; (m = RUNS.exec(s)); ) {
39
+ if (m.index !== end) (close(), (start = m.index));
40
+ end = m.index + m[0].length;
41
+ }
42
+ close();
43
+ if (!out) return s.normalize(form);
44
+ out.push(s.slice(from).normalize(form));
45
+ return out.join("");
46
+ }