onionskin-cli 0.0.2 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/CHANGELOG.md +111 -0
  2. package/INSTALL.md +5 -1
  3. package/NOTICE +34 -0
  4. package/README.md +101 -33
  5. package/docs/contracts/batch.schema.json +27 -4
  6. package/docs/contracts/ops.schema.json +117 -5
  7. package/package.json +4 -2
  8. package/skills/onionskin/SKILL.md +9 -6
  9. package/skills/onionskin/reference/batch.md +5 -3
  10. package/skills/onionskin/reference/formats.md +5 -1
  11. package/skills/onionskin/reference/ops.md +47 -3
  12. package/skills/onionskin/reference/troubleshooting.md +31 -0
  13. package/src/agent-detect.js +28 -2
  14. package/src/agent-prompt.js +12 -1
  15. package/src/agent-runner.js +77 -19
  16. package/src/anchor-work.js +154 -0
  17. package/src/anchor-worker.js +38 -0
  18. package/src/anchors.js +61 -11
  19. package/src/apply.js +119 -47
  20. package/src/batch.js +32 -8
  21. package/src/check-worker.js +8 -5
  22. package/src/cli.js +91 -25
  23. package/src/constants.js +34 -1
  24. package/src/diff.js +3 -1
  25. package/src/docx.js +368 -58
  26. package/src/edit-hash.js +19 -0
  27. package/src/fsutil.js +1 -1
  28. package/src/history-worker.js +8 -8
  29. package/src/history.js +135 -31
  30. package/src/html-active.js +384 -0
  31. package/src/html-envelope.js +51 -0
  32. package/src/html-parser.js +8 -0
  33. package/src/html-render-worker.js +25 -0
  34. package/src/html-render.js +880 -0
  35. package/src/html-utf8.js +65 -0
  36. package/src/html-write-worker.js +14 -0
  37. package/src/html-write.js +380 -0
  38. package/src/html.js +1257 -0
  39. package/src/local.js +68 -4
  40. package/src/merge-package.js +1110 -0
  41. package/src/merge-pptx.js +98 -38
  42. package/src/merge-proof.js +1019 -0
  43. package/src/merge-worker.js +60 -0
  44. package/src/merge-xlsx.js +84 -29
  45. package/src/merge.js +286 -83
  46. package/src/nfc.js +46 -0
  47. package/src/notes.js +122 -0
  48. package/src/offload.js +400 -1
  49. package/src/open-worker.js +27 -0
  50. package/src/ops/common.js +2 -1
  51. package/src/ops/docx.js +58 -29
  52. package/src/ops/html.js +394 -0
  53. package/src/ops/pptx.js +74 -21
  54. package/src/ops/run.js +23 -3
  55. package/src/ops/runs.js +215 -13
  56. package/src/ops/schema.js +8 -1
  57. package/src/ops/xlsx.js +12 -7
  58. package/src/ops-worker.js +17 -0
  59. package/src/package.js +26 -2
  60. package/src/person.js +8 -1
  61. package/src/pptx.js +85 -21
  62. package/src/quick-plan.js +13 -6
  63. package/src/redline-worker.js +10 -14
  64. package/src/reply.js +46 -19
  65. package/src/retain.js +223 -0
  66. package/src/review.js +32 -18
  67. package/src/rewrite-check.js +2 -2
  68. package/src/sanitize.js +16 -3
  69. package/src/server.js +88 -37
  70. package/src/session.js +1143 -208
  71. package/src/stop-gate.js +34 -0
  72. package/src/suggest-reply.js +42 -7
  73. package/src/suggest.js +14 -29
  74. package/src/text.js +161 -11
  75. package/src/update.js +288 -0
  76. package/src/voice-guide.js +4 -3
  77. package/src/voice-learn.js +4 -3
  78. package/src/voice-sample-worker.js +7 -6
  79. package/src/voice-worker.js +4 -6
  80. package/src/watch.js +188 -0
  81. package/src/worker-host.js +23 -0
  82. package/src/xlsx.js +355 -94
  83. package/src/xml-tree.js +6 -3
  84. package/src/xml.js +82 -34
  85. package/src/zip-write.js +2 -1
  86. package/src/zip.js +9 -5
  87. package/vendor/parse5/LICENSE.txt +32 -0
  88. package/vendor/parse5/parse5.mjs +7822 -0
  89. package/vendor/parse5.lock.json +23 -0
  90. package/web/dist/assets/index-DkNx0LSH.css +1 -0
  91. package/web/dist/assets/{index-BUtofdtr.js → index-DytKkMdi.js} +117 -107
  92. package/web/dist/index.html +2 -2
  93. package/web/dist/assets/index-Dfizlg0i.css +0 -1
package/src/docx.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import { pathToFileURL } from "node:url";
2
2
  import { engineNodePath } from "./ops/engine.js";
3
- import { walkXml } from "./xml.js";
4
- import { normText, similarity, textHash } from "./text.js";
3
+ import { readBudget, tooMany, walkXml } from "./xml.js";
4
+ import { OCCURRENCE, affords, bigramCodes, diceOf, hitsOf, normPaid, normText, searchCost, similarity, spend, textHash, weigh, weightOf } from "./text.js";
5
5
  import { openZip } from "./zip.js";
6
6
  import { summarizeRuns } from "./fmt.js";
7
7
 
@@ -43,6 +43,16 @@ function runSet(name, a) {
43
43
  if (name === "w:vertAlign") return only(a, "w:val") && (v === "superscript" || v === "subscript") ? ["va", v.slice(0, 3)] : null;
44
44
  return null;
45
45
  }
46
+ // What one reading of a Word document may hold. The server reads a document on its own thread at Send, at each delivery to an agent, for a
47
+ // queued comment and at the end of a run, so these bound how long one such reading can keep it from answering anything else (the outside
48
+ // review: a part of 1 million runs held it 4.3 s, a run of 3 million attributes 2.4 s). A real report of about 1,000 pages has about 16,000
49
+ // paragraphs, 240,000 elements, 350,000 attributes and 3 million characters (measured: read in about 240 ms here).
50
+ export const MAX_PARAGRAPHS = 50_000;
51
+ export const MAX_ELEMENTS = 400_000;
52
+ export const MAX_ATTRIBUTES = 800_000;
53
+ export const MAX_TEXT = 8_000_000;
54
+ /** One reading's budget (src/xml.js readBudget), shared by the parts it walks. */
55
+ export const docxBudget = () => readBudget({ elements: MAX_ELEMENTS, attrs: MAX_ATTRIBUTES, text: MAX_TEXT });
46
56
  const PPR_OWN = new Set(["w:pStyle", "w:jc", "w:numPr", "w:rPr", "w:sectPr"]); // read on their own, or not paragraph formatting the user sets
47
57
  const RANGE = /^(.+)(Start|End)$/;
48
58
 
@@ -55,7 +65,7 @@ const RANGE = /^(.+)(Start|End)$/;
55
65
  * (null when they do not add up to it); `list` {numId, ilvl} from its own w:numPr, else null; `ppr`, its other paragraph properties as
56
66
  * written; `sect`, it ends a section; `loose`, a bookmark, comment or other range starts or ends inside it without its other end.
57
67
  */
58
- export function parseDocxParagraphs(xml, { detail = false } = {}) {
68
+ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget() } = {}) {
59
69
  const paragraphs = [];
60
70
  const stack = []; // {name, ...}
61
71
  let bodyIndex = -1;
@@ -107,6 +117,7 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
107
117
  path = [info.tableIndex, info.row, info.col, cellK++];
108
118
  tableNo = tableCount;
109
119
  }
120
+ if (paragraphs.length >= MAX_PARAGRAPHS) throw tooMany(budget, MAX_PARAGRAPHS, "paragraphs");
110
121
  cur = { ordinal: paragraphs.length, path, tableNo, paraId: /^[0-9A-Fa-f]{8}$/.test(attrs["w14:paraId"] || "") ? attrs["w14:paraId"].toUpperCase() : null, text: "", instr: "", protected: false, style: null, align: "", runs: [] };
111
122
  if (detail) (Object.assign(cur, { spans: [], list: null, ppr: [], sect: false, loose: false }), (marks = new Set()), (pprTop = null));
112
123
  paragraphs.push(cur);
@@ -175,7 +186,7 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
175
186
  }
176
187
  if (detail && name === pprTop && stack[stack.length - 1] === "w:pPr") pprTop = null;
177
188
  },
178
- });
189
+ }, budget);
179
190
  for (const p of paragraphs) p.hash = textHash(p.text);
180
191
  const reasons = protectReasons(xml);
181
192
  // the engine lists paragraphs the same way; if the two ever disagree on the count, nothing can be trusted as editable
@@ -204,10 +215,28 @@ export function protectInfo(p) {
204
215
  }
205
216
 
206
217
  const HEADING = /^(heading\s?\d|title|subtitle)$/i;
207
- /** The text of the nearest heading above a paragraph (by paragraph style), or "" when there is none. */
218
+ // the nearest heading at or before each ordinal, found in one pass per reading: a scan up from every resolved comment was a pass over
219
+ // the paragraphs per comment, and the heading's text was normalized whole again each time, outside the budget
220
+ const HEADS = new WeakMap();
221
+ /** The text of the nearest heading above a paragraph (by paragraph style), or "" when there is none (or the budget cannot pay for it). */
208
222
  export function headingBefore(paragraphs, ordinal) {
209
- for (let i = ordinal - 1; i >= 0; i--) if (paragraphs[i].style && HEADING.test(paragraphs[i].style) && paragraphs[i].text.trim()) return normText(paragraphs[i].text).slice(0, 80);
210
- return "";
223
+ if (!Number.isInteger(ordinal) || ordinal <= 0) return "";
224
+ let at = HEADS.get(paragraphs);
225
+ if (!at) {
226
+ at = new Int32Array(paragraphs.length + 1);
227
+ let last = -1;
228
+ for (let i = 0; i < paragraphs.length; i++) {
229
+ at[i] = last; // the heading strictly before paragraph i
230
+ const p = paragraphs[i];
231
+ if (p.style && HEADING.test(p.style) && p.text.trim()) last = i;
232
+ }
233
+ at[paragraphs.length] = last;
234
+ HEADS.set(paragraphs, at);
235
+ }
236
+ const h = at[Math.min(ordinal, paragraphs.length)];
237
+ if (h < 0) return "";
238
+ const p = paragraphs[h];
239
+ return spend(normCost(p)) ? norm(p).slice(0, 80) : "";
211
240
  }
212
241
 
213
242
  /** `detail` (the merge only): formatting per run and the paragraph's list, see parseDocxParagraphs; `list.kind` is "bullet", "number" or "other". */
@@ -215,11 +244,12 @@ export function readDocxParagraphs(bytes, { detail = false } = {}) {
215
244
  const zip = openZip(bytes);
216
245
  const xml = zip.readText("word/document.xml");
217
246
  if (xml == null) throw new Error("word/document.xml missing");
218
- const paragraphs = parseDocxParagraphs(xml, { detail });
219
- const names = styleNames(zip.readText("word/styles.xml"));
247
+ const budget = docxBudget(); // the main part, the styles and the numbering of one reading share it
248
+ const paragraphs = parseDocxParagraphs(xml, { detail, budget });
249
+ const names = styleNames(zip.readText("word/styles.xml"), budget);
220
250
  if (names.size) for (const p of paragraphs) if (p.style && names.has(p.style)) p.styleName = names.get(p.style);
221
251
  if (detail && paragraphs.some((p) => p.list)) {
222
- const kinds = listKinds(zip.readText("word/numbering.xml"));
252
+ const kinds = listKinds(zip.readText("word/numbering.xml"), budget);
223
253
  for (const p of paragraphs) if (p.list) p.list = p.list.numId && p.list.numId !== "0" ? { ...p.list, kind: kinds.get(p.list.numId) || "other" } : null; // numId 0 switches a style's list off
224
254
  }
225
255
  return paragraphs;
@@ -229,7 +259,7 @@ export function readDocxParagraphs(bytes, { detail = false } = {}) {
229
259
  * w:numId -> "bullet" | "number" | "other", by the first level of its definition: the two kinds the editor's list buttons make and
230
260
  * the engine can set again. A list with level overrides, or one this cannot read, is "other".
231
261
  */
232
- function listKinds(xml) {
262
+ function listKinds(xml, budget) {
233
263
  const out = new Map();
234
264
  if (xml == null) return out;
235
265
  const fmt = new Map(); // abstractNumId -> numFmt of level 0
@@ -258,13 +288,13 @@ function listKinds(xml) {
258
288
  num = null;
259
289
  }
260
290
  },
261
- });
291
+ }, budget);
262
292
  } catch {} // an unreadable numbering part: every list is "other", which is reported, never set
263
293
  return out;
264
294
  }
265
295
 
266
296
  /** w:styleId -> the style's display name ("heading 2" is shown as "Heading 2"); empty when styles.xml is missing or unreadable. */
267
- function styleNames(xml) {
297
+ function styleNames(xml, budget) {
268
298
  const out = new Map();
269
299
  if (xml == null) return out;
270
300
  let id = null;
@@ -277,7 +307,7 @@ function styleNames(xml) {
277
307
  onClose(name) {
278
308
  if (name === "w:style") id = null;
279
309
  },
280
- });
310
+ }, budget);
281
311
  } catch {} // a damaged styles part only costs the readable style names
282
312
  return out;
283
313
  }
@@ -287,15 +317,96 @@ const PARA_TEXT_KEEP = 2000;
287
317
 
288
318
  // ---- engine location -> durable anchor ----------------------------------------------------------------------
289
319
 
320
+ // A paragraph's normalized text, kept per paragraph object: every comment resolved against the same reading reuses it.
321
+ const NORM = new WeakMap();
322
+ const norm = (p) => {
323
+ let v = NORM.get(p);
324
+ if (v === undefined) NORM.set(p, (v = normText(p.text)));
325
+ return v;
326
+ };
327
+ /** what norm(p) is about to read: the paragraph's text the first time (src/text.js weigh), nothing once it is kept (spent from the request's budget) */
328
+ const normCost = (p) => (NORM.has(p) ? 0 : weightOf(p));
329
+ // the byte tables a search is priced by (src/text.js hitsOf): of a paragraph's normalized text, and of its text as written
330
+ const NORM_HITS = new WeakMap();
331
+ const RAW_HITS = new WeakMap();
332
+ /** a native search for `q` in norm(p), which is already kept (src/text.js searchCost) */
333
+ const normSearch = (p, q) => searchCost(norm(p).length, q, hitsOf(NORM_HITS, p, norm(p), q));
334
+
335
+ // The paragraphs of one reading by w14:paraId and by text hash, in document order, built once: every anchor resolved against the reading
336
+ // looks its paragraph up instead of scanning them all.
337
+ const INDEX = new WeakMap();
338
+ function indexOf(paragraphs) {
339
+ let ix = INDEX.get(paragraphs);
340
+ if (ix) return ix;
341
+ ix = { byId: new Map(), byHash: new Map(), byPath: new Map(), byTop: new Map(), noPath: [], tops: null };
342
+ for (const p of paragraphs) {
343
+ if (p.path) (ix.byTop.get(p.path[0]) || ix.byTop.set(p.path[0], []).get(p.path[0])).push(p);
344
+ else ix.noPath.push(p);
345
+ if (p.paraId) (ix.byId.get(p.paraId) || ix.byId.set(p.paraId, []).get(p.paraId)).push(p);
346
+ (ix.byHash.get(p.hash) || ix.byHash.set(p.hash, []).get(p.hash)).push(p);
347
+ if (p.path && !ix.byPath.has(p.path.join(","))) ix.byPath.set(p.path.join(","), p); // the first, as a scan finds it
348
+ }
349
+ INDEX.set(paragraphs, ix);
350
+ return ix;
351
+ }
352
+
353
+ // The paragraphs in the order nearest() ranks them, one level of equal distance from the body block `want` at a time (document order
354
+ // within a level), those without a body position last: the first level holding a match holds the match a scan of all of them picks.
355
+ function* byDistance(paragraphs, ix, want) {
356
+ if (!Number.isSafeInteger(want) || !ix.byTop.size) return yield paragraphs;
357
+ // the body blocks that hold paragraphs, outward from `want` (a body of many tables and few paragraphs has gaps: they are not walked)
358
+ const tops = (ix.tops ||= [...ix.byTop.keys()].sort((a, b) => a - b));
359
+ let hi = 0;
360
+ for (let lo = 0, top = tops.length; lo < top; ) {
361
+ const mid = (lo + top) >> 1;
362
+ if (tops[mid] < want) lo = mid + 1;
363
+ else top = mid;
364
+ hi = lo;
365
+ }
366
+ let lo = hi - 1;
367
+ while (lo >= 0 || hi < tops.length) {
368
+ const dl = lo >= 0 ? want - tops[lo] : Infinity;
369
+ const dh = hi < tops.length ? tops[hi] - want : Infinity;
370
+ const d = Math.min(dl, dh);
371
+ const a = dl === d ? ix.byTop.get(tops[lo--]) : null;
372
+ const b = dh === d ? ix.byTop.get(tops[hi++]) : null;
373
+ yield a && b ? a.concat(b) : a || b;
374
+ }
375
+ if (ix.noPath.length) yield ix.noPath;
376
+ }
377
+
378
+ const BIGRAMS = new WeakMap();
379
+ const bigramsOf = (p) => {
380
+ let v = BIGRAMS.get(p);
381
+ if (!v) BIGRAMS.set(p, (v = bigramCodes(p.text)));
382
+ return v;
383
+ };
384
+
290
385
  function locate(paragraphs, { path, paraId, quote, prefix, text, prevHash, nextHash, dupCount, allowPath = false }) {
291
- const q = normText(quote || "");
386
+ // The page's quote and paragraph text (each can be a whole paragraph, per comment) are paid for before they are read: the text is
387
+ // normalized up to four times below (its hash, its length, the slot's similarity, its bigrams). Past the budget a step that needs it
388
+ // is skipped, and at Send the bare position still answers.
389
+ const q = normPaid(quote) ?? "";
390
+ if (text && !(affords(4 * text.length) && spend(4 * weigh(text)))) text = "";
292
391
  const want = path ? path[0] : null;
293
392
  const dist = (p) => (want == null || !p.path ? 1e6 : Math.abs(p.path[0] - want));
294
- const nearest = (list) => list.slice().sort((a, b) => dist(a) - dist(b) || a.ordinal - b.ordinal)[0];
393
+ // the nearest of `list` (smaller distance, then smaller ordinal) in one pass, charged to the request's budget: null when it runs out
394
+ // (sorting a copy of 50,000 look-alikes for each of 500 anchors held the thread for 4 s, outside the budget)
395
+ const nearest = (list) => {
396
+ if (!spend(list.length * 3)) return null;
397
+ let best = null;
398
+ let bd = 0;
399
+ for (const p of list) {
400
+ const d = dist(p);
401
+ if (!best || d < bd || (d === bd && p.ordinal < best.ordinal)) (best = p), (bd = d);
402
+ }
403
+ return best;
404
+ };
295
405
  const nb = (p) => (prevHash && paragraphs[p.ordinal - 1]?.hash === prevHash ? 1 : 0) + (nextHash && paragraphs[p.ordinal + 1]?.hash === nextHash ? 1 : 0);
406
+ const ix = indexOf(paragraphs);
296
407
  // 1. w14:paraId
297
408
  if (paraId) {
298
- const hit = paragraphs.filter((p) => p.paraId === paraId.toUpperCase());
409
+ const hit = ix.byId.get(paraId.toUpperCase()) || [];
299
410
  if (hit.length === 1) return { p: hit[0], method: "id" };
300
411
  }
301
412
  // 2. identical text; among duplicates prefer the one whose neighbors are unchanged, then the nearest.
@@ -304,44 +415,110 @@ function locate(paragraphs, { path, paraId, quote, prefix, text, prevHash, nextH
304
415
  let ambiguousHash = null;
305
416
  if (text) {
306
417
  const h = textHash(text);
307
- const hits = paragraphs.filter((p) => p.hash === h);
418
+ const hits = ix.byHash.get(h) || [];
308
419
  const ambiguous = (dupCount != null ? dupCount > 1 : normText(text).length < 20) && (prevHash || nextHash);
309
- if (hits.length && (!ambiguous || hits.some((p) => nb(p) > 0))) return { p: nearest(hits.filter((p) => nb(p) === Math.max(...hits.map(nb)))), method: "hash" };
310
- if (hits.length) ambiguousHash = h;
420
+ // one pass over the hits for the best neighbor count, then the nearest of those (a document of 50,000 identical paragraphs and 500
421
+ // anchors held the thread for 4 s: the scans and the sort were outside the budget). Spent up front, at about what a hit costs here.
422
+ if (hits.length && spend(hits.length * 24)) {
423
+ let best = null;
424
+ let top = 0;
425
+ let bd = 0;
426
+ for (const p of hits) {
427
+ const n = nb(p);
428
+ const d = dist(p);
429
+ if (!best || n > top || (n === top && (d < bd || (d === bd && p.ordinal < best.ordinal)))) (best = p), (top = n), (bd = d);
430
+ }
431
+ if (!ambiguous || top > 0) return { p: best, method: "hash" };
432
+ ambiguousHash = h;
433
+ }
311
434
  }
312
435
  // 3. same slot: both neighbors unchanged and exactly one paragraph sits between them (edited in place)
313
436
  if (prevHash && nextHash) {
314
- const slots = paragraphs.filter((p) => nb(p) === 2);
315
- if (slots.length === 1 && (!text || similarity(slots[0].text, text) >= 0.3 || (q && normText(slots[0].text).includes(q)))) return { p: slots[0], method: "neighbors" };
437
+ const after = ix.byHash.get(prevHash) || [];
438
+ const slots = spend(after.length * 4) ? after.map((p) => paragraphs[p.ordinal + 1]).filter((p) => p && nb(p) === 2) : [];
439
+ // the slot's whole text is compared: within the request's budget (a slot of 8 million characters, 50 comments: 56 s before)
440
+ if (slots.length === 1 && (!text || (spend(2 * weightOf(slots[0]) + normCost(slots[0])) && (similarity(slots[0].text, text) >= 0.3 || (q && norm(slots[0]).includes(q)))))) return { p: slots[0], method: "neighbors" };
316
441
  }
317
- // 4. exact quote. Ambiguous when the quote also occurs elsewhere and the prefix does not disambiguate: do not guess here.
442
+ // steps 4 and 5 scan every paragraph: only within the budget of the anchors resolved together (src/text.js scanBudget), 16 units a
443
+ // paragraph, and the text each step searches character by character
444
+ // 4. exact quote. Ambiguous when the quote also occurs elsewhere and the prefix does not disambiguate: do not guess here. The
445
+ // paragraphs are read nearest first (byDistance): the answer is the nearest match, so the search stops at the first distance that
446
+ // has one (50 comments on a paragraph the agent changed, in 26,000 paragraphs, searched every paragraph 50 times).
447
+ let read = true;
318
448
  if (q) {
319
- const hits = paragraphs.filter((p) => normText(p.text).includes(q) && !(ambiguousHash && p.hash === ambiguousHash)); // look-alikes rejected in step 2 stay rejected
320
- if (hits.length) {
321
- const pre = normText(prefix || "");
322
- const better = pre ? hits.filter((p) => normText(p.text).includes(pre + (pre.endsWith(" ") ? "" : " ") + q) || normText(p.text).includes(pre + q)) : [];
323
- if (better.length) return { p: nearest(better), method: "quote" };
324
- if (hits.length === 1 || (!pre && !text)) return { p: nearest(hits), method: "quote" };
449
+ const pre = normPaid(prefix) ?? "";
450
+ const longer = pre ? [pre + (pre.endsWith(" ") ? "" : " ") + q, pre + q] : null;
451
+ const hits = [];
452
+ const better = [];
453
+ for (const level of byDistance(paragraphs, ix, want)) {
454
+ for (const p of level) {
455
+ if (!(read = spend(16 + normCost(p)) && spend(normSearch(p, q)))) break; // a native substring search (src/text.js searchCost)
456
+ if (!norm(p).includes(q) || (ambiguousHash && p.hash === ambiguousHash)) continue; // look-alikes rejected in step 2 stay rejected
457
+ hits.push(p);
458
+ // two native substring searches per hit (src/text.js searchCost, as in the loop above)
459
+ if (longer && spend(2 * normSearch(p, longer[1])) && (norm(p).includes(longer[0]) || norm(p).includes(longer[1]))) better.push(p);
460
+ }
461
+ if (!read) break;
462
+ const found = better.length ? better : !pre && !text ? hits : [];
463
+ if (found.length) {
464
+ const p = nearest(found);
465
+ if (p) return { p, method: "quote" };
466
+ }
325
467
  }
326
- }
468
+ if (read && hits.length === 1) {
469
+ const p = nearest(hits);
470
+ if (p) return { p, method: "quote" };
471
+ }
472
+ } else read = spend(paragraphs.length * 16);
327
473
  // 5. fuzzy: similarity to the remembered text (or the quote), neighbors count in favor
328
474
  const probe = text || q;
329
- if (probe) {
475
+ let spent = !read;
476
+ if (probe && read) {
330
477
  // The best candidate must clearly beat the runner-up: in templated text a deleted paragraph looks like its siblings,
331
478
  // and "moved or removed" is a better answer than attaching the comment to the wrong one.
332
- const scored = [];
479
+ // the two best candidates only, found in one pass: the answer needs the winner and the runner-up, never the order of the rest
480
+ let best = null;
481
+ let second = null;
482
+ let count = 0;
483
+ const P = bigramCodes(text || q); // the probe's bigrams once; each paragraph's are kept with it (bigramsOf)
484
+ const n = P.codes.length;
333
485
  for (const p of paragraphs) {
334
486
  if (ambiguousHash && p.hash === ambiguousHash) continue; // rejected above: a look-alike elsewhere is not this paragraph
335
- const sim = text ? similarity(p.text, text) : normText(p.text).length ? similarity(normText(p.text).slice(0, q.length + 40), q) : 0;
336
- if (sim >= 0.6) scored.push({ p, sim, score: sim + nb(p) * 0.1 - dist(p) * 1e-6 }); // distance only breaks ties
487
+ let sim = 0;
488
+ if (text) {
489
+ if (!BIGRAMS.has(p) && !spend(weightOf(p))) {
490
+ spent = true; // its bigrams are read (and sorted) once per reading: the first time costs its text
491
+ break;
492
+ }
493
+ const B = bigramsOf(p);
494
+ // 0.6 needs the shorter at least 3/7 of the longer (Dice is at most 2 min / (m + n)): skip the others without comparing
495
+ const m = B.codes.length;
496
+ if (P.short === null && B.short === null && (Math.min(m, n) * 7 < Math.max(m, n) * 3)) continue;
497
+ if (!spend(m + n)) {
498
+ spent = true; // the budget is spent: "moved or removed" (src/text.js scanBudget), or the bare position at Send
499
+ break;
500
+ }
501
+ sim = diceOf(B, P);
502
+ } else if (norm(p).length) {
503
+ if (!spend(normCost(p) + q.length + 40 + n)) {
504
+ spent = true;
505
+ break;
506
+ }
507
+ sim = diceOf(bigramCodes(norm(p).slice(0, q.length + 40)), P);
508
+ }
509
+ if (sim >= 0.6) {
510
+ const c = { p, sim, score: sim + nb(p) * 0.1 - dist(p) * 1e-6 }; // distance only breaks ties
511
+ count++;
512
+ if (!best || c.score > best.score) (second = best), (best = c);
513
+ else if (!second || c.score > second.score) second = c;
514
+ }
337
515
  }
338
- scored.sort((a, b) => b.score - a.score);
339
- if (scored.length && (scored.length === 1 || scored[0].score - scored[1].score >= 0.04)) return { p: scored[0].p, method: "fuzzy", score: Math.round(scored[0].sim * 100) / 100 };
516
+ if (!spent && best && (count === 1 || best.score - second.score >= 0.04)) return { p: best.p, method: "fuzzy", score: Math.round(best.sim * 100) / 100 };
340
517
  }
341
518
  // At Send time the browser's own index is as fresh as the bytes, so the bare position is acceptable.
342
519
  // After a reload it never is (an insert above shifts it): resolveDurable does not pass allowPath.
343
520
  if (allowPath && path) {
344
- const hit = paragraphs.find((p) => p.path && p.path.length === path.length && p.path.every((v, i) => v === path[i]));
521
+ const hit = ix.byPath.get(path.join(",")); // by lookup, not a scan: the bare position must still answer once the budget is spent
345
522
  if (hit) return { p: hit, method: "path" };
346
523
  }
347
524
  return null;
@@ -358,21 +535,152 @@ export function durableFromEngine(paragraphs, loc, ctx = {}) {
358
535
  return durableFor(paragraphs, hit.p, loc.blockPath ? loc.blockPath[0] : null, ctx, loc);
359
536
  }
360
537
 
361
- /** Offsets of every non-overlapping occurrence of `q` in `text`. */
362
- function occurrences(text, q) {
538
+ /**
539
+ * Offsets of the non-overlapping occurrences of `q` in `text` (the first `max` of them), or null when the request's budget runs out: the
540
+ * search is counted as a native substring search (src/text.js searchCost) and each occurrence found at OCCURRENCE.
541
+ */
542
+ // the complete list of a quote's occurrences in a paragraph, kept per paragraph of a reading for the last few quotes (rule: what is
543
+ // derived from the same paragraph is computed once per reading): 50 comments on one selection of a repeated text listed the same
544
+ // 395,000 occurrences 50 times. Ceiling: OCC_KEEP quotes per paragraph, the oldest dropped first.
545
+ const OCC = new WeakMap();
546
+ const OCC_KEEP = 16;
547
+ function occurrencesIn(p, q) {
548
+ let m = OCC.get(p);
549
+ const hit = m?.get(q);
550
+ if (hit) return hit;
551
+ const occ = occurrences(p, q);
552
+ if (occ) {
553
+ if (!m) OCC.set(p, (m = new Map()));
554
+ if (m.size >= OCC_KEEP) m.delete(m.keys().next().value);
555
+ m.set(q, occ);
556
+ }
557
+ return occ;
558
+ }
559
+ function occurrences(p, q, max = Infinity) {
560
+ const { text } = p;
561
+ if (!spend(searchCost(text.length, q, hitsOf(RAW_HITS, p, text, q)))) return null;
363
562
  const out = [];
364
- for (let i = text.indexOf(q); i >= 0; i = text.indexOf(q, i + q.length)) out.push(i);
563
+ for (let i = text.indexOf(q); i >= 0 && out.length < max; i = text.indexOf(q, i + q.length)) {
564
+ if (!spend(OCCURRENCE)) return null;
565
+ out.push(i);
566
+ }
365
567
  return out;
366
568
  }
367
569
 
570
+ /**
571
+ * normText(text).slice(0, n), reading only as much of the text as that needs (windows cut as endsWithNorm cuts them), each character read
572
+ * spent from the budget; null past it. A comment's paragraph text is kept to 2,000 characters, and normalizing a whole paragraph of
573
+ * 8 million characters for each of 50 comments held the thread for seconds.
574
+ */
575
+ function normHead(text, n) {
576
+ for (let size = 2 * n + 8; ; size *= 2) {
577
+ const e = cutNear(text, Math.min(text.length, size), 1);
578
+ if (e === null || !spend(weigh(text.slice(0, e)))) return null;
579
+ const head = normText(text.slice(0, e));
580
+ if (e === text.length || head.length > n) return head.slice(0, n);
581
+ }
582
+ }
583
+
584
+ // normText(text.slice(0, at)).endsWith(prefix) and normText(text.slice(from)).startsWith(suffix), reading only as much of the text as
585
+ // the prefix or suffix needs. A window is cut only where nothing on one side changes how the other side normalizes (`cuttable`), so the
586
+ // window normalizes to exactly the end (or the start) of what the whole text normalizes to, give or take one space where it was cut.
587
+ // Once the window holds at least as many normalized characters as the prefix, it answers for the whole text; else it doubles.
588
+ // Every character read (the walk to a cut and the window normalized) is spent from the budget of the request (src/text.js): null when it
589
+ // runs out. An outside review: CJK text has no character below U+0300, every window grew to the whole paragraph, 3.65 s for one anchor.
590
+ const MARK = /\p{M}/u;
591
+ /**
592
+ * May a window start (or end) right before text[i]? Yes below U+0300 (no such character joins the one before it under NFC); never inside a
593
+ * surrogate pair or before a combining mark; otherwise when the character normalizes the same after the three before it as on its own
594
+ * (the longest chain of starters that compose, Hangul L V T and its like, is three: test/parse-limits.test.js checks every code point).
595
+ * Exported with the two window readers for that test.
596
+ */
597
+ export function cuttable(text, i) {
598
+ if (i <= 0 || i >= text.length) return true;
599
+ const c = text.charCodeAt(i);
600
+ if (c < 0x300) return true;
601
+ if (c >= 0xdc00 && c <= 0xdfff) return false;
602
+ const ch = String.fromCodePoint(text.codePointAt(i));
603
+ if (MARK.test(ch)) return false;
604
+ const before = text.slice(Math.max(0, i - 3), i);
605
+ return (before + ch).normalize("NFC") === before.normalize("NFC") + ch.normalize("NFC");
606
+ }
607
+ /** The nearest place at or before (dir -1) or after (dir 1) i where a window may be cut, each character walked spent; null past the budget. */
608
+ function cutNear(text, i, dir) {
609
+ let n = 0;
610
+ while (!cuttable(text, i)) {
611
+ i += dir;
612
+ if (++n % 1024 === 0 && !spend(1024)) return null;
613
+ }
614
+ return spend(n % 1024) ? i : null;
615
+ }
616
+ export function endsWithNorm(text, at, prefix) {
617
+ for (let size = 2 * prefix.length + 8; ; size *= 2) {
618
+ const w = cutNear(text, Math.max(0, at - size), -1);
619
+ if (w === null || !spend(weigh(text.slice(w, at)))) return null;
620
+ const n = normText(text.slice(w, at));
621
+ if (w === 0 || n.length >= prefix.length) return n.endsWith(prefix);
622
+ }
623
+ }
624
+ export function startsWithNorm(text, from, suffix) {
625
+ for (let size = 2 * suffix.length + 8; ; size *= 2) {
626
+ const e = cutNear(text, Math.min(text.length, from + size), 1);
627
+ if (e === null || !spend(weigh(text.slice(from, e)))) return null;
628
+ const n = normText(text.slice(from, e));
629
+ if (e === text.length || n.length >= suffix.length) return n.startsWith(suffix);
630
+ }
631
+ }
632
+
368
633
  /**
369
634
  * Which occurrence of a repeated quote a selection or a stored anchor means. Prefix and suffix are the normalized text around it;
370
- * `nth` is the occurrence index remembered at capture. Returns the offset, or -1 when the text does not occur.
635
+ * `nth` is the occurrence index remembered at capture. Returns {at: the offset, k: which occurrence it is}, or {at: -1, k: null} when the
636
+ * text does not occur or the budget ran out.
371
637
  */
372
- function pickOccurrence(text, q, { prefix = "", suffix = "", nth = null } = {}) {
373
- const occ = occurrences(text, q);
374
- if (!occ.length) return -1;
375
- const guarded = occ.filter((at) => (!prefix || normText(text.slice(0, at)).endsWith(prefix)) && (!suffix || normText(text.slice(at + q.length)).startsWith(suffix)));
638
+ function pickOccurrence(p, q, opts = {}) {
639
+ const text = p.text;
640
+ // the remembered occurrence first, listing only the occurrences up to it: when its surroundings still fit, it is the answer every rule
641
+ // of pickFrom gives, and the others need not be listed or read (an outside review: 300,000 "a" with nth 150,000 lost its offsets
642
+ // under the budget, though the remembered one still fit)
643
+ const { nth = null } = opts;
644
+ // only an index of the occurrences is one: a selection that starts between two of them ("aa" at offset 1 of "aaaa", which the
645
+ // non-overlapping search does not list) was stored with nth -1, and occ[-1] (undefined) made the window readers loop forever on the
646
+ // server's thread, the budget never asked (a window of no characters costs nothing)
647
+ const remembered = Number.isInteger(nth) && nth >= 0 ? nth : null;
648
+ if (remembered !== null && (opts.prefix || opts.suffix)) {
649
+ const head = occurrences(p, q, remembered + 1);
650
+ if (head === null) return { at: -1, k: null };
651
+ if (head.length === remembered + 1) {
652
+ const ok = fitsAt(text, q, head[remembered], opts);
653
+ if (ok === null) return { at: -1, k: null };
654
+ if (ok) return { at: head[remembered], k: remembered };
655
+ }
656
+ }
657
+ const occ = occurrencesIn(p, q);
658
+ const at = occ?.length ? pickFrom(text, q, occ, { ...opts, nth: remembered }) : -1;
659
+ return at < 0 ? { at: -1, k: null } : { at, k: occ.indexOf(at) };
660
+ }
661
+ /** does the text around the occurrence at `at` read as prefix and suffix (normalized, as an anchor keeps them)? null: the budget ran out */
662
+ function fitsAt(text, q, at, { prefix = "", suffix = "" }) {
663
+ prefix = String(prefix || "").slice(-CTX); // what an anchor keeps (durableFor); a longer one from elsewhere is read as its last CTX
664
+ suffix = String(suffix || "").slice(0, CTX);
665
+ const pre = !prefix || endsWithNorm(text, at, prefix);
666
+ if (pre === null) return null;
667
+ const suf = pre && (!suffix || startsWithNorm(text, at + q.length, suffix));
668
+ return suf === null ? null : !!(pre && suf);
669
+ }
670
+ function pickFrom(text, q, occ, { prefix = "", suffix = "", nth = null }) {
671
+ if (!(Number.isInteger(nth) && nth >= 0 && nth < occ.length)) nth = null;
672
+ prefix = String(prefix || "").slice(-CTX);
673
+ suffix = String(suffix || "").slice(0, CTX);
674
+ const fits = (at) => fitsAt(text, q, at, { prefix, suffix });
675
+ // each occurrence reads about CTX characters on each side, out of the budget of the request (src/text.js): a paragraph of 100,000
676
+ // "a" read the whole text before every one of them and took 10 seconds in an outside review. Past the budget, no offset.
677
+ if ((prefix || suffix) && occ.length > 1 && !spend(occ.length * 2 * (prefix.length + suffix.length + 16))) return -1;
678
+ const guarded = [];
679
+ for (const at of occ) {
680
+ const ok = fits(at);
681
+ if (ok === null) return -1;
682
+ if (ok) guarded.push(at);
683
+ }
376
684
  if (guarded.length === 1) return guarded[0];
377
685
  if (nth != null && nth < occ.length && guarded.includes(occ[nth])) return occ[nth];
378
686
  if (guarded.length) return guarded[0];
@@ -380,20 +688,22 @@ function pickOccurrence(text, q, { prefix = "", suffix = "", nth = null } = {})
380
688
  }
381
689
 
382
690
  function durableFor(paragraphs, p, engineTop, ctx, loc = {}) {
383
- const norm = normText(p.text);
384
- const q = normText(ctx.quote || "");
385
- const prefix = normText(ctx.prefix || "").slice(-CTX);
386
- const suffix = normText(ctx.suffix || "").slice(0, CTX);
691
+ // the page's quote, prefix and suffix, paid for before they are normalized; past the budget the anchor is the paragraph alone
692
+ const q = normPaid(ctx.quote) ?? "";
693
+ const prefix = (normPaid(ctx.prefix) ?? "").slice(-CTX);
694
+ const suffix = (normPaid(ctx.suffix) ?? "").slice(0, CTX);
387
695
  let start = null;
388
696
  let end = null;
389
697
  let nth = null;
390
698
  if (q) {
391
699
  // the engine's own offsets when they point at the quote; else the occurrence whose surroundings match prefix and suffix
392
- const at = Number.isInteger(loc.start) && Number.isInteger(loc.end) && p.text.slice(loc.start, loc.end) === ctx.quote ? loc.start : pickOccurrence(p.text, ctx.quote, { prefix, suffix });
393
- if (at >= 0) {
394
- start = at;
395
- end = at + ctx.quote.length;
396
- nth = occurrences(p.text, ctx.quote).indexOf(at);
700
+ const own = Number.isInteger(loc.start) && Number.isInteger(loc.end) && p.text.slice(loc.start, loc.end) === ctx.quote;
701
+ const k = own ? occurrencesIn(p, ctx.quote)?.indexOf(loc.start) ?? -1 : -1;
702
+ const picked = own ? { at: loc.start, k: k >= 0 ? k : null } : pickOccurrence(p, ctx.quote, { prefix, suffix });
703
+ if (picked.at >= 0) {
704
+ start = picked.at;
705
+ end = picked.at + ctx.quote.length;
706
+ nth = picked.k;
397
707
  }
398
708
  }
399
709
  return {
@@ -407,8 +717,8 @@ function durableFor(paragraphs, p, engineTop, ctx, loc = {}) {
407
717
  textHash: p.hash,
408
718
  prevHash: paragraphs[p.ordinal - 1]?.hash || null,
409
719
  nextHash: paragraphs[p.ordinal + 1]?.hash || null,
410
- dupCount: paragraphs.filter((x) => x.hash === p.hash).length,
411
- paraText: norm.slice(0, PARA_TEXT_KEEP),
720
+ dupCount: indexOf(paragraphs).byHash.get(p.hash)?.length ?? 0,
721
+ paraText: normHead(p.text, PARA_TEXT_KEEP) ?? "",
412
722
  quote: q,
413
723
  prefix,
414
724
  suffix,
@@ -429,10 +739,10 @@ export function resolveDurable(paragraphs, d) {
429
739
  const q = d.quote || "";
430
740
  let start = null;
431
741
  let end = null;
432
- const at = q ? pickOccurrence(p.text, q, { prefix: d.prefix, suffix: d.suffix, nth: d.nth }) : -1;
742
+ const { at, k } = q ? pickOccurrence(p, q, { prefix: d.prefix, suffix: d.suffix, nth: d.nth }) : { at: -1, k: null };
433
743
  if (at >= 0) {
434
744
  start = at;
435
745
  end = at + q.length;
436
746
  }
437
- return { method: hit.method, score: hit.score, ordinal: p.ordinal, path: p.path, tableNo: p.tableNo ?? null, paraId: p.paraId, textHash: p.hash, protected: !!p.protected, ...(p.protected ? { protectInfo: protectInfo(p) } : {}), dupCount: paragraphs.filter((x) => x.hash === p.hash).length, nth: at >= 0 ? occurrences(p.text, q).indexOf(at) : null, prevHash: paragraphs[p.ordinal - 1]?.hash || null, nextHash: paragraphs[p.ordinal + 1]?.hash || null, text: normText(p.text).slice(0, PARA_TEXT_KEEP), start, end, engineDelta: d.engineDelta || 0 };
747
+ return { method: hit.method, score: hit.score, ordinal: p.ordinal, path: p.path, tableNo: p.tableNo ?? null, paraId: p.paraId, textHash: p.hash, protected: !!p.protected, ...(p.protected ? { protectInfo: protectInfo(p) } : {}), dupCount: indexOf(paragraphs).byHash.get(p.hash)?.length ?? 0, nth: at >= 0 ? k : null, prevHash: paragraphs[p.ordinal - 1]?.hash || null, nextHash: paragraphs[p.ordinal + 1]?.hash || null, text: normHead(p.text, PARA_TEXT_KEEP) ?? "", start, end, engineDelta: d.engineDelta || 0 };
438
748
  }
@@ -0,0 +1,19 @@
1
+ // The text hash of a pending HTML text edit (PLAN-html R9): one function for the page (web/src/html-view.js) and the server
2
+ // (src/html-write.js), so both read the same 14 hex digits. Pure, no imports. Fixed vectors: test/html-view.test.js.
3
+
4
+ /**
5
+ * cyrb53 over the exact text (UTF-16 code units, nothing normalized), 14 hex digits. Not a security hash: it tells "the text the edit
6
+ * was made on" from another text, and the version's sha256 travels with it.
7
+ */
8
+ export function editHash(s) {
9
+ let h1 = 0xdeadbeef;
10
+ let h2 = 0x41c6ce57;
11
+ for (let i = 0; i < s.length; i++) {
12
+ const c = s.charCodeAt(i);
13
+ h1 = Math.imul(h1 ^ c, 2654435761);
14
+ h2 = Math.imul(h2 ^ c, 1597334677);
15
+ }
16
+ h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909);
17
+ h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909);
18
+ return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(16).padStart(14, "0");
19
+ }
package/src/fsutil.js CHANGED
@@ -19,7 +19,7 @@ export function checkRegularOwned(file) {
19
19
  /** Reads a document, refusing by size first (an oversized file is never pulled into memory). */
20
20
  export function readDoc(file, max = MAX_DOC_BYTES) {
21
21
  const st = fs.statSync(file);
22
- if (st.size > max) throw Object.assign(new Error(`${path.basename(file)} is larger than ${Math.round(max / 1024 / 1024)} MB`), { code: "TOO_LARGE" });
22
+ if (st.size > max) throw Object.assign(new Error(`${path.basename(file)} is larger than ${Math.round(max / 1024 / 1024)} MB, more than Onionskin can read. Make it smaller (pictures and videos are often most of a file) or split it, and open that`), { code: "TOO_LARGE" });
23
23
  return fs.readFileSync(file);
24
24
  }
25
25
 
@@ -1,16 +1,16 @@
1
1
  import fs from "node:fs";
2
- import { parentPort, workerData } from "node:worker_threads";
2
+ import "./html-parser.js"; // an HTML file's versions are compared here too: the parser lives in worker threads only
3
3
  import { compareBytes } from "./history.js";
4
4
 
5
- // One message back: a result per job, in order. A version that cannot be read or parsed fails its own job only.
6
- const bytes = new Map(); // blob path -> Buffer, so a version shared by two pairs is read (and parsed) once
7
- const read = (f) => bytes.get(f) || (bytes.set(f, fs.readFileSync(f)), bytes.get(f));
8
- parentPort.postMessage(
9
- workerData.jobs.map((j) => {
5
+ // Run by src/worker-host.js. One message back: a result per job, in order. A version that cannot be read or parsed fails its own job only.
6
+ export function work(data) {
7
+ const bytes = new Map(); // blob path -> Buffer, so a version shared by two pairs is read (and parsed) once
8
+ const read = (f) => bytes.get(f) || (bytes.set(f, fs.readFileSync(f)), bytes.get(f));
9
+ return data.jobs.map((j) => {
10
10
  try {
11
11
  return { ok: true, result: compareBytes(j.format, read(j.before), read(j.after), { places: j.places }) };
12
12
  } catch (err) {
13
13
  return { ok: false, error: String(err?.message || err) };
14
14
  }
15
- })
16
- );
15
+ });
16
+ }