onionskin-cli 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +100 -0
- package/INSTALL.md +5 -1
- package/NOTICE +34 -0
- package/README.md +101 -33
- package/docs/contracts/batch.schema.json +27 -4
- package/docs/contracts/ops.schema.json +117 -5
- package/package.json +4 -2
- package/skills/onionskin/SKILL.md +9 -6
- package/skills/onionskin/reference/batch.md +5 -3
- package/skills/onionskin/reference/formats.md +5 -1
- package/skills/onionskin/reference/ops.md +47 -3
- package/skills/onionskin/reference/troubleshooting.md +31 -0
- package/src/agent-detect.js +28 -2
- package/src/agent-prompt.js +12 -1
- package/src/agent-runner.js +16 -5
- package/src/anchor-work.js +154 -0
- package/src/anchor-worker.js +38 -0
- package/src/anchors.js +61 -11
- package/src/apply.js +119 -47
- package/src/batch.js +32 -8
- package/src/check-worker.js +8 -5
- package/src/cli.js +91 -25
- package/src/constants.js +34 -1
- package/src/diff.js +3 -1
- package/src/docx.js +368 -58
- package/src/edit-hash.js +19 -0
- package/src/fsutil.js +1 -1
- package/src/history-worker.js +8 -8
- package/src/history.js +135 -31
- package/src/html-active.js +384 -0
- package/src/html-envelope.js +51 -0
- package/src/html-parser.js +8 -0
- package/src/html-render-worker.js +25 -0
- package/src/html-render.js +880 -0
- package/src/html-utf8.js +65 -0
- package/src/html-write-worker.js +14 -0
- package/src/html-write.js +380 -0
- package/src/html.js +1257 -0
- package/src/local.js +68 -4
- package/src/merge-package.js +1110 -0
- package/src/merge-pptx.js +98 -38
- package/src/merge-proof.js +1019 -0
- package/src/merge-worker.js +60 -0
- package/src/merge-xlsx.js +84 -29
- package/src/merge.js +286 -83
- package/src/nfc.js +46 -0
- package/src/notes.js +122 -0
- package/src/offload.js +400 -1
- package/src/open-worker.js +27 -0
- package/src/ops/common.js +2 -1
- package/src/ops/docx.js +58 -29
- package/src/ops/html.js +394 -0
- package/src/ops/pptx.js +74 -21
- package/src/ops/run.js +23 -3
- package/src/ops/runs.js +215 -13
- package/src/ops/schema.js +8 -1
- package/src/ops/xlsx.js +12 -7
- package/src/ops-worker.js +17 -0
- package/src/package.js +26 -2
- package/src/person.js +2 -1
- package/src/pptx.js +85 -21
- package/src/quick-plan.js +13 -6
- package/src/redline-worker.js +10 -14
- package/src/reply.js +46 -19
- package/src/retain.js +223 -0
- package/src/review.js +32 -18
- package/src/rewrite-check.js +2 -2
- package/src/sanitize.js +16 -3
- package/src/server.js +80 -36
- package/src/session.js +1138 -208
- package/src/stop-gate.js +34 -0
- package/src/suggest-reply.js +42 -7
- package/src/suggest.js +14 -29
- package/src/text.js +161 -11
- package/src/update.js +288 -0
- package/src/voice-guide.js +4 -3
- package/src/voice-learn.js +4 -3
- package/src/voice-sample-worker.js +7 -6
- package/src/voice-worker.js +4 -6
- package/src/worker-host.js +23 -0
- package/src/xlsx.js +355 -94
- package/src/xml-tree.js +6 -3
- package/src/xml.js +82 -34
- package/src/zip-write.js +2 -1
- package/src/zip.js +9 -5
- package/vendor/parse5/LICENSE.txt +32 -0
- package/vendor/parse5/parse5.mjs +7822 -0
- package/vendor/parse5.lock.json +23 -0
- package/web/dist/assets/index-DkNx0LSH.css +1 -0
- package/web/dist/assets/{index-DS-WST1E.js → index-DytKkMdi.js} +112 -108
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/docx.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { pathToFileURL } from "node:url";
|
|
2
2
|
import { engineNodePath } from "./ops/engine.js";
|
|
3
|
-
import { walkXml } from "./xml.js";
|
|
4
|
-
import { normText, similarity, textHash } from "./text.js";
|
|
3
|
+
import { readBudget, tooMany, walkXml } from "./xml.js";
|
|
4
|
+
import { OCCURRENCE, affords, bigramCodes, diceOf, hitsOf, normPaid, normText, searchCost, similarity, spend, textHash, weigh, weightOf } from "./text.js";
|
|
5
5
|
import { openZip } from "./zip.js";
|
|
6
6
|
import { summarizeRuns } from "./fmt.js";
|
|
7
7
|
|
|
@@ -43,6 +43,16 @@ function runSet(name, a) {
|
|
|
43
43
|
if (name === "w:vertAlign") return only(a, "w:val") && (v === "superscript" || v === "subscript") ? ["va", v.slice(0, 3)] : null;
|
|
44
44
|
return null;
|
|
45
45
|
}
|
|
46
|
+
// What one reading of a Word document may hold. The server reads a document on its own thread at Send, at each delivery to an agent, for a
|
|
47
|
+
// queued comment and at the end of a run, so these bound how long one such reading can keep it from answering anything else (the outside
|
|
48
|
+
// review: a part of 1 million runs held it 4.3 s, a run of 3 million attributes 2.4 s). A real report of about 1,000 pages has about 16,000
|
|
49
|
+
// paragraphs, 240,000 elements, 350,000 attributes and 3 million characters (measured: read in about 240 ms here).
|
|
50
|
+
export const MAX_PARAGRAPHS = 50_000;
|
|
51
|
+
export const MAX_ELEMENTS = 400_000;
|
|
52
|
+
export const MAX_ATTRIBUTES = 800_000;
|
|
53
|
+
export const MAX_TEXT = 8_000_000;
|
|
54
|
+
/** One reading's budget (src/xml.js readBudget), shared by the parts it walks. */
|
|
55
|
+
export const docxBudget = () => readBudget({ elements: MAX_ELEMENTS, attrs: MAX_ATTRIBUTES, text: MAX_TEXT });
|
|
46
56
|
const PPR_OWN = new Set(["w:pStyle", "w:jc", "w:numPr", "w:rPr", "w:sectPr"]); // read on their own, or not paragraph formatting the user sets
|
|
47
57
|
const RANGE = /^(.+)(Start|End)$/;
|
|
48
58
|
|
|
@@ -55,7 +65,7 @@ const RANGE = /^(.+)(Start|End)$/;
|
|
|
55
65
|
* (null when they do not add up to it); `list` {numId, ilvl} from its own w:numPr, else null; `ppr`, its other paragraph properties as
|
|
56
66
|
* written; `sect`, it ends a section; `loose`, a bookmark, comment or other range starts or ends inside it without its other end.
|
|
57
67
|
*/
|
|
58
|
-
export function parseDocxParagraphs(xml, { detail = false } = {}) {
|
|
68
|
+
export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget() } = {}) {
|
|
59
69
|
const paragraphs = [];
|
|
60
70
|
const stack = []; // {name, ...}
|
|
61
71
|
let bodyIndex = -1;
|
|
@@ -107,6 +117,7 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
|
|
|
107
117
|
path = [info.tableIndex, info.row, info.col, cellK++];
|
|
108
118
|
tableNo = tableCount;
|
|
109
119
|
}
|
|
120
|
+
if (paragraphs.length >= MAX_PARAGRAPHS) throw tooMany(budget, MAX_PARAGRAPHS, "paragraphs");
|
|
110
121
|
cur = { ordinal: paragraphs.length, path, tableNo, paraId: /^[0-9A-Fa-f]{8}$/.test(attrs["w14:paraId"] || "") ? attrs["w14:paraId"].toUpperCase() : null, text: "", instr: "", protected: false, style: null, align: "", runs: [] };
|
|
111
122
|
if (detail) (Object.assign(cur, { spans: [], list: null, ppr: [], sect: false, loose: false }), (marks = new Set()), (pprTop = null));
|
|
112
123
|
paragraphs.push(cur);
|
|
@@ -175,7 +186,7 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
|
|
|
175
186
|
}
|
|
176
187
|
if (detail && name === pprTop && stack[stack.length - 1] === "w:pPr") pprTop = null;
|
|
177
188
|
},
|
|
178
|
-
});
|
|
189
|
+
}, budget);
|
|
179
190
|
for (const p of paragraphs) p.hash = textHash(p.text);
|
|
180
191
|
const reasons = protectReasons(xml);
|
|
181
192
|
// the engine lists paragraphs the same way; if the two ever disagree on the count, nothing can be trusted as editable
|
|
@@ -204,10 +215,28 @@ export function protectInfo(p) {
|
|
|
204
215
|
}
|
|
205
216
|
|
|
206
217
|
const HEADING = /^(heading\s?\d|title|subtitle)$/i;
|
|
207
|
-
|
|
218
|
+
// the nearest heading at or before each ordinal, found in one pass per reading: a scan up from every resolved comment was a pass over
|
|
219
|
+
// the paragraphs per comment, and the heading's text was normalized whole again each time, outside the budget
|
|
220
|
+
const HEADS = new WeakMap();
|
|
221
|
+
/** The text of the nearest heading above a paragraph (by paragraph style), or "" when there is none (or the budget cannot pay for it). */
|
|
208
222
|
export function headingBefore(paragraphs, ordinal) {
|
|
209
|
-
|
|
210
|
-
|
|
223
|
+
if (!Number.isInteger(ordinal) || ordinal <= 0) return "";
|
|
224
|
+
let at = HEADS.get(paragraphs);
|
|
225
|
+
if (!at) {
|
|
226
|
+
at = new Int32Array(paragraphs.length + 1);
|
|
227
|
+
let last = -1;
|
|
228
|
+
for (let i = 0; i < paragraphs.length; i++) {
|
|
229
|
+
at[i] = last; // the heading strictly before paragraph i
|
|
230
|
+
const p = paragraphs[i];
|
|
231
|
+
if (p.style && HEADING.test(p.style) && p.text.trim()) last = i;
|
|
232
|
+
}
|
|
233
|
+
at[paragraphs.length] = last;
|
|
234
|
+
HEADS.set(paragraphs, at);
|
|
235
|
+
}
|
|
236
|
+
const h = at[Math.min(ordinal, paragraphs.length)];
|
|
237
|
+
if (h < 0) return "";
|
|
238
|
+
const p = paragraphs[h];
|
|
239
|
+
return spend(normCost(p)) ? norm(p).slice(0, 80) : "";
|
|
211
240
|
}
|
|
212
241
|
|
|
213
242
|
/** `detail` (the merge only): formatting per run and the paragraph's list, see parseDocxParagraphs; `list.kind` is "bullet", "number" or "other". */
|
|
@@ -215,11 +244,12 @@ export function readDocxParagraphs(bytes, { detail = false } = {}) {
|
|
|
215
244
|
const zip = openZip(bytes);
|
|
216
245
|
const xml = zip.readText("word/document.xml");
|
|
217
246
|
if (xml == null) throw new Error("word/document.xml missing");
|
|
218
|
-
const
|
|
219
|
-
const
|
|
247
|
+
const budget = docxBudget(); // the main part, the styles and the numbering of one reading share it
|
|
248
|
+
const paragraphs = parseDocxParagraphs(xml, { detail, budget });
|
|
249
|
+
const names = styleNames(zip.readText("word/styles.xml"), budget);
|
|
220
250
|
if (names.size) for (const p of paragraphs) if (p.style && names.has(p.style)) p.styleName = names.get(p.style);
|
|
221
251
|
if (detail && paragraphs.some((p) => p.list)) {
|
|
222
|
-
const kinds = listKinds(zip.readText("word/numbering.xml"));
|
|
252
|
+
const kinds = listKinds(zip.readText("word/numbering.xml"), budget);
|
|
223
253
|
for (const p of paragraphs) if (p.list) p.list = p.list.numId && p.list.numId !== "0" ? { ...p.list, kind: kinds.get(p.list.numId) || "other" } : null; // numId 0 switches a style's list off
|
|
224
254
|
}
|
|
225
255
|
return paragraphs;
|
|
@@ -229,7 +259,7 @@ export function readDocxParagraphs(bytes, { detail = false } = {}) {
|
|
|
229
259
|
* w:numId -> "bullet" | "number" | "other", by the first level of its definition: the two kinds the editor's list buttons make and
|
|
230
260
|
* the engine can set again. A list with level overrides, or one this cannot read, is "other".
|
|
231
261
|
*/
|
|
232
|
-
function listKinds(xml) {
|
|
262
|
+
function listKinds(xml, budget) {
|
|
233
263
|
const out = new Map();
|
|
234
264
|
if (xml == null) return out;
|
|
235
265
|
const fmt = new Map(); // abstractNumId -> numFmt of level 0
|
|
@@ -258,13 +288,13 @@ function listKinds(xml) {
|
|
|
258
288
|
num = null;
|
|
259
289
|
}
|
|
260
290
|
},
|
|
261
|
-
});
|
|
291
|
+
}, budget);
|
|
262
292
|
} catch {} // an unreadable numbering part: every list is "other", which is reported, never set
|
|
263
293
|
return out;
|
|
264
294
|
}
|
|
265
295
|
|
|
266
296
|
/** w:styleId -> the style's display name ("heading 2" is shown as "Heading 2"); empty when styles.xml is missing or unreadable. */
|
|
267
|
-
function styleNames(xml) {
|
|
297
|
+
function styleNames(xml, budget) {
|
|
268
298
|
const out = new Map();
|
|
269
299
|
if (xml == null) return out;
|
|
270
300
|
let id = null;
|
|
@@ -277,7 +307,7 @@ function styleNames(xml) {
|
|
|
277
307
|
onClose(name) {
|
|
278
308
|
if (name === "w:style") id = null;
|
|
279
309
|
},
|
|
280
|
-
});
|
|
310
|
+
}, budget);
|
|
281
311
|
} catch {} // a damaged styles part only costs the readable style names
|
|
282
312
|
return out;
|
|
283
313
|
}
|
|
@@ -287,15 +317,96 @@ const PARA_TEXT_KEEP = 2000;
|
|
|
287
317
|
|
|
288
318
|
// ---- engine location -> durable anchor ----------------------------------------------------------------------
|
|
289
319
|
|
|
320
|
+
// A paragraph's normalized text, kept per paragraph object: every comment resolved against the same reading reuses it.
|
|
321
|
+
const NORM = new WeakMap();
|
|
322
|
+
const norm = (p) => {
|
|
323
|
+
let v = NORM.get(p);
|
|
324
|
+
if (v === undefined) NORM.set(p, (v = normText(p.text)));
|
|
325
|
+
return v;
|
|
326
|
+
};
|
|
327
|
+
/** what norm(p) is about to read: the paragraph's text the first time (src/text.js weigh), nothing once it is kept (spent from the request's budget) */
|
|
328
|
+
const normCost = (p) => (NORM.has(p) ? 0 : weightOf(p));
|
|
329
|
+
// the byte tables a search is priced by (src/text.js hitsOf): of a paragraph's normalized text, and of its text as written
|
|
330
|
+
const NORM_HITS = new WeakMap();
|
|
331
|
+
const RAW_HITS = new WeakMap();
|
|
332
|
+
/** a native search for `q` in norm(p), which is already kept (src/text.js searchCost) */
|
|
333
|
+
const normSearch = (p, q) => searchCost(norm(p).length, q, hitsOf(NORM_HITS, p, norm(p), q));
|
|
334
|
+
|
|
335
|
+
// The paragraphs of one reading by w14:paraId and by text hash, in document order, built once: every anchor resolved against the reading
|
|
336
|
+
// looks its paragraph up instead of scanning them all.
|
|
337
|
+
const INDEX = new WeakMap();
|
|
338
|
+
function indexOf(paragraphs) {
|
|
339
|
+
let ix = INDEX.get(paragraphs);
|
|
340
|
+
if (ix) return ix;
|
|
341
|
+
ix = { byId: new Map(), byHash: new Map(), byPath: new Map(), byTop: new Map(), noPath: [], tops: null };
|
|
342
|
+
for (const p of paragraphs) {
|
|
343
|
+
if (p.path) (ix.byTop.get(p.path[0]) || ix.byTop.set(p.path[0], []).get(p.path[0])).push(p);
|
|
344
|
+
else ix.noPath.push(p);
|
|
345
|
+
if (p.paraId) (ix.byId.get(p.paraId) || ix.byId.set(p.paraId, []).get(p.paraId)).push(p);
|
|
346
|
+
(ix.byHash.get(p.hash) || ix.byHash.set(p.hash, []).get(p.hash)).push(p);
|
|
347
|
+
if (p.path && !ix.byPath.has(p.path.join(","))) ix.byPath.set(p.path.join(","), p); // the first, as a scan finds it
|
|
348
|
+
}
|
|
349
|
+
INDEX.set(paragraphs, ix);
|
|
350
|
+
return ix;
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// The paragraphs in the order nearest() ranks them, one level of equal distance from the body block `want` at a time (document order
|
|
354
|
+
// within a level), those without a body position last: the first level holding a match holds the match a scan of all of them picks.
|
|
355
|
+
function* byDistance(paragraphs, ix, want) {
|
|
356
|
+
if (!Number.isSafeInteger(want) || !ix.byTop.size) return yield paragraphs;
|
|
357
|
+
// the body blocks that hold paragraphs, outward from `want` (a body of many tables and few paragraphs has gaps: they are not walked)
|
|
358
|
+
const tops = (ix.tops ||= [...ix.byTop.keys()].sort((a, b) => a - b));
|
|
359
|
+
let hi = 0;
|
|
360
|
+
for (let lo = 0, top = tops.length; lo < top; ) {
|
|
361
|
+
const mid = (lo + top) >> 1;
|
|
362
|
+
if (tops[mid] < want) lo = mid + 1;
|
|
363
|
+
else top = mid;
|
|
364
|
+
hi = lo;
|
|
365
|
+
}
|
|
366
|
+
let lo = hi - 1;
|
|
367
|
+
while (lo >= 0 || hi < tops.length) {
|
|
368
|
+
const dl = lo >= 0 ? want - tops[lo] : Infinity;
|
|
369
|
+
const dh = hi < tops.length ? tops[hi] - want : Infinity;
|
|
370
|
+
const d = Math.min(dl, dh);
|
|
371
|
+
const a = dl === d ? ix.byTop.get(tops[lo--]) : null;
|
|
372
|
+
const b = dh === d ? ix.byTop.get(tops[hi++]) : null;
|
|
373
|
+
yield a && b ? a.concat(b) : a || b;
|
|
374
|
+
}
|
|
375
|
+
if (ix.noPath.length) yield ix.noPath;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
const BIGRAMS = new WeakMap();
|
|
379
|
+
const bigramsOf = (p) => {
|
|
380
|
+
let v = BIGRAMS.get(p);
|
|
381
|
+
if (!v) BIGRAMS.set(p, (v = bigramCodes(p.text)));
|
|
382
|
+
return v;
|
|
383
|
+
};
|
|
384
|
+
|
|
290
385
|
function locate(paragraphs, { path, paraId, quote, prefix, text, prevHash, nextHash, dupCount, allowPath = false }) {
|
|
291
|
-
|
|
386
|
+
// The page's quote and paragraph text (each can be a whole paragraph, per comment) are paid for before they are read: the text is
|
|
387
|
+
// normalized up to four times below (its hash, its length, the slot's similarity, its bigrams). Past the budget a step that needs it
|
|
388
|
+
// is skipped, and at Send the bare position still answers.
|
|
389
|
+
const q = normPaid(quote) ?? "";
|
|
390
|
+
if (text && !(affords(4 * text.length) && spend(4 * weigh(text)))) text = "";
|
|
292
391
|
const want = path ? path[0] : null;
|
|
293
392
|
const dist = (p) => (want == null || !p.path ? 1e6 : Math.abs(p.path[0] - want));
|
|
294
|
-
|
|
393
|
+
// the nearest of `list` (smaller distance, then smaller ordinal) in one pass, charged to the request's budget: null when it runs out
|
|
394
|
+
// (sorting a copy of 50,000 look-alikes for each of 500 anchors held the thread for 4 s, outside the budget)
|
|
395
|
+
const nearest = (list) => {
|
|
396
|
+
if (!spend(list.length * 3)) return null;
|
|
397
|
+
let best = null;
|
|
398
|
+
let bd = 0;
|
|
399
|
+
for (const p of list) {
|
|
400
|
+
const d = dist(p);
|
|
401
|
+
if (!best || d < bd || (d === bd && p.ordinal < best.ordinal)) (best = p), (bd = d);
|
|
402
|
+
}
|
|
403
|
+
return best;
|
|
404
|
+
};
|
|
295
405
|
const nb = (p) => (prevHash && paragraphs[p.ordinal - 1]?.hash === prevHash ? 1 : 0) + (nextHash && paragraphs[p.ordinal + 1]?.hash === nextHash ? 1 : 0);
|
|
406
|
+
const ix = indexOf(paragraphs);
|
|
296
407
|
// 1. w14:paraId
|
|
297
408
|
if (paraId) {
|
|
298
|
-
const hit =
|
|
409
|
+
const hit = ix.byId.get(paraId.toUpperCase()) || [];
|
|
299
410
|
if (hit.length === 1) return { p: hit[0], method: "id" };
|
|
300
411
|
}
|
|
301
412
|
// 2. identical text; among duplicates prefer the one whose neighbors are unchanged, then the nearest.
|
|
@@ -304,44 +415,110 @@ function locate(paragraphs, { path, paraId, quote, prefix, text, prevHash, nextH
|
|
|
304
415
|
let ambiguousHash = null;
|
|
305
416
|
if (text) {
|
|
306
417
|
const h = textHash(text);
|
|
307
|
-
const hits =
|
|
418
|
+
const hits = ix.byHash.get(h) || [];
|
|
308
419
|
const ambiguous = (dupCount != null ? dupCount > 1 : normText(text).length < 20) && (prevHash || nextHash);
|
|
309
|
-
|
|
310
|
-
|
|
420
|
+
// one pass over the hits for the best neighbor count, then the nearest of those (a document of 50,000 identical paragraphs and 500
|
|
421
|
+
// anchors held the thread for 4 s: the scans and the sort were outside the budget). Spent up front, at about what a hit costs here.
|
|
422
|
+
if (hits.length && spend(hits.length * 24)) {
|
|
423
|
+
let best = null;
|
|
424
|
+
let top = 0;
|
|
425
|
+
let bd = 0;
|
|
426
|
+
for (const p of hits) {
|
|
427
|
+
const n = nb(p);
|
|
428
|
+
const d = dist(p);
|
|
429
|
+
if (!best || n > top || (n === top && (d < bd || (d === bd && p.ordinal < best.ordinal)))) (best = p), (top = n), (bd = d);
|
|
430
|
+
}
|
|
431
|
+
if (!ambiguous || top > 0) return { p: best, method: "hash" };
|
|
432
|
+
ambiguousHash = h;
|
|
433
|
+
}
|
|
311
434
|
}
|
|
312
435
|
// 3. same slot: both neighbors unchanged and exactly one paragraph sits between them (edited in place)
|
|
313
436
|
if (prevHash && nextHash) {
|
|
314
|
-
const
|
|
315
|
-
|
|
437
|
+
const after = ix.byHash.get(prevHash) || [];
|
|
438
|
+
const slots = spend(after.length * 4) ? after.map((p) => paragraphs[p.ordinal + 1]).filter((p) => p && nb(p) === 2) : [];
|
|
439
|
+
// the slot's whole text is compared: within the request's budget (a slot of 8 million characters, 50 comments: 56 s before)
|
|
440
|
+
if (slots.length === 1 && (!text || (spend(2 * weightOf(slots[0]) + normCost(slots[0])) && (similarity(slots[0].text, text) >= 0.3 || (q && norm(slots[0]).includes(q)))))) return { p: slots[0], method: "neighbors" };
|
|
316
441
|
}
|
|
317
|
-
// 4
|
|
442
|
+
// steps 4 and 5 scan every paragraph: only within the budget of the anchors resolved together (src/text.js scanBudget), 16 units a
|
|
443
|
+
// paragraph, and the text each step searches character by character
|
|
444
|
+
// 4. exact quote. Ambiguous when the quote also occurs elsewhere and the prefix does not disambiguate: do not guess here. The
|
|
445
|
+
// paragraphs are read nearest first (byDistance): the answer is the nearest match, so the search stops at the first distance that
|
|
446
|
+
// has one (50 comments on a paragraph the agent changed, in 26,000 paragraphs, searched every paragraph 50 times).
|
|
447
|
+
let read = true;
|
|
318
448
|
if (q) {
|
|
319
|
-
const
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
449
|
+
const pre = normPaid(prefix) ?? "";
|
|
450
|
+
const longer = pre ? [pre + (pre.endsWith(" ") ? "" : " ") + q, pre + q] : null;
|
|
451
|
+
const hits = [];
|
|
452
|
+
const better = [];
|
|
453
|
+
for (const level of byDistance(paragraphs, ix, want)) {
|
|
454
|
+
for (const p of level) {
|
|
455
|
+
if (!(read = spend(16 + normCost(p)) && spend(normSearch(p, q)))) break; // a native substring search (src/text.js searchCost)
|
|
456
|
+
if (!norm(p).includes(q) || (ambiguousHash && p.hash === ambiguousHash)) continue; // look-alikes rejected in step 2 stay rejected
|
|
457
|
+
hits.push(p);
|
|
458
|
+
// two native substring searches per hit (src/text.js searchCost, as in the loop above)
|
|
459
|
+
if (longer && spend(2 * normSearch(p, longer[1])) && (norm(p).includes(longer[0]) || norm(p).includes(longer[1]))) better.push(p);
|
|
460
|
+
}
|
|
461
|
+
if (!read) break;
|
|
462
|
+
const found = better.length ? better : !pre && !text ? hits : [];
|
|
463
|
+
if (found.length) {
|
|
464
|
+
const p = nearest(found);
|
|
465
|
+
if (p) return { p, method: "quote" };
|
|
466
|
+
}
|
|
325
467
|
}
|
|
326
|
-
|
|
468
|
+
if (read && hits.length === 1) {
|
|
469
|
+
const p = nearest(hits);
|
|
470
|
+
if (p) return { p, method: "quote" };
|
|
471
|
+
}
|
|
472
|
+
} else read = spend(paragraphs.length * 16);
|
|
327
473
|
// 5. fuzzy: similarity to the remembered text (or the quote), neighbors count in favor
|
|
328
474
|
const probe = text || q;
|
|
329
|
-
|
|
475
|
+
let spent = !read;
|
|
476
|
+
if (probe && read) {
|
|
330
477
|
// The best candidate must clearly beat the runner-up: in templated text a deleted paragraph looks like its siblings,
|
|
331
478
|
// and "moved or removed" is a better answer than attaching the comment to the wrong one.
|
|
332
|
-
|
|
479
|
+
// the two best candidates only, found in one pass: the answer needs the winner and the runner-up, never the order of the rest
|
|
480
|
+
let best = null;
|
|
481
|
+
let second = null;
|
|
482
|
+
let count = 0;
|
|
483
|
+
const P = bigramCodes(text || q); // the probe's bigrams once; each paragraph's are kept with it (bigramsOf)
|
|
484
|
+
const n = P.codes.length;
|
|
333
485
|
for (const p of paragraphs) {
|
|
334
486
|
if (ambiguousHash && p.hash === ambiguousHash) continue; // rejected above: a look-alike elsewhere is not this paragraph
|
|
335
|
-
|
|
336
|
-
if (
|
|
487
|
+
let sim = 0;
|
|
488
|
+
if (text) {
|
|
489
|
+
if (!BIGRAMS.has(p) && !spend(weightOf(p))) {
|
|
490
|
+
spent = true; // its bigrams are read (and sorted) once per reading: the first time costs its text
|
|
491
|
+
break;
|
|
492
|
+
}
|
|
493
|
+
const B = bigramsOf(p);
|
|
494
|
+
// 0.6 needs the shorter at least 3/7 of the longer (Dice is at most 2 min / (m + n)): skip the others without comparing
|
|
495
|
+
const m = B.codes.length;
|
|
496
|
+
if (P.short === null && B.short === null && (Math.min(m, n) * 7 < Math.max(m, n) * 3)) continue;
|
|
497
|
+
if (!spend(m + n)) {
|
|
498
|
+
spent = true; // the budget is spent: "moved or removed" (src/text.js scanBudget), or the bare position at Send
|
|
499
|
+
break;
|
|
500
|
+
}
|
|
501
|
+
sim = diceOf(B, P);
|
|
502
|
+
} else if (norm(p).length) {
|
|
503
|
+
if (!spend(normCost(p) + q.length + 40 + n)) {
|
|
504
|
+
spent = true;
|
|
505
|
+
break;
|
|
506
|
+
}
|
|
507
|
+
sim = diceOf(bigramCodes(norm(p).slice(0, q.length + 40)), P);
|
|
508
|
+
}
|
|
509
|
+
if (sim >= 0.6) {
|
|
510
|
+
const c = { p, sim, score: sim + nb(p) * 0.1 - dist(p) * 1e-6 }; // distance only breaks ties
|
|
511
|
+
count++;
|
|
512
|
+
if (!best || c.score > best.score) (second = best), (best = c);
|
|
513
|
+
else if (!second || c.score > second.score) second = c;
|
|
514
|
+
}
|
|
337
515
|
}
|
|
338
|
-
|
|
339
|
-
if (scored.length && (scored.length === 1 || scored[0].score - scored[1].score >= 0.04)) return { p: scored[0].p, method: "fuzzy", score: Math.round(scored[0].sim * 100) / 100 };
|
|
516
|
+
if (!spent && best && (count === 1 || best.score - second.score >= 0.04)) return { p: best.p, method: "fuzzy", score: Math.round(best.sim * 100) / 100 };
|
|
340
517
|
}
|
|
341
518
|
// At Send time the browser's own index is as fresh as the bytes, so the bare position is acceptable.
|
|
342
519
|
// After a reload it never is (an insert above shifts it): resolveDurable does not pass allowPath.
|
|
343
520
|
if (allowPath && path) {
|
|
344
|
-
const hit =
|
|
521
|
+
const hit = ix.byPath.get(path.join(",")); // by lookup, not a scan: the bare position must still answer once the budget is spent
|
|
345
522
|
if (hit) return { p: hit, method: "path" };
|
|
346
523
|
}
|
|
347
524
|
return null;
|
|
@@ -358,21 +535,152 @@ export function durableFromEngine(paragraphs, loc, ctx = {}) {
|
|
|
358
535
|
return durableFor(paragraphs, hit.p, loc.blockPath ? loc.blockPath[0] : null, ctx, loc);
|
|
359
536
|
}
|
|
360
537
|
|
|
361
|
-
/**
|
|
362
|
-
|
|
538
|
+
/**
|
|
539
|
+
* Offsets of the non-overlapping occurrences of `q` in `text` (the first `max` of them), or null when the request's budget runs out: the
|
|
540
|
+
* search is counted as a native substring search (src/text.js searchCost) and each occurrence found at OCCURRENCE.
|
|
541
|
+
*/
|
|
542
|
+
// the complete list of a quote's occurrences in a paragraph, kept per paragraph of a reading for the last few quotes (rule: what is
|
|
543
|
+
// derived from the same paragraph is computed once per reading): 50 comments on one selection of a repeated text listed the same
|
|
544
|
+
// 395,000 occurrences 50 times. Ceiling: OCC_KEEP quotes per paragraph, the oldest dropped first.
|
|
545
|
+
const OCC = new WeakMap();
|
|
546
|
+
const OCC_KEEP = 16;
|
|
547
|
+
function occurrencesIn(p, q) {
|
|
548
|
+
let m = OCC.get(p);
|
|
549
|
+
const hit = m?.get(q);
|
|
550
|
+
if (hit) return hit;
|
|
551
|
+
const occ = occurrences(p, q);
|
|
552
|
+
if (occ) {
|
|
553
|
+
if (!m) OCC.set(p, (m = new Map()));
|
|
554
|
+
if (m.size >= OCC_KEEP) m.delete(m.keys().next().value);
|
|
555
|
+
m.set(q, occ);
|
|
556
|
+
}
|
|
557
|
+
return occ;
|
|
558
|
+
}
|
|
559
|
+
function occurrences(p, q, max = Infinity) {
|
|
560
|
+
const { text } = p;
|
|
561
|
+
if (!spend(searchCost(text.length, q, hitsOf(RAW_HITS, p, text, q)))) return null;
|
|
363
562
|
const out = [];
|
|
364
|
-
for (let i = text.indexOf(q); i >= 0; i = text.indexOf(q, i + q.length))
|
|
563
|
+
for (let i = text.indexOf(q); i >= 0 && out.length < max; i = text.indexOf(q, i + q.length)) {
|
|
564
|
+
if (!spend(OCCURRENCE)) return null;
|
|
565
|
+
out.push(i);
|
|
566
|
+
}
|
|
365
567
|
return out;
|
|
366
568
|
}
|
|
367
569
|
|
|
570
|
+
/**
|
|
571
|
+
* normText(text).slice(0, n), reading only as much of the text as that needs (windows cut as endsWithNorm cuts them), each character read
|
|
572
|
+
* spent from the budget; null past it. A comment's paragraph text is kept to 2,000 characters, and normalizing a whole paragraph of
|
|
573
|
+
* 8 million characters for each of 50 comments held the thread for seconds.
|
|
574
|
+
*/
|
|
575
|
+
function normHead(text, n) {
|
|
576
|
+
for (let size = 2 * n + 8; ; size *= 2) {
|
|
577
|
+
const e = cutNear(text, Math.min(text.length, size), 1);
|
|
578
|
+
if (e === null || !spend(weigh(text.slice(0, e)))) return null;
|
|
579
|
+
const head = normText(text.slice(0, e));
|
|
580
|
+
if (e === text.length || head.length > n) return head.slice(0, n);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
// normText(text.slice(0, at)).endsWith(prefix) and normText(text.slice(from)).startsWith(suffix), reading only as much of the text as
|
|
585
|
+
// the prefix or suffix needs. A window is cut only where nothing on one side changes how the other side normalizes (`cuttable`), so the
|
|
586
|
+
// window normalizes to exactly the end (or the start) of what the whole text normalizes to, give or take one space where it was cut.
|
|
587
|
+
// Once the window holds at least as many normalized characters as the prefix, it answers for the whole text; else it doubles.
|
|
588
|
+
// Every character read (the walk to a cut and the window normalized) is spent from the budget of the request (src/text.js): null when it
|
|
589
|
+
// runs out. An outside review: CJK text has no character below U+0300, every window grew to the whole paragraph, 3.65 s for one anchor.
|
|
590
|
+
const MARK = /\p{M}/u;
|
|
591
|
+
/**
|
|
592
|
+
* May a window start (or end) right before text[i]? Yes below U+0300 (no such character joins the one before it under NFC); never inside a
|
|
593
|
+
* surrogate pair or before a combining mark; otherwise when the character normalizes the same after the three before it as on its own
|
|
594
|
+
* (the longest chain of starters that compose, Hangul L V T and its like, is three: test/parse-limits.test.js checks every code point).
|
|
595
|
+
* Exported with the two window readers for that test.
|
|
596
|
+
*/
|
|
597
|
+
export function cuttable(text, i) {
|
|
598
|
+
if (i <= 0 || i >= text.length) return true;
|
|
599
|
+
const c = text.charCodeAt(i);
|
|
600
|
+
if (c < 0x300) return true;
|
|
601
|
+
if (c >= 0xdc00 && c <= 0xdfff) return false;
|
|
602
|
+
const ch = String.fromCodePoint(text.codePointAt(i));
|
|
603
|
+
if (MARK.test(ch)) return false;
|
|
604
|
+
const before = text.slice(Math.max(0, i - 3), i);
|
|
605
|
+
return (before + ch).normalize("NFC") === before.normalize("NFC") + ch.normalize("NFC");
|
|
606
|
+
}
|
|
607
|
+
/** The nearest place at or before (dir -1) or after (dir 1) i where a window may be cut, each character walked spent; null past the budget. */
|
|
608
|
+
function cutNear(text, i, dir) {
|
|
609
|
+
let n = 0;
|
|
610
|
+
while (!cuttable(text, i)) {
|
|
611
|
+
i += dir;
|
|
612
|
+
if (++n % 1024 === 0 && !spend(1024)) return null;
|
|
613
|
+
}
|
|
614
|
+
return spend(n % 1024) ? i : null;
|
|
615
|
+
}
|
|
616
|
+
export function endsWithNorm(text, at, prefix) {
|
|
617
|
+
for (let size = 2 * prefix.length + 8; ; size *= 2) {
|
|
618
|
+
const w = cutNear(text, Math.max(0, at - size), -1);
|
|
619
|
+
if (w === null || !spend(weigh(text.slice(w, at)))) return null;
|
|
620
|
+
const n = normText(text.slice(w, at));
|
|
621
|
+
if (w === 0 || n.length >= prefix.length) return n.endsWith(prefix);
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
export function startsWithNorm(text, from, suffix) {
|
|
625
|
+
for (let size = 2 * suffix.length + 8; ; size *= 2) {
|
|
626
|
+
const e = cutNear(text, Math.min(text.length, from + size), 1);
|
|
627
|
+
if (e === null || !spend(weigh(text.slice(from, e)))) return null;
|
|
628
|
+
const n = normText(text.slice(from, e));
|
|
629
|
+
if (e === text.length || n.length >= suffix.length) return n.startsWith(suffix);
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
368
633
|
/**
|
|
369
634
|
* Which occurrence of a repeated quote a selection or a stored anchor means. Prefix and suffix are the normalized text around it;
|
|
370
|
-
* `nth` is the occurrence index remembered at capture. Returns the offset, or -1 when the
|
|
635
|
+
* `nth` is the occurrence index remembered at capture. Returns {at: the offset, k: which occurrence it is}, or {at: -1, k: null} when the
|
|
636
|
+
* text does not occur or the budget ran out.
|
|
371
637
|
*/
|
|
372
|
-
function pickOccurrence(
|
|
373
|
-
const
|
|
374
|
-
|
|
375
|
-
|
|
638
|
+
function pickOccurrence(p, q, opts = {}) {
|
|
639
|
+
const text = p.text;
|
|
640
|
+
// the remembered occurrence first, listing only the occurrences up to it: when its surroundings still fit, it is the answer every rule
|
|
641
|
+
// of pickFrom gives, and the others need not be listed or read (an outside review: 300,000 "a" with nth 150,000 lost its offsets
|
|
642
|
+
// under the budget, though the remembered one still fit)
|
|
643
|
+
const { nth = null } = opts;
|
|
644
|
+
// only an index of the occurrences is one: a selection that starts between two of them ("aa" at offset 1 of "aaaa", which the
|
|
645
|
+
// non-overlapping search does not list) was stored with nth -1, and occ[-1] (undefined) made the window readers loop forever on the
|
|
646
|
+
// server's thread, the budget never asked (a window of no characters costs nothing)
|
|
647
|
+
const remembered = Number.isInteger(nth) && nth >= 0 ? nth : null;
|
|
648
|
+
if (remembered !== null && (opts.prefix || opts.suffix)) {
|
|
649
|
+
const head = occurrences(p, q, remembered + 1);
|
|
650
|
+
if (head === null) return { at: -1, k: null };
|
|
651
|
+
if (head.length === remembered + 1) {
|
|
652
|
+
const ok = fitsAt(text, q, head[remembered], opts);
|
|
653
|
+
if (ok === null) return { at: -1, k: null };
|
|
654
|
+
if (ok) return { at: head[remembered], k: remembered };
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
const occ = occurrencesIn(p, q);
|
|
658
|
+
const at = occ?.length ? pickFrom(text, q, occ, { ...opts, nth: remembered }) : -1;
|
|
659
|
+
return at < 0 ? { at: -1, k: null } : { at, k: occ.indexOf(at) };
|
|
660
|
+
}
|
|
661
|
+
/** does the text around the occurrence at `at` read as prefix and suffix (normalized, as an anchor keeps them)? null: the budget ran out */
|
|
662
|
+
function fitsAt(text, q, at, { prefix = "", suffix = "" }) {
|
|
663
|
+
prefix = String(prefix || "").slice(-CTX); // what an anchor keeps (durableFor); a longer one from elsewhere is read as its last CTX
|
|
664
|
+
suffix = String(suffix || "").slice(0, CTX);
|
|
665
|
+
const pre = !prefix || endsWithNorm(text, at, prefix);
|
|
666
|
+
if (pre === null) return null;
|
|
667
|
+
const suf = pre && (!suffix || startsWithNorm(text, at + q.length, suffix));
|
|
668
|
+
return suf === null ? null : !!(pre && suf);
|
|
669
|
+
}
|
|
670
|
+
function pickFrom(text, q, occ, { prefix = "", suffix = "", nth = null }) {
|
|
671
|
+
if (!(Number.isInteger(nth) && nth >= 0 && nth < occ.length)) nth = null;
|
|
672
|
+
prefix = String(prefix || "").slice(-CTX);
|
|
673
|
+
suffix = String(suffix || "").slice(0, CTX);
|
|
674
|
+
const fits = (at) => fitsAt(text, q, at, { prefix, suffix });
|
|
675
|
+
// each occurrence reads about CTX characters on each side, out of the budget of the request (src/text.js): a paragraph of 100,000
|
|
676
|
+
// "a" read the whole text before every one of them and took 10 seconds in an outside review. Past the budget, no offset.
|
|
677
|
+
if ((prefix || suffix) && occ.length > 1 && !spend(occ.length * 2 * (prefix.length + suffix.length + 16))) return -1;
|
|
678
|
+
const guarded = [];
|
|
679
|
+
for (const at of occ) {
|
|
680
|
+
const ok = fits(at);
|
|
681
|
+
if (ok === null) return -1;
|
|
682
|
+
if (ok) guarded.push(at);
|
|
683
|
+
}
|
|
376
684
|
if (guarded.length === 1) return guarded[0];
|
|
377
685
|
if (nth != null && nth < occ.length && guarded.includes(occ[nth])) return occ[nth];
|
|
378
686
|
if (guarded.length) return guarded[0];
|
|
@@ -380,20 +688,22 @@ function pickOccurrence(text, q, { prefix = "", suffix = "", nth = null } = {})
|
|
|
380
688
|
}
|
|
381
689
|
|
|
382
690
|
function durableFor(paragraphs, p, engineTop, ctx, loc = {}) {
|
|
383
|
-
|
|
384
|
-
const q =
|
|
385
|
-
const prefix =
|
|
386
|
-
const suffix =
|
|
691
|
+
// the page's quote, prefix and suffix, paid for before they are normalized; past the budget the anchor is the paragraph alone
|
|
692
|
+
const q = normPaid(ctx.quote) ?? "";
|
|
693
|
+
const prefix = (normPaid(ctx.prefix) ?? "").slice(-CTX);
|
|
694
|
+
const suffix = (normPaid(ctx.suffix) ?? "").slice(0, CTX);
|
|
387
695
|
let start = null;
|
|
388
696
|
let end = null;
|
|
389
697
|
let nth = null;
|
|
390
698
|
if (q) {
|
|
391
699
|
// the engine's own offsets when they point at the quote; else the occurrence whose surroundings match prefix and suffix
|
|
392
|
-
const
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
700
|
+
const own = Number.isInteger(loc.start) && Number.isInteger(loc.end) && p.text.slice(loc.start, loc.end) === ctx.quote;
|
|
701
|
+
const k = own ? occurrencesIn(p, ctx.quote)?.indexOf(loc.start) ?? -1 : -1;
|
|
702
|
+
const picked = own ? { at: loc.start, k: k >= 0 ? k : null } : pickOccurrence(p, ctx.quote, { prefix, suffix });
|
|
703
|
+
if (picked.at >= 0) {
|
|
704
|
+
start = picked.at;
|
|
705
|
+
end = picked.at + ctx.quote.length;
|
|
706
|
+
nth = picked.k;
|
|
397
707
|
}
|
|
398
708
|
}
|
|
399
709
|
return {
|
|
@@ -407,8 +717,8 @@ function durableFor(paragraphs, p, engineTop, ctx, loc = {}) {
|
|
|
407
717
|
textHash: p.hash,
|
|
408
718
|
prevHash: paragraphs[p.ordinal - 1]?.hash || null,
|
|
409
719
|
nextHash: paragraphs[p.ordinal + 1]?.hash || null,
|
|
410
|
-
dupCount: paragraphs.
|
|
411
|
-
paraText:
|
|
720
|
+
dupCount: indexOf(paragraphs).byHash.get(p.hash)?.length ?? 0,
|
|
721
|
+
paraText: normHead(p.text, PARA_TEXT_KEEP) ?? "",
|
|
412
722
|
quote: q,
|
|
413
723
|
prefix,
|
|
414
724
|
suffix,
|
|
@@ -429,10 +739,10 @@ export function resolveDurable(paragraphs, d) {
|
|
|
429
739
|
const q = d.quote || "";
|
|
430
740
|
let start = null;
|
|
431
741
|
let end = null;
|
|
432
|
-
const at = q ? pickOccurrence(p
|
|
742
|
+
const { at, k } = q ? pickOccurrence(p, q, { prefix: d.prefix, suffix: d.suffix, nth: d.nth }) : { at: -1, k: null };
|
|
433
743
|
if (at >= 0) {
|
|
434
744
|
start = at;
|
|
435
745
|
end = at + q.length;
|
|
436
746
|
}
|
|
437
|
-
return { method: hit.method, score: hit.score, ordinal: p.ordinal, path: p.path, tableNo: p.tableNo ?? null, paraId: p.paraId, textHash: p.hash, protected: !!p.protected, ...(p.protected ? { protectInfo: protectInfo(p) } : {}), dupCount: paragraphs.
|
|
747
|
+
return { method: hit.method, score: hit.score, ordinal: p.ordinal, path: p.path, tableNo: p.tableNo ?? null, paraId: p.paraId, textHash: p.hash, protected: !!p.protected, ...(p.protected ? { protectInfo: protectInfo(p) } : {}), dupCount: indexOf(paragraphs).byHash.get(p.hash)?.length ?? 0, nth: at >= 0 ? k : null, prevHash: paragraphs[p.ordinal - 1]?.hash || null, nextHash: paragraphs[p.ordinal + 1]?.hash || null, text: normHead(p.text, PARA_TEXT_KEEP) ?? "", start, end, engineDelta: d.engineDelta || 0 };
|
|
438
748
|
}
|
package/src/edit-hash.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// The text hash of a pending HTML text edit (PLAN-html R9): one function for the page (web/src/html-view.js) and the server
|
|
2
|
+
// (src/html-write.js), so both read the same 14 hex digits. Pure, no imports. Fixed vectors: test/html-view.test.js.
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* cyrb53 over the exact text (UTF-16 code units, nothing normalized), 14 hex digits. Not a security hash: it tells "the text the edit
|
|
6
|
+
* was made on" from another text, and the version's sha256 travels with it.
|
|
7
|
+
*/
|
|
8
|
+
export function editHash(s) {
|
|
9
|
+
let h1 = 0xdeadbeef;
|
|
10
|
+
let h2 = 0x41c6ce57;
|
|
11
|
+
for (let i = 0; i < s.length; i++) {
|
|
12
|
+
const c = s.charCodeAt(i);
|
|
13
|
+
h1 = Math.imul(h1 ^ c, 2654435761);
|
|
14
|
+
h2 = Math.imul(h2 ^ c, 1597334677);
|
|
15
|
+
}
|
|
16
|
+
h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909);
|
|
17
|
+
h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909);
|
|
18
|
+
return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(16).padStart(14, "0");
|
|
19
|
+
}
|
package/src/fsutil.js
CHANGED
|
@@ -19,7 +19,7 @@ export function checkRegularOwned(file) {
|
|
|
19
19
|
/** Reads a document, refusing by size first (an oversized file is never pulled into memory). */
|
|
20
20
|
export function readDoc(file, max = MAX_DOC_BYTES) {
|
|
21
21
|
const st = fs.statSync(file);
|
|
22
|
-
if (st.size > max) throw Object.assign(new Error(`${path.basename(file)} is larger than ${Math.round(max / 1024 / 1024)} MB`), { code: "TOO_LARGE" });
|
|
22
|
+
if (st.size > max) throw Object.assign(new Error(`${path.basename(file)} is larger than ${Math.round(max / 1024 / 1024)} MB, more than Onionskin can read. Make it smaller (pictures and videos are often most of a file) or split it, and open that`), { code: "TOO_LARGE" });
|
|
23
23
|
return fs.readFileSync(file);
|
|
24
24
|
}
|
|
25
25
|
|
package/src/history-worker.js
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
|
-
import
|
|
2
|
+
import "./html-parser.js"; // an HTML file's versions are compared here too: the parser lives in worker threads only
|
|
3
3
|
import { compareBytes } from "./history.js";
|
|
4
4
|
|
|
5
|
-
// One message back: a result per job, in order. A version that cannot be read or parsed fails its own job only.
|
|
6
|
-
|
|
7
|
-
const
|
|
8
|
-
|
|
9
|
-
|
|
5
|
+
// Run by src/worker-host.js. One message back: a result per job, in order. A version that cannot be read or parsed fails its own job only.
|
|
6
|
+
export function work(data) {
|
|
7
|
+
const bytes = new Map(); // blob path -> Buffer, so a version shared by two pairs is read (and parsed) once
|
|
8
|
+
const read = (f) => bytes.get(f) || (bytes.set(f, fs.readFileSync(f)), bytes.get(f));
|
|
9
|
+
return data.jobs.map((j) => {
|
|
10
10
|
try {
|
|
11
11
|
return { ok: true, result: compareBytes(j.format, read(j.before), read(j.after), { places: j.places }) };
|
|
12
12
|
} catch (err) {
|
|
13
13
|
return { ok: false, error: String(err?.message || err) };
|
|
14
14
|
}
|
|
15
|
-
})
|
|
16
|
-
|
|
15
|
+
});
|
|
16
|
+
}
|