onionskin-cli 0.0.3 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +198 -0
- package/INSTALL.md +5 -1
- package/NOTICE +34 -0
- package/README.md +102 -34
- package/docs/contracts/batch.schema.json +27 -4
- package/docs/contracts/ops.schema.json +117 -5
- package/package.json +4 -2
- package/scripts/checker/check.py +24 -5
- package/scripts/checker/requirements.txt +482 -5
- package/scripts/requirements.txt +482 -5
- package/scripts/rewriter/apple/main.swift +1 -1
- package/scripts/rewriter/apple/onionskin-fm +0 -0
- package/scripts/rewriter/apple/onionskin-fm.json +1 -1
- package/skills/onionskin/SKILL.md +12 -9
- package/skills/onionskin/reference/batch.md +5 -3
- package/skills/onionskin/reference/formats.md +5 -1
- package/skills/onionskin/reference/ops.md +48 -4
- package/skills/onionskin/reference/troubleshooting.md +31 -0
- package/src/agent-detect.js +28 -2
- package/src/agent-prompt.js +12 -1
- package/src/agent-runner.js +110 -20
- package/src/anchor-work.js +154 -0
- package/src/anchor-worker.js +38 -0
- package/src/anchors.js +61 -11
- package/src/apply.js +119 -47
- package/src/batch.js +32 -8
- package/src/check-worker.js +8 -5
- package/src/checker.js +238 -33
- package/src/cli.js +248 -42
- package/src/constants.js +34 -1
- package/src/diff.js +3 -1
- package/src/doctor.js +12 -3
- package/src/docx.js +464 -87
- package/src/edit-hash.js +19 -0
- package/src/fsutil.js +14 -6
- package/src/history-worker.js +8 -8
- package/src/history.js +144 -31
- package/src/html-active.js +384 -0
- package/src/html-envelope.js +51 -0
- package/src/html-parser.js +8 -0
- package/src/html-render-worker.js +25 -0
- package/src/html-render.js +939 -0
- package/src/html-utf8.js +65 -0
- package/src/html-write-worker.js +14 -0
- package/src/html-write.js +380 -0
- package/src/html.js +1259 -0
- package/src/local.js +68 -4
- package/src/merge-package.js +1166 -0
- package/src/merge-pptx.js +98 -38
- package/src/merge-proof.js +1019 -0
- package/src/merge-worker.js +60 -0
- package/src/merge-xlsx.js +84 -29
- package/src/merge.js +286 -83
- package/src/nfc.js +46 -0
- package/src/notes.js +122 -0
- package/src/offload.js +419 -1
- package/src/open-worker.js +27 -0
- package/src/ops/common.js +2 -1
- package/src/ops/docx-table.js +5 -3
- package/src/ops/docx.js +58 -29
- package/src/ops/html.js +394 -0
- package/src/ops/pptx.js +82 -23
- package/src/ops/run.js +23 -3
- package/src/ops/runs.js +215 -13
- package/src/ops/schema.js +8 -1
- package/src/ops/xlsx.js +121 -10
- package/src/ops-worker.js +17 -0
- package/src/package.js +43 -4
- package/src/person.js +2 -1
- package/src/pptx.js +323 -23
- package/src/question.js +10 -3
- package/src/quick-plan.js +13 -6
- package/src/quick.js +1 -1
- package/src/redline-check.js +12 -2
- package/src/redline-worker.js +10 -14
- package/src/reply.js +46 -19
- package/src/retain.js +223 -0
- package/src/review.js +32 -18
- package/src/rewrite-check.js +2 -2
- package/src/rewriter.js +38 -8
- package/src/sanitize.js +16 -3
- package/src/server-entry.js +4 -3
- package/src/server.js +175 -62
- package/src/session.js +1151 -210
- package/src/setup.js +20 -2
- package/src/stop-gate.js +34 -0
- package/src/suggest-reply.js +42 -7
- package/src/suggest.js +14 -29
- package/src/text.js +161 -11
- package/src/update.js +517 -0
- package/src/voice-guide.js +15 -9
- package/src/voice-learn.js +15 -4
- package/src/voice-sample-worker.js +7 -6
- package/src/voice-store.js +11 -4
- package/src/voice-worker.js +4 -6
- package/src/voice.js +2 -1
- package/src/worker-host.js +23 -0
- package/src/xlsx.js +378 -97
- package/src/xml-tree.js +6 -3
- package/src/xml.js +82 -34
- package/src/zip-write.js +10 -5
- package/src/zip.js +73 -18
- package/vendor/parse5/LICENSE.txt +32 -0
- package/vendor/parse5/parse5.mjs +7822 -0
- package/vendor/parse5.lock.json +23 -0
- package/web/dist/assets/{index-DS-WST1E.js → index-CjsTf5gj.js} +58 -52
- package/web/dist/assets/index-DnbLtHhG.css +1 -0
- package/web/dist/index.html +2 -2
- package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/docx.js
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { pathToFileURL } from "node:url";
|
|
2
2
|
import { engineNodePath } from "./ops/engine.js";
|
|
3
|
-
import { walkXml } from "./xml.js";
|
|
4
|
-
import { normText, similarity, textHash } from "./text.js";
|
|
3
|
+
import { readBudget, tooMany, walkXml } from "./xml.js";
|
|
4
|
+
import { OCCURRENCE, affords, bigramCodes, diceOf, hitsOf, normPaid, normText, searchCost, similarity, spend, textHash, weigh, weightOf } from "./text.js";
|
|
5
5
|
import { openZip } from "./zip.js";
|
|
6
6
|
import { summarizeRuns } from "./fmt.js";
|
|
7
|
+
import { HAS_NON_PLAIN_WS, cuttable, normOccurrences } from "./pptx.js";
|
|
8
|
+
|
|
9
|
+
export { cuttable };
|
|
7
10
|
|
|
8
11
|
// One classifier for `protected`: the engine's own (default deny), bundled for Node. The batch, `apply` and the editor agree because they ask the same code.
|
|
9
12
|
const { protectReasons } = await import(pathToFileURL(engineNodePath()).href);
|
|
@@ -43,6 +46,16 @@ function runSet(name, a) {
|
|
|
43
46
|
if (name === "w:vertAlign") return only(a, "w:val") && (v === "superscript" || v === "subscript") ? ["va", v.slice(0, 3)] : null;
|
|
44
47
|
return null;
|
|
45
48
|
}
|
|
49
|
+
// What one reading of a Word document may hold. The server reads a document on its own thread at Send, at each delivery to an agent, for a
|
|
50
|
+
// queued comment and at the end of a run, so these bound how long one such reading can keep it from answering anything else (the outside
|
|
51
|
+
// review: a part of 1 million runs held it 4.3 s, a run of 3 million attributes 2.4 s). A real report of about 1,000 pages has about 16,000
|
|
52
|
+
// paragraphs, 240,000 elements, 350,000 attributes and 3 million characters (measured: read in about 240 ms here).
|
|
53
|
+
export const MAX_PARAGRAPHS = 50_000;
|
|
54
|
+
export const MAX_ELEMENTS = 400_000;
|
|
55
|
+
export const MAX_ATTRIBUTES = 800_000;
|
|
56
|
+
export const MAX_TEXT = 8_000_000;
|
|
57
|
+
/** One reading's budget (src/xml.js readBudget), shared by the parts it walks. */
|
|
58
|
+
export const docxBudget = () => readBudget({ elements: MAX_ELEMENTS, attrs: MAX_ATTRIBUTES, text: MAX_TEXT });
|
|
46
59
|
const PPR_OWN = new Set(["w:pStyle", "w:jc", "w:numPr", "w:rPr", "w:sectPr"]); // read on their own, or not paragraph formatting the user sets
|
|
47
60
|
const RANGE = /^(.+)(Start|End)$/;
|
|
48
61
|
|
|
@@ -55,7 +68,7 @@ const RANGE = /^(.+)(Start|End)$/;
|
|
|
55
68
|
* (null when they do not add up to it); `list` {numId, ilvl} from its own w:numPr, else null; `ppr`, its other paragraph properties as
|
|
56
69
|
* written; `sect`, it ends a section; `loose`, a bookmark, comment or other range starts or ends inside it without its other end.
|
|
57
70
|
*/
|
|
58
|
-
export function parseDocxParagraphs(xml, { detail = false } = {}) {
|
|
71
|
+
export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget() } = {}) {
|
|
59
72
|
const paragraphs = [];
|
|
60
73
|
const stack = []; // {name, ...}
|
|
61
74
|
let bodyIndex = -1;
|
|
@@ -64,7 +77,9 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
|
|
|
64
77
|
let tableCount = 0; // 1-based order among top-level tables (the label users read), unlike body_path[0] which counts every body item
|
|
65
78
|
let cellK = 0;
|
|
66
79
|
let cur = null;
|
|
67
|
-
|
|
80
|
+
const revStack = [];
|
|
81
|
+
let mcFallbackDepth = 0;
|
|
82
|
+
const visible = () => !revStack.includes("del") && (revStack.length === 0 || revStack[revStack.length - 1] !== "moveFrom");
|
|
68
83
|
let inT = false;
|
|
69
84
|
let inInstr = false;
|
|
70
85
|
let inBody = false;
|
|
@@ -77,9 +92,14 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
|
|
|
77
92
|
walkXml(xml, {
|
|
78
93
|
onOpen(name, attrs) {
|
|
79
94
|
stack.push(name);
|
|
95
|
+
if (name === "mc:Fallback") {
|
|
96
|
+
mcFallbackDepth++;
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
if (mcFallbackDepth > 0) return;
|
|
80
100
|
if (name === "w:body") inBody = true;
|
|
81
101
|
if (!inBody) return;
|
|
82
|
-
if (name === "w:tbl") {
|
|
102
|
+
if (name === "w:tbl" && pDepth === 0) {
|
|
83
103
|
tblDepth++;
|
|
84
104
|
if (tblDepth === 1) {
|
|
85
105
|
bodyIndex++;
|
|
@@ -88,16 +108,15 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
|
|
|
88
108
|
info.row = -1;
|
|
89
109
|
info.col = -1;
|
|
90
110
|
}
|
|
91
|
-
} else if (name === "w:tr" && tblDepth === 1) {
|
|
111
|
+
} else if (name === "w:tr" && tblDepth === 1 && pDepth === 0) {
|
|
92
112
|
info.row++;
|
|
93
113
|
info.col = -1;
|
|
94
|
-
} else if (name === "w:tc" && tblDepth === 1) {
|
|
114
|
+
} else if (name === "w:tc" && tblDepth === 1 && pDepth === 0) {
|
|
95
115
|
info.col++;
|
|
96
116
|
cellK = 0;
|
|
97
117
|
} else if (name === "w:p") {
|
|
98
118
|
pDepth++;
|
|
99
|
-
|
|
100
|
-
if (nested) return;
|
|
119
|
+
if (pDepth > 1) return;
|
|
101
120
|
let path = null;
|
|
102
121
|
let tableNo = null;
|
|
103
122
|
if (tblDepth === 0) {
|
|
@@ -107,18 +126,19 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
|
|
|
107
126
|
path = [info.tableIndex, info.row, info.col, cellK++];
|
|
108
127
|
tableNo = tableCount;
|
|
109
128
|
}
|
|
129
|
+
if (paragraphs.length >= MAX_PARAGRAPHS) throw tooMany(budget, MAX_PARAGRAPHS, "paragraphs");
|
|
110
130
|
cur = { ordinal: paragraphs.length, path, tableNo, paraId: /^[0-9A-Fa-f]{8}$/.test(attrs["w14:paraId"] || "") ? attrs["w14:paraId"].toUpperCase() : null, text: "", instr: "", protected: false, style: null, align: "", runs: [] };
|
|
111
131
|
if (detail) (Object.assign(cur, { spans: [], list: null, ppr: [], sect: false, loose: false }), (marks = new Set()), (pprTop = null));
|
|
112
132
|
paragraphs.push(cur);
|
|
113
|
-
} else if (cur) {
|
|
133
|
+
} else if (cur && pDepth === 1) {
|
|
114
134
|
if (detail) {
|
|
115
135
|
const n = stack.length;
|
|
116
|
-
if (
|
|
136
|
+
if (stack[n - 2] === "w:pPr" && stack[n - 3] === "w:p") {
|
|
117
137
|
pprTop = name;
|
|
118
138
|
if (name === "w:sectPr") cur.sect = true;
|
|
119
139
|
}
|
|
120
140
|
if (pprTop && !PPR_OWN.has(pprTop)) cur.ppr.push(written(name, attrs));
|
|
121
|
-
else if (pprTop === "w:numPr" &&
|
|
141
|
+
else if (pprTop === "w:numPr" && stack[n - 2] === "w:numPr" && (name === "w:numId" || name === "w:ilvl")) (cur.list ||= { numId: "", ilvl: 0 })[name === "w:numId" ? "numId" : "ilvl"] = name === "w:numId" ? attrs["w:val"] || "" : Number(attrs["w:val"]) || 0;
|
|
122
142
|
const mark = RANGE.exec(name);
|
|
123
143
|
if (mark && "w:id" in attrs) {
|
|
124
144
|
const key = `${mark[1]} ${attrs["w:id"]}`;
|
|
@@ -136,35 +156,33 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
|
|
|
136
156
|
else if (name === "w:r" && !run) run = { text: false, p: {}, at: cur.text.length, depth: stack.length, set: {}, rest: [] };
|
|
137
157
|
else if (name === "w:rPr") rprDepth++;
|
|
138
158
|
else if (run && rprDepth === 1 && stack.includes("w:rPr")) runProp(run.p, name, attrs);
|
|
139
|
-
if (name === "w:del")
|
|
140
|
-
else if (name === "w:
|
|
159
|
+
if (name === "w:del") revStack.push("del");
|
|
160
|
+
else if (name === "w:moveFrom") revStack.push("moveFrom");
|
|
161
|
+
else if (name === "w:ins") revStack.push("ins");
|
|
162
|
+
else if (name === "w:t" && visible()) (inT = true, run && (run.text = true));
|
|
141
163
|
else if (name === "w:instrText") inInstr = true;
|
|
142
|
-
else if (name === "w:tab" &&
|
|
164
|
+
else if (name === "w:tab" && visible()) cur.text += "\t";
|
|
143
165
|
else if (name === "w:br" || name === "w:cr") {
|
|
144
|
-
if (
|
|
166
|
+
if (visible() && attrs["w:type"] !== "page" && attrs["w:type"] !== "column") cur.text += "\n";
|
|
145
167
|
} else if (name === "w:drawing" || name === "w:pict") {
|
|
146
|
-
if (
|
|
168
|
+
if (visible()) cur.text += OBJ;
|
|
147
169
|
}
|
|
148
170
|
}
|
|
149
171
|
},
|
|
150
172
|
onText(t) {
|
|
151
|
-
if (
|
|
152
|
-
|
|
173
|
+
if (mcFallbackDepth > 0) return;
|
|
174
|
+
if (cur && pDepth === 1 && inT) cur.text += t;
|
|
175
|
+
else if (cur && pDepth === 1 && inInstr && cur.instr.length < 200) cur.instr += t;
|
|
153
176
|
},
|
|
154
177
|
onClose(name) {
|
|
155
178
|
stack.pop();
|
|
156
|
-
if (
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
else if (name === "w:r" && run) {
|
|
162
|
-
if (run.text && cur) cur.runs.push(run.p);
|
|
163
|
-
if (detail && cur && cur.text.length > run.at) cur.spans.push({ n: cur.text.length - run.at, set: run.set, rest: run.rest.join(" ") });
|
|
164
|
-
run = null;
|
|
165
|
-
} else if (name === "w:p") {
|
|
179
|
+
if (mcFallbackDepth > 0) {
|
|
180
|
+
if (name === "mc:Fallback") mcFallbackDepth--;
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
if (name === "w:p") {
|
|
166
184
|
pDepth--;
|
|
167
|
-
if (
|
|
185
|
+
if (pDepth > 0) return;
|
|
168
186
|
if (cur) (cur.fmt = summarizeRuns(cur.runs), delete cur.runs);
|
|
169
187
|
if (cur && detail) {
|
|
170
188
|
if (cur.spans.reduce((n, x) => n + x.n, 0) !== cur.text.length) cur.spans = null;
|
|
@@ -172,10 +190,23 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
|
|
|
172
190
|
cur.loose = marks.size > 0;
|
|
173
191
|
}
|
|
174
192
|
cur = null;
|
|
193
|
+
revStack.length = 0;
|
|
194
|
+
} else if (name === "w:tbl" && pDepth === 0) {
|
|
195
|
+
tblDepth--;
|
|
196
|
+
} else if (cur && pDepth === 1) {
|
|
197
|
+
if (name === "w:t") inT = false;
|
|
198
|
+
else if (name === "w:instrText") inInstr = false;
|
|
199
|
+
else if (name === "w:del" || name === "w:moveFrom" || name === "w:ins") revStack.pop();
|
|
200
|
+
else if (name === "w:rPr") rprDepth = Math.max(0, rprDepth - 1);
|
|
201
|
+
else if (name === "w:r" && run) {
|
|
202
|
+
if (run.text && cur) cur.runs.push(run.p);
|
|
203
|
+
if (detail && cur && cur.text.length > run.at) cur.spans.push({ n: cur.text.length - run.at, set: run.set, rest: run.rest.join(" ") });
|
|
204
|
+
run = null;
|
|
205
|
+
}
|
|
206
|
+
if (detail && name === pprTop && stack[stack.length - 1] === "w:pPr") pprTop = null;
|
|
175
207
|
}
|
|
176
|
-
if (detail && name === pprTop && stack[stack.length - 1] === "w:pPr") pprTop = null;
|
|
177
208
|
},
|
|
178
|
-
});
|
|
209
|
+
}, budget);
|
|
179
210
|
for (const p of paragraphs) p.hash = textHash(p.text);
|
|
180
211
|
const reasons = protectReasons(xml);
|
|
181
212
|
// the engine lists paragraphs the same way; if the two ever disagree on the count, nothing can be trusted as editable
|
|
@@ -204,10 +235,28 @@ export function protectInfo(p) {
|
|
|
204
235
|
}
|
|
205
236
|
|
|
206
237
|
const HEADING = /^(heading\s?\d|title|subtitle)$/i;
|
|
207
|
-
|
|
238
|
+
// the nearest heading at or before each ordinal, found in one pass per reading: a scan up from every resolved comment was a pass over
|
|
239
|
+
// the paragraphs per comment, and the heading's text was normalized whole again each time, outside the budget
|
|
240
|
+
const HEADS = new WeakMap();
|
|
241
|
+
/** The text of the nearest heading above a paragraph (by paragraph style), or "" when there is none (or the budget cannot pay for it). */
|
|
208
242
|
export function headingBefore(paragraphs, ordinal) {
|
|
209
|
-
|
|
210
|
-
|
|
243
|
+
if (!Number.isInteger(ordinal) || ordinal <= 0) return "";
|
|
244
|
+
let at = HEADS.get(paragraphs);
|
|
245
|
+
if (!at) {
|
|
246
|
+
at = new Int32Array(paragraphs.length + 1);
|
|
247
|
+
let last = -1;
|
|
248
|
+
for (let i = 0; i < paragraphs.length; i++) {
|
|
249
|
+
at[i] = last; // the heading strictly before paragraph i
|
|
250
|
+
const p = paragraphs[i];
|
|
251
|
+
if (p.style && HEADING.test(p.style) && p.text.trim()) last = i;
|
|
252
|
+
}
|
|
253
|
+
at[paragraphs.length] = last;
|
|
254
|
+
HEADS.set(paragraphs, at);
|
|
255
|
+
}
|
|
256
|
+
const h = at[Math.min(ordinal, paragraphs.length)];
|
|
257
|
+
if (h < 0) return "";
|
|
258
|
+
const p = paragraphs[h];
|
|
259
|
+
return spend(normCost(p)) ? norm(p).slice(0, 80) : "";
|
|
211
260
|
}
|
|
212
261
|
|
|
213
262
|
/** `detail` (the merge only): formatting per run and the paragraph's list, see parseDocxParagraphs; `list.kind` is "bullet", "number" or "other". */
|
|
@@ -215,11 +264,12 @@ export function readDocxParagraphs(bytes, { detail = false } = {}) {
|
|
|
215
264
|
const zip = openZip(bytes);
|
|
216
265
|
const xml = zip.readText("word/document.xml");
|
|
217
266
|
if (xml == null) throw new Error("word/document.xml missing");
|
|
218
|
-
const
|
|
219
|
-
const
|
|
267
|
+
const budget = docxBudget(); // the main part, the styles and the numbering of one reading share it
|
|
268
|
+
const paragraphs = parseDocxParagraphs(xml, { detail, budget });
|
|
269
|
+
const names = styleNames(zip.readText("word/styles.xml"), budget);
|
|
220
270
|
if (names.size) for (const p of paragraphs) if (p.style && names.has(p.style)) p.styleName = names.get(p.style);
|
|
221
271
|
if (detail && paragraphs.some((p) => p.list)) {
|
|
222
|
-
const kinds = listKinds(zip.readText("word/numbering.xml"));
|
|
272
|
+
const kinds = listKinds(zip.readText("word/numbering.xml"), budget);
|
|
223
273
|
for (const p of paragraphs) if (p.list) p.list = p.list.numId && p.list.numId !== "0" ? { ...p.list, kind: kinds.get(p.list.numId) || "other" } : null; // numId 0 switches a style's list off
|
|
224
274
|
}
|
|
225
275
|
return paragraphs;
|
|
@@ -229,7 +279,7 @@ export function readDocxParagraphs(bytes, { detail = false } = {}) {
|
|
|
229
279
|
* w:numId -> "bullet" | "number" | "other", by the first level of its definition: the two kinds the editor's list buttons make and
|
|
230
280
|
* the engine can set again. A list with level overrides, or one this cannot read, is "other".
|
|
231
281
|
*/
|
|
232
|
-
function listKinds(xml) {
|
|
282
|
+
function listKinds(xml, budget) {
|
|
233
283
|
const out = new Map();
|
|
234
284
|
if (xml == null) return out;
|
|
235
285
|
const fmt = new Map(); // abstractNumId -> numFmt of level 0
|
|
@@ -258,13 +308,13 @@ function listKinds(xml) {
|
|
|
258
308
|
num = null;
|
|
259
309
|
}
|
|
260
310
|
},
|
|
261
|
-
});
|
|
311
|
+
}, budget);
|
|
262
312
|
} catch {} // an unreadable numbering part: every list is "other", which is reported, never set
|
|
263
313
|
return out;
|
|
264
314
|
}
|
|
265
315
|
|
|
266
316
|
/** w:styleId -> the style's display name ("heading 2" is shown as "Heading 2"); empty when styles.xml is missing or unreadable. */
|
|
267
|
-
function styleNames(xml) {
|
|
317
|
+
function styleNames(xml, budget) {
|
|
268
318
|
const out = new Map();
|
|
269
319
|
if (xml == null) return out;
|
|
270
320
|
let id = null;
|
|
@@ -277,7 +327,7 @@ function styleNames(xml) {
|
|
|
277
327
|
onClose(name) {
|
|
278
328
|
if (name === "w:style") id = null;
|
|
279
329
|
},
|
|
280
|
-
});
|
|
330
|
+
}, budget);
|
|
281
331
|
} catch {} // a damaged styles part only costs the readable style names
|
|
282
332
|
return out;
|
|
283
333
|
}
|
|
@@ -287,15 +337,96 @@ const PARA_TEXT_KEEP = 2000;
|
|
|
287
337
|
|
|
288
338
|
// ---- engine location -> durable anchor ----------------------------------------------------------------------
|
|
289
339
|
|
|
340
|
+
// A paragraph's normalized text, kept per paragraph object: every comment resolved against the same reading reuses it.
|
|
341
|
+
const NORM = new WeakMap();
|
|
342
|
+
const norm = (p) => {
|
|
343
|
+
let v = NORM.get(p);
|
|
344
|
+
if (v === undefined) NORM.set(p, (v = normText(p.text)));
|
|
345
|
+
return v;
|
|
346
|
+
};
|
|
347
|
+
/** what norm(p) is about to read: the paragraph's text the first time (src/text.js weigh), nothing once it is kept (spent from the request's budget) */
|
|
348
|
+
const normCost = (p) => (NORM.has(p) ? 0 : weightOf(p));
|
|
349
|
+
// the byte tables a search is priced by (src/text.js hitsOf): of a paragraph's normalized text, and of its text as written
|
|
350
|
+
const NORM_HITS = new WeakMap();
|
|
351
|
+
const RAW_HITS = new WeakMap();
|
|
352
|
+
/** a native search for `q` in norm(p), which is already kept (src/text.js searchCost) */
|
|
353
|
+
const normSearch = (p, q) => searchCost(norm(p).length, q, hitsOf(NORM_HITS, p, norm(p), q));
|
|
354
|
+
|
|
355
|
+
// The paragraphs of one reading by w14:paraId and by text hash, in document order, built once: every anchor resolved against the reading
|
|
356
|
+
// looks its paragraph up instead of scanning them all.
|
|
357
|
+
const INDEX = new WeakMap();
|
|
358
|
+
function indexOf(paragraphs) {
|
|
359
|
+
let ix = INDEX.get(paragraphs);
|
|
360
|
+
if (ix) return ix;
|
|
361
|
+
ix = { byId: new Map(), byHash: new Map(), byPath: new Map(), byTop: new Map(), noPath: [], tops: null };
|
|
362
|
+
for (const p of paragraphs) {
|
|
363
|
+
if (p.path) (ix.byTop.get(p.path[0]) || ix.byTop.set(p.path[0], []).get(p.path[0])).push(p);
|
|
364
|
+
else ix.noPath.push(p);
|
|
365
|
+
if (p.paraId) (ix.byId.get(p.paraId) || ix.byId.set(p.paraId, []).get(p.paraId)).push(p);
|
|
366
|
+
(ix.byHash.get(p.hash) || ix.byHash.set(p.hash, []).get(p.hash)).push(p);
|
|
367
|
+
if (p.path && !ix.byPath.has(p.path.join(","))) ix.byPath.set(p.path.join(","), p); // the first, as a scan finds it
|
|
368
|
+
}
|
|
369
|
+
INDEX.set(paragraphs, ix);
|
|
370
|
+
return ix;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
// The paragraphs in the order nearest() ranks them, one level of equal distance from the body block `want` at a time (document order
|
|
374
|
+
// within a level), those without a body position last: the first level holding a match holds the match a scan of all of them picks.
|
|
375
|
+
function* byDistance(paragraphs, ix, want) {
|
|
376
|
+
if (!Number.isSafeInteger(want) || !ix.byTop.size) return yield paragraphs;
|
|
377
|
+
// the body blocks that hold paragraphs, outward from `want` (a body of many tables and few paragraphs has gaps: they are not walked)
|
|
378
|
+
const tops = (ix.tops ||= [...ix.byTop.keys()].sort((a, b) => a - b));
|
|
379
|
+
let hi = 0;
|
|
380
|
+
for (let lo = 0, top = tops.length; lo < top; ) {
|
|
381
|
+
const mid = (lo + top) >> 1;
|
|
382
|
+
if (tops[mid] < want) lo = mid + 1;
|
|
383
|
+
else top = mid;
|
|
384
|
+
hi = lo;
|
|
385
|
+
}
|
|
386
|
+
let lo = hi - 1;
|
|
387
|
+
while (lo >= 0 || hi < tops.length) {
|
|
388
|
+
const dl = lo >= 0 ? want - tops[lo] : Infinity;
|
|
389
|
+
const dh = hi < tops.length ? tops[hi] - want : Infinity;
|
|
390
|
+
const d = Math.min(dl, dh);
|
|
391
|
+
const a = dl === d ? ix.byTop.get(tops[lo--]) : null;
|
|
392
|
+
const b = dh === d ? ix.byTop.get(tops[hi++]) : null;
|
|
393
|
+
yield a && b ? a.concat(b) : a || b;
|
|
394
|
+
}
|
|
395
|
+
if (ix.noPath.length) yield ix.noPath;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
const BIGRAMS = new WeakMap();
|
|
399
|
+
const bigramsOf = (p) => {
|
|
400
|
+
let v = BIGRAMS.get(p);
|
|
401
|
+
if (!v) BIGRAMS.set(p, (v = bigramCodes(p.text)));
|
|
402
|
+
return v;
|
|
403
|
+
};
|
|
404
|
+
|
|
290
405
|
function locate(paragraphs, { path, paraId, quote, prefix, text, prevHash, nextHash, dupCount, allowPath = false }) {
|
|
291
|
-
|
|
406
|
+
// The page's quote and paragraph text (each can be a whole paragraph, per comment) are paid for before they are read: the text is
|
|
407
|
+
// normalized up to four times below (its hash, its length, the slot's similarity, its bigrams). Past the budget a step that needs it
|
|
408
|
+
// is skipped, and at Send the bare position still answers.
|
|
409
|
+
const q = normPaid(quote) ?? "";
|
|
410
|
+
if (text && !(affords(4 * text.length) && spend(4 * weigh(text)))) text = "";
|
|
292
411
|
const want = path ? path[0] : null;
|
|
293
412
|
const dist = (p) => (want == null || !p.path ? 1e6 : Math.abs(p.path[0] - want));
|
|
294
|
-
|
|
413
|
+
// the nearest of `list` (smaller distance, then smaller ordinal) in one pass, charged to the request's budget: null when it runs out
|
|
414
|
+
// (sorting a copy of 50,000 look-alikes for each of 500 anchors held the thread for 4 s, outside the budget)
|
|
415
|
+
const nearest = (list) => {
|
|
416
|
+
if (!spend(list.length * 3)) return null;
|
|
417
|
+
let best = null;
|
|
418
|
+
let bd = 0;
|
|
419
|
+
for (const p of list) {
|
|
420
|
+
const d = dist(p);
|
|
421
|
+
if (!best || d < bd || (d === bd && p.ordinal < best.ordinal)) (best = p), (bd = d);
|
|
422
|
+
}
|
|
423
|
+
return best;
|
|
424
|
+
};
|
|
295
425
|
const nb = (p) => (prevHash && paragraphs[p.ordinal - 1]?.hash === prevHash ? 1 : 0) + (nextHash && paragraphs[p.ordinal + 1]?.hash === nextHash ? 1 : 0);
|
|
426
|
+
const ix = indexOf(paragraphs);
|
|
296
427
|
// 1. w14:paraId
|
|
297
428
|
if (paraId) {
|
|
298
|
-
const hit =
|
|
429
|
+
const hit = ix.byId.get(paraId.toUpperCase()) || [];
|
|
299
430
|
if (hit.length === 1) return { p: hit[0], method: "id" };
|
|
300
431
|
}
|
|
301
432
|
// 2. identical text; among duplicates prefer the one whose neighbors are unchanged, then the nearest.
|
|
@@ -304,44 +435,110 @@ function locate(paragraphs, { path, paraId, quote, prefix, text, prevHash, nextH
|
|
|
304
435
|
let ambiguousHash = null;
|
|
305
436
|
if (text) {
|
|
306
437
|
const h = textHash(text);
|
|
307
|
-
const hits =
|
|
438
|
+
const hits = ix.byHash.get(h) || [];
|
|
308
439
|
const ambiguous = (dupCount != null ? dupCount > 1 : normText(text).length < 20) && (prevHash || nextHash);
|
|
309
|
-
|
|
310
|
-
|
|
440
|
+
// one pass over the hits for the best neighbor count, then the nearest of those (a document of 50,000 identical paragraphs and 500
|
|
441
|
+
// anchors held the thread for 4 s: the scans and the sort were outside the budget). Spent up front, at about what a hit costs here.
|
|
442
|
+
if (hits.length && spend(hits.length * 24)) {
|
|
443
|
+
let best = null;
|
|
444
|
+
let top = 0;
|
|
445
|
+
let bd = 0;
|
|
446
|
+
for (const p of hits) {
|
|
447
|
+
const n = nb(p);
|
|
448
|
+
const d = dist(p);
|
|
449
|
+
if (!best || n > top || (n === top && (d < bd || (d === bd && p.ordinal < best.ordinal)))) (best = p), (top = n), (bd = d);
|
|
450
|
+
}
|
|
451
|
+
if (!ambiguous || top > 0) return { p: best, method: "hash" };
|
|
452
|
+
ambiguousHash = h;
|
|
453
|
+
}
|
|
311
454
|
}
|
|
312
455
|
// 3. same slot: both neighbors unchanged and exactly one paragraph sits between them (edited in place)
|
|
313
456
|
if (prevHash && nextHash) {
|
|
314
|
-
const
|
|
315
|
-
|
|
457
|
+
const after = ix.byHash.get(prevHash) || [];
|
|
458
|
+
const slots = spend(after.length * 4) ? after.map((p) => paragraphs[p.ordinal + 1]).filter((p) => p && nb(p) === 2) : [];
|
|
459
|
+
// the slot's whole text is compared: within the request's budget (a slot of 8 million characters, 50 comments: 56 s before)
|
|
460
|
+
if (slots.length === 1 && (!text || (spend(2 * weightOf(slots[0]) + normCost(slots[0])) && (similarity(slots[0].text, text) >= 0.3 || (q && norm(slots[0]).includes(q)))))) return { p: slots[0], method: "neighbors" };
|
|
316
461
|
}
|
|
317
|
-
// 4
|
|
462
|
+
// steps 4 and 5 scan every paragraph: only within the budget of the anchors resolved together (src/text.js scanBudget), 16 units a
|
|
463
|
+
// paragraph, and the text each step searches character by character
|
|
464
|
+
// 4. exact quote. Ambiguous when the quote also occurs elsewhere and the prefix does not disambiguate: do not guess here. The
|
|
465
|
+
// paragraphs are read nearest first (byDistance): the answer is the nearest match, so the search stops at the first distance that
|
|
466
|
+
// has one (50 comments on a paragraph the agent changed, in 26,000 paragraphs, searched every paragraph 50 times).
|
|
467
|
+
let read = true;
|
|
318
468
|
if (q) {
|
|
319
|
-
const
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
469
|
+
const pre = normPaid(prefix) ?? "";
|
|
470
|
+
const longer = pre ? [pre + (pre.endsWith(" ") ? "" : " ") + q, pre + q] : null;
|
|
471
|
+
const hits = [];
|
|
472
|
+
const better = [];
|
|
473
|
+
for (const level of byDistance(paragraphs, ix, want)) {
|
|
474
|
+
for (const p of level) {
|
|
475
|
+
if (!(read = spend(16 + normCost(p)) && spend(normSearch(p, q)))) break; // a native substring search (src/text.js searchCost)
|
|
476
|
+
if (!norm(p).includes(q) || (ambiguousHash && p.hash === ambiguousHash)) continue; // look-alikes rejected in step 2 stay rejected
|
|
477
|
+
hits.push(p);
|
|
478
|
+
// two native substring searches per hit (src/text.js searchCost, as in the loop above)
|
|
479
|
+
if (longer && spend(2 * normSearch(p, longer[1])) && (norm(p).includes(longer[0]) || norm(p).includes(longer[1]))) better.push(p);
|
|
480
|
+
}
|
|
481
|
+
if (!read) break;
|
|
482
|
+
const found = better.length ? better : !pre && !text ? hits : [];
|
|
483
|
+
if (found.length) {
|
|
484
|
+
const p = nearest(found);
|
|
485
|
+
if (p) return { p, method: "quote" };
|
|
486
|
+
}
|
|
325
487
|
}
|
|
326
|
-
|
|
488
|
+
if (read && hits.length === 1) {
|
|
489
|
+
const p = nearest(hits);
|
|
490
|
+
if (p) return { p, method: "quote" };
|
|
491
|
+
}
|
|
492
|
+
} else read = spend(paragraphs.length * 16);
|
|
327
493
|
// 5. fuzzy: similarity to the remembered text (or the quote), neighbors count in favor
|
|
328
494
|
const probe = text || q;
|
|
329
|
-
|
|
495
|
+
let spent = !read;
|
|
496
|
+
if (probe && read) {
|
|
330
497
|
// The best candidate must clearly beat the runner-up: in templated text a deleted paragraph looks like its siblings,
|
|
331
498
|
// and "moved or removed" is a better answer than attaching the comment to the wrong one.
|
|
332
|
-
|
|
499
|
+
// the two best candidates only, found in one pass: the answer needs the winner and the runner-up, never the order of the rest
|
|
500
|
+
let best = null;
|
|
501
|
+
let second = null;
|
|
502
|
+
let count = 0;
|
|
503
|
+
const P = bigramCodes(text || q); // the probe's bigrams once; each paragraph's are kept with it (bigramsOf)
|
|
504
|
+
const n = P.codes.length;
|
|
333
505
|
for (const p of paragraphs) {
|
|
334
506
|
if (ambiguousHash && p.hash === ambiguousHash) continue; // rejected above: a look-alike elsewhere is not this paragraph
|
|
335
|
-
|
|
336
|
-
if (
|
|
507
|
+
let sim = 0;
|
|
508
|
+
if (text) {
|
|
509
|
+
if (!BIGRAMS.has(p) && !spend(weightOf(p))) {
|
|
510
|
+
spent = true; // its bigrams are read (and sorted) once per reading: the first time costs its text
|
|
511
|
+
break;
|
|
512
|
+
}
|
|
513
|
+
const B = bigramsOf(p);
|
|
514
|
+
// 0.6 needs the shorter at least 3/7 of the longer (Dice is at most 2 min / (m + n)): skip the others without comparing
|
|
515
|
+
const m = B.codes.length;
|
|
516
|
+
if (P.short === null && B.short === null && (Math.min(m, n) * 7 < Math.max(m, n) * 3)) continue;
|
|
517
|
+
if (!spend(m + n)) {
|
|
518
|
+
spent = true; // the budget is spent: "moved or removed" (src/text.js scanBudget), or the bare position at Send
|
|
519
|
+
break;
|
|
520
|
+
}
|
|
521
|
+
sim = diceOf(B, P);
|
|
522
|
+
} else if (norm(p).length) {
|
|
523
|
+
if (!spend(normCost(p) + q.length + 40 + n)) {
|
|
524
|
+
spent = true;
|
|
525
|
+
break;
|
|
526
|
+
}
|
|
527
|
+
sim = diceOf(bigramCodes(norm(p).slice(0, q.length + 40)), P);
|
|
528
|
+
}
|
|
529
|
+
if (sim >= 0.6) {
|
|
530
|
+
const c = { p, sim, score: sim + nb(p) * 0.1 - dist(p) * 1e-6 }; // distance only breaks ties
|
|
531
|
+
count++;
|
|
532
|
+
if (!best || c.score > best.score) (second = best), (best = c);
|
|
533
|
+
else if (!second || c.score > second.score) second = c;
|
|
534
|
+
}
|
|
337
535
|
}
|
|
338
|
-
|
|
339
|
-
if (scored.length && (scored.length === 1 || scored[0].score - scored[1].score >= 0.04)) return { p: scored[0].p, method: "fuzzy", score: Math.round(scored[0].sim * 100) / 100 };
|
|
536
|
+
if (!spent && best && (count === 1 || best.score - second.score >= 0.04)) return { p: best.p, method: "fuzzy", score: Math.round(best.sim * 100) / 100 };
|
|
340
537
|
}
|
|
341
538
|
// At Send time the browser's own index is as fresh as the bytes, so the bare position is acceptable.
|
|
342
539
|
// After a reload it never is (an insert above shifts it): resolveDurable does not pass allowPath.
|
|
343
540
|
if (allowPath && path) {
|
|
344
|
-
const hit =
|
|
541
|
+
const hit = ix.byPath.get(path.join(",")); // by lookup, not a scan: the bare position must still answer once the budget is spent
|
|
345
542
|
if (hit) return { p: hit, method: "path" };
|
|
346
543
|
}
|
|
347
544
|
return null;
|
|
@@ -358,42 +555,205 @@ export function durableFromEngine(paragraphs, loc, ctx = {}) {
|
|
|
358
555
|
return durableFor(paragraphs, hit.p, loc.blockPath ? loc.blockPath[0] : null, ctx, loc);
|
|
359
556
|
}
|
|
360
557
|
|
|
361
|
-
/**
|
|
362
|
-
|
|
558
|
+
/**
|
|
559
|
+
* Offsets of the non-overlapping occurrences of `q` in `text` (the first `max` of them), or null when the request's budget runs out: the
|
|
560
|
+
* search is counted as a native substring search (src/text.js searchCost) and each occurrence found at OCCURRENCE.
|
|
561
|
+
*/
|
|
562
|
+
// the complete list of a quote's occurrences in a paragraph, kept per paragraph of a reading for the last few quotes (rule: what is
|
|
563
|
+
// derived from the same paragraph is computed once per reading): 50 comments on one selection of a repeated text listed the same
|
|
564
|
+
// 395,000 occurrences 50 times. Ceiling: OCC_KEEP quotes per paragraph, the oldest dropped first.
|
|
565
|
+
const OCC = new WeakMap();
|
|
566
|
+
const OCC_KEEP = 16;
|
|
567
|
+
function occurrencesIn(p, q) {
|
|
568
|
+
let m = OCC.get(p);
|
|
569
|
+
const hit = m?.get(q);
|
|
570
|
+
if (hit) return hit;
|
|
571
|
+
const occ = occurrences(p, q);
|
|
572
|
+
if (occ) {
|
|
573
|
+
if (!m) OCC.set(p, (m = new Map()));
|
|
574
|
+
if (m.size >= OCC_KEEP) m.delete(m.keys().next().value);
|
|
575
|
+
m.set(q, occ);
|
|
576
|
+
}
|
|
577
|
+
return occ;
|
|
578
|
+
}
|
|
579
|
+
function occurrences(p, q, max = Infinity) {
|
|
580
|
+
const { text } = p;
|
|
581
|
+
if (!spend(searchCost(text.length, q, hitsOf(RAW_HITS, p, text, q)))) return null;
|
|
363
582
|
const out = [];
|
|
364
|
-
for (let i = text.indexOf(q); i >= 0; i = text.indexOf(q, i + q.length))
|
|
583
|
+
for (let i = text.indexOf(q); i >= 0 && out.length < max; i = text.indexOf(q, i + q.length)) {
|
|
584
|
+
if (!spend(OCCURRENCE)) return null;
|
|
585
|
+
out.push(i);
|
|
586
|
+
}
|
|
365
587
|
return out;
|
|
366
588
|
}
|
|
367
589
|
|
|
590
|
+
/**
|
|
591
|
+
* normText(text).slice(0, n), reading only as much of the text as that needs (windows cut as endsWithNorm cuts them), each character read
|
|
592
|
+
* spent from the budget; null past it. A comment's paragraph text is kept to 2,000 characters, and normalizing a whole paragraph of
|
|
593
|
+
* 8 million characters for each of 50 comments held the thread for seconds.
|
|
594
|
+
*/
|
|
595
|
+
function normHead(text, n) {
|
|
596
|
+
for (let size = 2 * n + 8; ; size *= 2) {
|
|
597
|
+
const e = cutNear(text, Math.min(text.length, size), 1);
|
|
598
|
+
if (e === null || !spend(weigh(text.slice(0, e)))) return null;
|
|
599
|
+
const head = normText(text.slice(0, e));
|
|
600
|
+
if (e === text.length || head.length > n) return head.slice(0, n);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
// normText(text.slice(0, at)).endsWith(prefix) and normText(text.slice(from)).startsWith(suffix), reading only as much of the text as
|
|
605
|
+
// the prefix or suffix needs. A window is cut only where nothing on one side changes how the other side normalizes (`cuttable`), so the
|
|
606
|
+
// window normalizes to exactly the end (or the start) of what the whole text normalizes to, give or take one space where it was cut.
|
|
607
|
+
// Once the window holds at least as many normalized characters as the prefix, it answers for the whole text; else it doubles.
|
|
608
|
+
// Every character read (the walk to a cut and the window normalized) is spent from the budget of the request (src/text.js): null when it
|
|
609
|
+
// runs out. An outside review: CJK text has no character below U+0300, every window grew to the whole paragraph, 3.65 s for one anchor.
|
|
610
|
+
/** The nearest place at or before (dir -1) or after (dir 1) i where a window may be cut, each character walked spent; null past the budget. */
|
|
611
|
+
function cutNear(text, i, dir) {
|
|
612
|
+
let n = 0;
|
|
613
|
+
while (!cuttable(text, i)) {
|
|
614
|
+
i += dir;
|
|
615
|
+
if (++n % 1024 === 0 && !spend(1024)) return null;
|
|
616
|
+
}
|
|
617
|
+
return spend(n % 1024) ? i : null;
|
|
618
|
+
}
|
|
619
|
+
export function endsWithNorm(text, at, prefix) {
|
|
620
|
+
for (let size = 2 * prefix.length + 8; ; size *= 2) {
|
|
621
|
+
const w = cutNear(text, Math.max(0, at - size), -1);
|
|
622
|
+
if (w === null || !spend(weigh(text.slice(w, at)))) return null;
|
|
623
|
+
const n = normText(text.slice(w, at));
|
|
624
|
+
if (w === 0 || n.length >= prefix.length) return n.endsWith(prefix);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
export function startsWithNorm(text, from, suffix) {
|
|
628
|
+
for (let size = 2 * suffix.length + 8; ; size *= 2) {
|
|
629
|
+
const e = cutNear(text, Math.min(text.length, from + size), 1);
|
|
630
|
+
if (e === null || !spend(weigh(text.slice(from, e)))) return null;
|
|
631
|
+
const n = normText(text.slice(from, e));
|
|
632
|
+
if (e === text.length || n.length >= suffix.length) return n.startsWith(suffix);
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
368
636
|
/**
|
|
369
637
|
* Which occurrence of a repeated quote a selection or a stored anchor means. Prefix and suffix are the normalized text around it;
|
|
370
|
-
* `nth` is the occurrence index remembered at capture. Returns the offset, or -1 when the
|
|
638
|
+
* `nth` is the occurrence index remembered at capture. Returns {at: the offset, k: which occurrence it is}, or {at: -1, k: null} when the
|
|
639
|
+
* text does not occur or the budget ran out.
|
|
371
640
|
*/
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
641
|
+
/** does the text around the occurrence at `at` (of `qLen` UTF-16 units) read as prefix and suffix (normalized, as an anchor keeps them)? null: the budget ran out */
|
|
642
|
+
function fitsAt(text, qLen, at, { prefix = "", suffix = "" }) {
|
|
643
|
+
prefix = String(prefix || "").slice(-CTX); // what an anchor keeps (durableFor); a longer one from elsewhere is read as its last CTX
|
|
644
|
+
suffix = String(suffix || "").slice(0, CTX);
|
|
645
|
+
const pre = !prefix || endsWithNorm(text, at, prefix);
|
|
646
|
+
if (pre === null) return null;
|
|
647
|
+
const suf = pre && (!suffix || startsWithNorm(text, at + qLen, suffix));
|
|
648
|
+
return suf === null ? null : !!(pre && suf);
|
|
649
|
+
}
|
|
650
|
+
function pickOccurrence(p, q, opts = {}) {
|
|
651
|
+
const text = p.text;
|
|
652
|
+
// the remembered occurrence first, listing only the occurrences up to it: when its surroundings still fit, it is the answer every rule
|
|
653
|
+
// of pickFrom gives, and the others need not be listed or read (an outside review: 300,000 "a" with nth 150,000 lost its offsets
|
|
654
|
+
// under the budget, though the remembered one still fit)
|
|
655
|
+
const { nth = null } = opts;
|
|
656
|
+
// only an index of the occurrences is one: a selection that starts between two of them ("aa" at offset 1 of "aaaa", which the
|
|
657
|
+
// non-overlapping search does not list) was stored with nth -1, and occ[-1] (undefined) made the window readers loop forever on the
|
|
658
|
+
// server's thread, the budget never asked (a window of no characters costs nothing)
|
|
659
|
+
const remembered = Number.isInteger(nth) && nth >= 0 ? nth : null;
|
|
660
|
+
if (remembered !== null && (opts.prefix || opts.suffix)) {
|
|
661
|
+
const head = occurrences(p, q, remembered + 1);
|
|
662
|
+
if (head === null) return { at: -1, k: null };
|
|
663
|
+
if (head.length === remembered + 1) {
|
|
664
|
+
const ok = fitsAt(text, q.length, head[remembered], opts);
|
|
665
|
+
if (ok === null) return { at: -1, k: null };
|
|
666
|
+
if (ok) return { at: head[remembered], k: remembered };
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
const occ = occurrencesIn(p, q);
|
|
670
|
+
const at = occ?.length ? pickFrom(text, q.length, occ, { ...opts, nth: remembered }) : -1;
|
|
671
|
+
return at < 0 ? { at: -1, k: null } : { at, k: occ.indexOf(at) };
|
|
672
|
+
}
|
|
673
|
+
function pickNormOccurrence(p, q, opts = {}) {
|
|
674
|
+
const occ = normOccurrences(p, q);
|
|
675
|
+
if (!occ?.length) return { at: -1, len: 0, k: null };
|
|
676
|
+
let { nth = null } = opts;
|
|
677
|
+
if (!(Number.isInteger(nth) && nth >= 0 && nth < occ.length)) nth = null;
|
|
678
|
+
const prefix = String(opts.prefix || "").slice(-CTX);
|
|
679
|
+
const suffix = String(opts.suffix || "").slice(0, CTX);
|
|
680
|
+
if ((prefix || suffix) && occ.length > 1 && !spend(occ.length * 2 * (prefix.length + suffix.length + 16))) return { at: -1, len: 0, k: null };
|
|
681
|
+
const guarded = [];
|
|
682
|
+
for (let i = 0; i < occ.length; i++) {
|
|
683
|
+
const o = occ[i];
|
|
684
|
+
const ok = fitsAt(p.text, o.end - o.start, o.start, { prefix, suffix });
|
|
685
|
+
if (ok === null) return { at: -1, len: 0, k: null };
|
|
686
|
+
if (ok) guarded.push(i);
|
|
687
|
+
}
|
|
688
|
+
let idx;
|
|
689
|
+
if (guarded.length === 1) idx = guarded[0];
|
|
690
|
+
else if (nth != null && guarded.includes(nth)) idx = nth;
|
|
691
|
+
else if (Number.isInteger(opts.hintStart) && guarded.length) {
|
|
692
|
+
idx = guarded.reduce((best, i) => (Math.abs(occ[i].start - opts.hintStart) < Math.abs(occ[best].start - opts.hintStart) ? i : best), guarded[0]);
|
|
693
|
+
} else if (guarded.length) idx = guarded[0];
|
|
694
|
+
else idx = nth != null ? nth : 0;
|
|
695
|
+
const hit = occ[idx];
|
|
696
|
+
return { at: hit.start, len: hit.end - hit.start, k: idx };
|
|
697
|
+
}
|
|
698
|
+
function pickFrom(text, qLen, occ, { prefix = "", suffix = "", nth = null, hintStart = null }) {
|
|
699
|
+
if (!(Number.isInteger(nth) && nth >= 0 && nth < occ.length)) nth = null;
|
|
700
|
+
prefix = String(prefix || "").slice(-CTX);
|
|
701
|
+
suffix = String(suffix || "").slice(0, CTX);
|
|
702
|
+
const fits = (at) => fitsAt(text, qLen, at, { prefix, suffix });
|
|
703
|
+
// each occurrence reads about CTX characters on each side, out of the budget of the request (src/text.js): a paragraph of 100,000
|
|
704
|
+
// "a" read the whole text before every one of them and took 10 seconds in an outside review. Past the budget, no offset.
|
|
705
|
+
if ((prefix || suffix) && occ.length > 1 && !spend(occ.length * 2 * (prefix.length + suffix.length + 16))) return -1;
|
|
706
|
+
const guarded = [];
|
|
707
|
+
for (const at of occ) {
|
|
708
|
+
const ok = fits(at);
|
|
709
|
+
if (ok === null) return -1;
|
|
710
|
+
if (ok) guarded.push(at);
|
|
711
|
+
}
|
|
376
712
|
if (guarded.length === 1) return guarded[0];
|
|
377
713
|
if (nth != null && nth < occ.length && guarded.includes(occ[nth])) return occ[nth];
|
|
714
|
+
if (Number.isInteger(hintStart) && guarded.length) {
|
|
715
|
+
return guarded.reduce((best, at) => (Math.abs(at - hintStart) < Math.abs(best - hintStart) ? at : best), guarded[0]);
|
|
716
|
+
}
|
|
378
717
|
if (guarded.length) return guarded[0];
|
|
379
718
|
return nth != null && nth < occ.length ? occ[nth] : occ[0];
|
|
380
719
|
}
|
|
381
720
|
|
|
382
721
|
function durableFor(paragraphs, p, engineTop, ctx, loc = {}) {
|
|
383
|
-
|
|
384
|
-
const q =
|
|
385
|
-
const prefix =
|
|
386
|
-
const suffix =
|
|
722
|
+
// the page's quote, prefix and suffix, paid for before they are normalized; past the budget the anchor is the paragraph alone
|
|
723
|
+
const q = normPaid(ctx.quote) ?? "";
|
|
724
|
+
const prefix = (normPaid(ctx.prefix) ?? "").slice(-CTX);
|
|
725
|
+
const suffix = (normPaid(ctx.suffix) ?? "").slice(0, CTX);
|
|
387
726
|
let start = null;
|
|
388
727
|
let end = null;
|
|
389
728
|
let nth = null;
|
|
390
729
|
if (q) {
|
|
391
730
|
// the engine's own offsets when they point at the quote; else the occurrence whose surroundings match prefix and suffix
|
|
392
|
-
const
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
731
|
+
const hintStart = Number.isInteger(loc.start) ? loc.start : null;
|
|
732
|
+
const preferNorm = (ctx.quote !== q && ctx.quote.trim() !== q) || (q.includes(" ") && HAS_NON_PLAIN_WS.test(p.text));
|
|
733
|
+
const own = hintStart !== null && Number.isInteger(loc.end) && p.text.slice(loc.start, loc.end) === ctx.quote;
|
|
734
|
+
let picked;
|
|
735
|
+
if (own) {
|
|
736
|
+
let k = -1;
|
|
737
|
+
if (preferNorm) {
|
|
738
|
+
const nocc = normOccurrences(p, q);
|
|
739
|
+
if (nocc) {
|
|
740
|
+
k = nocc.findIndex((o) => o.start === loc.start && o.end === loc.end);
|
|
741
|
+
if (k < 0) k = nocc.findIndex((o) => o.start >= loc.start && o.end <= loc.end);
|
|
742
|
+
}
|
|
743
|
+
} else {
|
|
744
|
+
k = occurrencesIn(p, ctx.quote)?.indexOf(loc.start) ?? -1;
|
|
745
|
+
}
|
|
746
|
+
picked = { at: loc.start, k: k >= 0 ? k : null, len: ctx.quote.length };
|
|
747
|
+
} else if (preferNorm) {
|
|
748
|
+
picked = pickNormOccurrence(p, q, { prefix, suffix, hintStart });
|
|
749
|
+
} else {
|
|
750
|
+
picked = { ...pickOccurrence(p, ctx.quote, { prefix, suffix, hintStart }), len: ctx.quote.length };
|
|
751
|
+
if (picked.at < 0) picked = pickNormOccurrence(p, q, { prefix, suffix, hintStart });
|
|
752
|
+
}
|
|
753
|
+
if (picked.at >= 0) {
|
|
754
|
+
start = picked.at;
|
|
755
|
+
end = picked.at + picked.len;
|
|
756
|
+
nth = picked.k;
|
|
397
757
|
}
|
|
398
758
|
}
|
|
399
759
|
return {
|
|
@@ -407,8 +767,8 @@ function durableFor(paragraphs, p, engineTop, ctx, loc = {}) {
|
|
|
407
767
|
textHash: p.hash,
|
|
408
768
|
prevHash: paragraphs[p.ordinal - 1]?.hash || null,
|
|
409
769
|
nextHash: paragraphs[p.ordinal + 1]?.hash || null,
|
|
410
|
-
dupCount: paragraphs.
|
|
411
|
-
paraText:
|
|
770
|
+
dupCount: indexOf(paragraphs).byHash.get(p.hash)?.length ?? 0,
|
|
771
|
+
paraText: normHead(p.text, PARA_TEXT_KEEP) ?? "",
|
|
412
772
|
quote: q,
|
|
413
773
|
prefix,
|
|
414
774
|
suffix,
|
|
@@ -429,10 +789,27 @@ export function resolveDurable(paragraphs, d) {
|
|
|
429
789
|
const q = d.quote || "";
|
|
430
790
|
let start = null;
|
|
431
791
|
let end = null;
|
|
432
|
-
const
|
|
792
|
+
const ctxOpts = { prefix: d.prefix ?? d.before ?? "", suffix: d.suffix ?? d.after ?? "", nth: d.nth ?? d.occ ?? null };
|
|
793
|
+
const preferNorm = !!q && q === normText(q) && q.includes(" ") && HAS_NON_PLAIN_WS.test(p.text);
|
|
794
|
+
let at = -1;
|
|
795
|
+
let len = q.length;
|
|
796
|
+
let k = null;
|
|
797
|
+
if (q) {
|
|
798
|
+
if (preferNorm) {
|
|
799
|
+
const picked = pickNormOccurrence(p, q, ctxOpts);
|
|
800
|
+
if (picked.at >= 0) ({ at, len, k } = picked);
|
|
801
|
+
else ({ at, k } = pickOccurrence(p, q, ctxOpts));
|
|
802
|
+
} else {
|
|
803
|
+
({ at, k } = pickOccurrence(p, q, ctxOpts));
|
|
804
|
+
if (at < 0 || (Number.isInteger(ctxOpts.nth) && ctxOpts.nth > 0 && k !== ctxOpts.nth)) {
|
|
805
|
+
const picked = pickNormOccurrence(p, q, ctxOpts);
|
|
806
|
+
if (picked.at >= 0) ({ at, len, k } = picked);
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
}
|
|
433
810
|
if (at >= 0) {
|
|
434
811
|
start = at;
|
|
435
|
-
end = at +
|
|
812
|
+
end = at + len;
|
|
436
813
|
}
|
|
437
|
-
return { method: hit.method, score: hit.score, ordinal: p.ordinal, path: p.path, tableNo: p.tableNo ?? null, paraId: p.paraId, textHash: p.hash, protected: !!p.protected, ...(p.protected ? { protectInfo: protectInfo(p) } : {}), dupCount: paragraphs.
|
|
814
|
+
return { method: hit.method, score: hit.score, ordinal: p.ordinal, path: p.path, tableNo: p.tableNo ?? null, paraId: p.paraId, textHash: p.hash, protected: !!p.protected, ...(p.protected ? { protectInfo: protectInfo(p) } : {}), dupCount: indexOf(paragraphs).byHash.get(p.hash)?.length ?? 0, nth: at >= 0 ? k : null, prevHash: paragraphs[p.ordinal - 1]?.hash || null, nextHash: paragraphs[p.ordinal + 1]?.hash || null, text: normHead(p.text, PARA_TEXT_KEEP) ?? "", start, end, engineDelta: d.engineDelta || 0 };
|
|
438
815
|
}
|