onionskin-cli 0.0.3 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CHANGELOG.md +198 -0
  2. package/INSTALL.md +5 -1
  3. package/NOTICE +34 -0
  4. package/README.md +102 -34
  5. package/docs/contracts/batch.schema.json +27 -4
  6. package/docs/contracts/ops.schema.json +117 -5
  7. package/package.json +4 -2
  8. package/scripts/checker/check.py +24 -5
  9. package/scripts/checker/requirements.txt +482 -5
  10. package/scripts/requirements.txt +482 -5
  11. package/scripts/rewriter/apple/main.swift +1 -1
  12. package/scripts/rewriter/apple/onionskin-fm +0 -0
  13. package/scripts/rewriter/apple/onionskin-fm.json +1 -1
  14. package/skills/onionskin/SKILL.md +12 -9
  15. package/skills/onionskin/reference/batch.md +5 -3
  16. package/skills/onionskin/reference/formats.md +5 -1
  17. package/skills/onionskin/reference/ops.md +48 -4
  18. package/skills/onionskin/reference/troubleshooting.md +31 -0
  19. package/src/agent-detect.js +28 -2
  20. package/src/agent-prompt.js +12 -1
  21. package/src/agent-runner.js +110 -20
  22. package/src/anchor-work.js +154 -0
  23. package/src/anchor-worker.js +38 -0
  24. package/src/anchors.js +61 -11
  25. package/src/apply.js +119 -47
  26. package/src/batch.js +32 -8
  27. package/src/check-worker.js +8 -5
  28. package/src/checker.js +238 -33
  29. package/src/cli.js +248 -42
  30. package/src/constants.js +34 -1
  31. package/src/diff.js +3 -1
  32. package/src/doctor.js +12 -3
  33. package/src/docx.js +464 -87
  34. package/src/edit-hash.js +19 -0
  35. package/src/fsutil.js +14 -6
  36. package/src/history-worker.js +8 -8
  37. package/src/history.js +144 -31
  38. package/src/html-active.js +384 -0
  39. package/src/html-envelope.js +51 -0
  40. package/src/html-parser.js +8 -0
  41. package/src/html-render-worker.js +25 -0
  42. package/src/html-render.js +939 -0
  43. package/src/html-utf8.js +65 -0
  44. package/src/html-write-worker.js +14 -0
  45. package/src/html-write.js +380 -0
  46. package/src/html.js +1259 -0
  47. package/src/local.js +68 -4
  48. package/src/merge-package.js +1166 -0
  49. package/src/merge-pptx.js +98 -38
  50. package/src/merge-proof.js +1019 -0
  51. package/src/merge-worker.js +60 -0
  52. package/src/merge-xlsx.js +84 -29
  53. package/src/merge.js +286 -83
  54. package/src/nfc.js +46 -0
  55. package/src/notes.js +122 -0
  56. package/src/offload.js +419 -1
  57. package/src/open-worker.js +27 -0
  58. package/src/ops/common.js +2 -1
  59. package/src/ops/docx-table.js +5 -3
  60. package/src/ops/docx.js +58 -29
  61. package/src/ops/html.js +394 -0
  62. package/src/ops/pptx.js +82 -23
  63. package/src/ops/run.js +23 -3
  64. package/src/ops/runs.js +215 -13
  65. package/src/ops/schema.js +8 -1
  66. package/src/ops/xlsx.js +121 -10
  67. package/src/ops-worker.js +17 -0
  68. package/src/package.js +43 -4
  69. package/src/person.js +2 -1
  70. package/src/pptx.js +323 -23
  71. package/src/question.js +10 -3
  72. package/src/quick-plan.js +13 -6
  73. package/src/quick.js +1 -1
  74. package/src/redline-check.js +12 -2
  75. package/src/redline-worker.js +10 -14
  76. package/src/reply.js +46 -19
  77. package/src/retain.js +223 -0
  78. package/src/review.js +32 -18
  79. package/src/rewrite-check.js +2 -2
  80. package/src/rewriter.js +38 -8
  81. package/src/sanitize.js +16 -3
  82. package/src/server-entry.js +4 -3
  83. package/src/server.js +175 -62
  84. package/src/session.js +1151 -210
  85. package/src/setup.js +20 -2
  86. package/src/stop-gate.js +34 -0
  87. package/src/suggest-reply.js +42 -7
  88. package/src/suggest.js +14 -29
  89. package/src/text.js +161 -11
  90. package/src/update.js +517 -0
  91. package/src/voice-guide.js +15 -9
  92. package/src/voice-learn.js +15 -4
  93. package/src/voice-sample-worker.js +7 -6
  94. package/src/voice-store.js +11 -4
  95. package/src/voice-worker.js +4 -6
  96. package/src/voice.js +2 -1
  97. package/src/worker-host.js +23 -0
  98. package/src/xlsx.js +378 -97
  99. package/src/xml-tree.js +6 -3
  100. package/src/xml.js +82 -34
  101. package/src/zip-write.js +10 -5
  102. package/src/zip.js +73 -18
  103. package/vendor/parse5/LICENSE.txt +32 -0
  104. package/vendor/parse5/parse5.mjs +7822 -0
  105. package/vendor/parse5.lock.json +23 -0
  106. package/web/dist/assets/{index-DS-WST1E.js → index-CjsTf5gj.js} +58 -52
  107. package/web/dist/assets/index-DnbLtHhG.css +1 -0
  108. package/web/dist/index.html +2 -2
  109. package/web/dist/assets/index-BCAEKiOG.css +0 -1
package/src/docx.js CHANGED
@@ -1,9 +1,12 @@
1
1
  import { pathToFileURL } from "node:url";
2
2
  import { engineNodePath } from "./ops/engine.js";
3
- import { walkXml } from "./xml.js";
4
- import { normText, similarity, textHash } from "./text.js";
3
+ import { readBudget, tooMany, walkXml } from "./xml.js";
4
+ import { OCCURRENCE, affords, bigramCodes, diceOf, hitsOf, normPaid, normText, searchCost, similarity, spend, textHash, weigh, weightOf } from "./text.js";
5
5
  import { openZip } from "./zip.js";
6
6
  import { summarizeRuns } from "./fmt.js";
7
+ import { HAS_NON_PLAIN_WS, cuttable, normOccurrences } from "./pptx.js";
8
+
9
+ export { cuttable };
7
10
 
8
11
  // One classifier for `protected`: the engine's own (default deny), bundled for Node. The batch, `apply` and the editor agree because they ask the same code.
9
12
  const { protectReasons } = await import(pathToFileURL(engineNodePath()).href);
@@ -43,6 +46,16 @@ function runSet(name, a) {
43
46
  if (name === "w:vertAlign") return only(a, "w:val") && (v === "superscript" || v === "subscript") ? ["va", v.slice(0, 3)] : null;
44
47
  return null;
45
48
  }
49
+ // What one reading of a Word document may hold. The server reads a document on its own thread at Send, at each delivery to an agent, for a
50
+ // queued comment and at the end of a run, so these bound how long one such reading can keep it from answering anything else (the outside
51
+ // review: a part of 1 million runs held it 4.3 s, a run of 3 million attributes 2.4 s). A real report of about 1,000 pages has about 16,000
52
+ // paragraphs, 240,000 elements, 350,000 attributes and 3 million characters (measured: read in about 240 ms here).
53
+ export const MAX_PARAGRAPHS = 50_000;
54
+ export const MAX_ELEMENTS = 400_000;
55
+ export const MAX_ATTRIBUTES = 800_000;
56
+ export const MAX_TEXT = 8_000_000;
57
+ /** One reading's budget (src/xml.js readBudget), shared by the parts it walks. */
58
+ export const docxBudget = () => readBudget({ elements: MAX_ELEMENTS, attrs: MAX_ATTRIBUTES, text: MAX_TEXT });
46
59
  const PPR_OWN = new Set(["w:pStyle", "w:jc", "w:numPr", "w:rPr", "w:sectPr"]); // read on their own, or not paragraph formatting the user sets
47
60
  const RANGE = /^(.+)(Start|End)$/;
48
61
 
@@ -55,7 +68,7 @@ const RANGE = /^(.+)(Start|End)$/;
55
68
  * (null when they do not add up to it); `list` {numId, ilvl} from its own w:numPr, else null; `ppr`, its other paragraph properties as
56
69
  * written; `sect`, it ends a section; `loose`, a bookmark, comment or other range starts or ends inside it without its other end.
57
70
  */
58
- export function parseDocxParagraphs(xml, { detail = false } = {}) {
71
+ export function parseDocxParagraphs(xml, { detail = false, budget = docxBudget() } = {}) {
59
72
  const paragraphs = [];
60
73
  const stack = []; // {name, ...}
61
74
  let bodyIndex = -1;
@@ -64,7 +77,9 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
64
77
  let tableCount = 0; // 1-based order among top-level tables (the label users read), unlike body_path[0] which counts every body item
65
78
  let cellK = 0;
66
79
  let cur = null;
67
- let delDepth = 0;
80
+ const revStack = [];
81
+ let mcFallbackDepth = 0;
82
+ const visible = () => !revStack.includes("del") && (revStack.length === 0 || revStack[revStack.length - 1] !== "moveFrom");
68
83
  let inT = false;
69
84
  let inInstr = false;
70
85
  let inBody = false;
@@ -77,9 +92,14 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
77
92
  walkXml(xml, {
78
93
  onOpen(name, attrs) {
79
94
  stack.push(name);
95
+ if (name === "mc:Fallback") {
96
+ mcFallbackDepth++;
97
+ return;
98
+ }
99
+ if (mcFallbackDepth > 0) return;
80
100
  if (name === "w:body") inBody = true;
81
101
  if (!inBody) return;
82
- if (name === "w:tbl") {
102
+ if (name === "w:tbl" && pDepth === 0) {
83
103
  tblDepth++;
84
104
  if (tblDepth === 1) {
85
105
  bodyIndex++;
@@ -88,16 +108,15 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
88
108
  info.row = -1;
89
109
  info.col = -1;
90
110
  }
91
- } else if (name === "w:tr" && tblDepth === 1) {
111
+ } else if (name === "w:tr" && tblDepth === 1 && pDepth === 0) {
92
112
  info.row++;
93
113
  info.col = -1;
94
- } else if (name === "w:tc" && tblDepth === 1) {
114
+ } else if (name === "w:tc" && tblDepth === 1 && pDepth === 0) {
95
115
  info.col++;
96
116
  cellK = 0;
97
117
  } else if (name === "w:p") {
98
118
  pDepth++;
99
- const nested = stack.slice(0, -1).includes("w:p"); // a text box paragraph inside a paragraph
100
- if (nested) return;
119
+ if (pDepth > 1) return;
101
120
  let path = null;
102
121
  let tableNo = null;
103
122
  if (tblDepth === 0) {
@@ -107,18 +126,19 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
107
126
  path = [info.tableIndex, info.row, info.col, cellK++];
108
127
  tableNo = tableCount;
109
128
  }
129
+ if (paragraphs.length >= MAX_PARAGRAPHS) throw tooMany(budget, MAX_PARAGRAPHS, "paragraphs");
110
130
  cur = { ordinal: paragraphs.length, path, tableNo, paraId: /^[0-9A-Fa-f]{8}$/.test(attrs["w14:paraId"] || "") ? attrs["w14:paraId"].toUpperCase() : null, text: "", instr: "", protected: false, style: null, align: "", runs: [] };
111
131
  if (detail) (Object.assign(cur, { spans: [], list: null, ppr: [], sect: false, loose: false }), (marks = new Set()), (pprTop = null));
112
132
  paragraphs.push(cur);
113
- } else if (cur) {
133
+ } else if (cur && pDepth === 1) {
114
134
  if (detail) {
115
135
  const n = stack.length;
116
- if (pDepth === 1 && stack[n - 2] === "w:pPr" && stack[n - 3] === "w:p") {
136
+ if (stack[n - 2] === "w:pPr" && stack[n - 3] === "w:p") {
117
137
  pprTop = name;
118
138
  if (name === "w:sectPr") cur.sect = true;
119
139
  }
120
140
  if (pprTop && !PPR_OWN.has(pprTop)) cur.ppr.push(written(name, attrs));
121
- else if (pprTop === "w:numPr" && pDepth === 1 && stack[n - 2] === "w:numPr" && (name === "w:numId" || name === "w:ilvl")) (cur.list ||= { numId: "", ilvl: 0 })[name === "w:numId" ? "numId" : "ilvl"] = name === "w:numId" ? attrs["w:val"] || "" : Number(attrs["w:val"]) || 0;
141
+ else if (pprTop === "w:numPr" && stack[n - 2] === "w:numPr" && (name === "w:numId" || name === "w:ilvl")) (cur.list ||= { numId: "", ilvl: 0 })[name === "w:numId" ? "numId" : "ilvl"] = name === "w:numId" ? attrs["w:val"] || "" : Number(attrs["w:val"]) || 0;
122
142
  const mark = RANGE.exec(name);
123
143
  if (mark && "w:id" in attrs) {
124
144
  const key = `${mark[1]} ${attrs["w:id"]}`;
@@ -136,35 +156,33 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
136
156
  else if (name === "w:r" && !run) run = { text: false, p: {}, at: cur.text.length, depth: stack.length, set: {}, rest: [] };
137
157
  else if (name === "w:rPr") rprDepth++;
138
158
  else if (run && rprDepth === 1 && stack.includes("w:rPr")) runProp(run.p, name, attrs);
139
- if (name === "w:del") delDepth++;
140
- else if (name === "w:t" && delDepth === 0) (inT = true, run && (run.text = true));
159
+ if (name === "w:del") revStack.push("del");
160
+ else if (name === "w:moveFrom") revStack.push("moveFrom");
161
+ else if (name === "w:ins") revStack.push("ins");
162
+ else if (name === "w:t" && visible()) (inT = true, run && (run.text = true));
141
163
  else if (name === "w:instrText") inInstr = true;
142
- else if (name === "w:tab" && delDepth === 0) cur.text += "\t";
164
+ else if (name === "w:tab" && visible()) cur.text += "\t";
143
165
  else if (name === "w:br" || name === "w:cr") {
144
- if (delDepth === 0 && attrs["w:type"] !== "page" && attrs["w:type"] !== "column") cur.text += "\n";
166
+ if (visible() && attrs["w:type"] !== "page" && attrs["w:type"] !== "column") cur.text += "\n";
145
167
  } else if (name === "w:drawing" || name === "w:pict") {
146
- if (delDepth === 0) cur.text += OBJ;
168
+ if (visible()) cur.text += OBJ;
147
169
  }
148
170
  }
149
171
  },
150
172
  onText(t) {
151
- if (cur && inT) cur.text += t;
152
- else if (cur && inInstr && cur.instr.length < 200) cur.instr += t;
173
+ if (mcFallbackDepth > 0) return;
174
+ if (cur && pDepth === 1 && inT) cur.text += t;
175
+ else if (cur && pDepth === 1 && inInstr && cur.instr.length < 200) cur.instr += t;
153
176
  },
154
177
  onClose(name) {
155
178
  stack.pop();
156
- if (name === "w:t") inT = false;
157
- else if (name === "w:instrText") inInstr = false;
158
- else if (name === "w:del") delDepth = Math.max(0, delDepth - 1);
159
- else if (name === "w:tbl") tblDepth--;
160
- else if (name === "w:rPr") rprDepth = Math.max(0, rprDepth - 1);
161
- else if (name === "w:r" && run) {
162
- if (run.text && cur) cur.runs.push(run.p);
163
- if (detail && cur && cur.text.length > run.at) cur.spans.push({ n: cur.text.length - run.at, set: run.set, rest: run.rest.join(" ") });
164
- run = null;
165
- } else if (name === "w:p") {
179
+ if (mcFallbackDepth > 0) {
180
+ if (name === "mc:Fallback") mcFallbackDepth--;
181
+ return;
182
+ }
183
+ if (name === "w:p") {
166
184
  pDepth--;
167
- if (stack.includes("w:p")) return;
185
+ if (pDepth > 0) return;
168
186
  if (cur) (cur.fmt = summarizeRuns(cur.runs), delete cur.runs);
169
187
  if (cur && detail) {
170
188
  if (cur.spans.reduce((n, x) => n + x.n, 0) !== cur.text.length) cur.spans = null;
@@ -172,10 +190,23 @@ export function parseDocxParagraphs(xml, { detail = false } = {}) {
172
190
  cur.loose = marks.size > 0;
173
191
  }
174
192
  cur = null;
193
+ revStack.length = 0;
194
+ } else if (name === "w:tbl" && pDepth === 0) {
195
+ tblDepth--;
196
+ } else if (cur && pDepth === 1) {
197
+ if (name === "w:t") inT = false;
198
+ else if (name === "w:instrText") inInstr = false;
199
+ else if (name === "w:del" || name === "w:moveFrom" || name === "w:ins") revStack.pop();
200
+ else if (name === "w:rPr") rprDepth = Math.max(0, rprDepth - 1);
201
+ else if (name === "w:r" && run) {
202
+ if (run.text && cur) cur.runs.push(run.p);
203
+ if (detail && cur && cur.text.length > run.at) cur.spans.push({ n: cur.text.length - run.at, set: run.set, rest: run.rest.join(" ") });
204
+ run = null;
205
+ }
206
+ if (detail && name === pprTop && stack[stack.length - 1] === "w:pPr") pprTop = null;
175
207
  }
176
- if (detail && name === pprTop && stack[stack.length - 1] === "w:pPr") pprTop = null;
177
208
  },
178
- });
209
+ }, budget);
179
210
  for (const p of paragraphs) p.hash = textHash(p.text);
180
211
  const reasons = protectReasons(xml);
181
212
  // the engine lists paragraphs the same way; if the two ever disagree on the count, nothing can be trusted as editable
@@ -204,10 +235,28 @@ export function protectInfo(p) {
204
235
  }
205
236
 
206
237
  const HEADING = /^(heading\s?\d|title|subtitle)$/i;
207
- /** The text of the nearest heading above a paragraph (by paragraph style), or "" when there is none. */
238
+ // the nearest heading at or before each ordinal, found in one pass per reading: a scan up from every resolved comment was a pass over
239
+ // the paragraphs per comment, and the heading's text was normalized whole again each time, outside the budget
240
+ const HEADS = new WeakMap();
241
+ /** The text of the nearest heading above a paragraph (by paragraph style), or "" when there is none (or the budget cannot pay for it). */
208
242
  export function headingBefore(paragraphs, ordinal) {
209
- for (let i = ordinal - 1; i >= 0; i--) if (paragraphs[i].style && HEADING.test(paragraphs[i].style) && paragraphs[i].text.trim()) return normText(paragraphs[i].text).slice(0, 80);
210
- return "";
243
+ if (!Number.isInteger(ordinal) || ordinal <= 0) return "";
244
+ let at = HEADS.get(paragraphs);
245
+ if (!at) {
246
+ at = new Int32Array(paragraphs.length + 1);
247
+ let last = -1;
248
+ for (let i = 0; i < paragraphs.length; i++) {
249
+ at[i] = last; // the heading strictly before paragraph i
250
+ const p = paragraphs[i];
251
+ if (p.style && HEADING.test(p.style) && p.text.trim()) last = i;
252
+ }
253
+ at[paragraphs.length] = last;
254
+ HEADS.set(paragraphs, at);
255
+ }
256
+ const h = at[Math.min(ordinal, paragraphs.length)];
257
+ if (h < 0) return "";
258
+ const p = paragraphs[h];
259
+ return spend(normCost(p)) ? norm(p).slice(0, 80) : "";
211
260
  }
212
261
 
213
262
  /** `detail` (the merge only): formatting per run and the paragraph's list, see parseDocxParagraphs; `list.kind` is "bullet", "number" or "other". */
@@ -215,11 +264,12 @@ export function readDocxParagraphs(bytes, { detail = false } = {}) {
215
264
  const zip = openZip(bytes);
216
265
  const xml = zip.readText("word/document.xml");
217
266
  if (xml == null) throw new Error("word/document.xml missing");
218
- const paragraphs = parseDocxParagraphs(xml, { detail });
219
- const names = styleNames(zip.readText("word/styles.xml"));
267
+ const budget = docxBudget(); // the main part, the styles and the numbering of one reading share it
268
+ const paragraphs = parseDocxParagraphs(xml, { detail, budget });
269
+ const names = styleNames(zip.readText("word/styles.xml"), budget);
220
270
  if (names.size) for (const p of paragraphs) if (p.style && names.has(p.style)) p.styleName = names.get(p.style);
221
271
  if (detail && paragraphs.some((p) => p.list)) {
222
- const kinds = listKinds(zip.readText("word/numbering.xml"));
272
+ const kinds = listKinds(zip.readText("word/numbering.xml"), budget);
223
273
  for (const p of paragraphs) if (p.list) p.list = p.list.numId && p.list.numId !== "0" ? { ...p.list, kind: kinds.get(p.list.numId) || "other" } : null; // numId 0 switches a style's list off
224
274
  }
225
275
  return paragraphs;
@@ -229,7 +279,7 @@ export function readDocxParagraphs(bytes, { detail = false } = {}) {
229
279
  * w:numId -> "bullet" | "number" | "other", by the first level of its definition: the two kinds the editor's list buttons make and
230
280
  * the engine can set again. A list with level overrides, or one this cannot read, is "other".
231
281
  */
232
- function listKinds(xml) {
282
+ function listKinds(xml, budget) {
233
283
  const out = new Map();
234
284
  if (xml == null) return out;
235
285
  const fmt = new Map(); // abstractNumId -> numFmt of level 0
@@ -258,13 +308,13 @@ function listKinds(xml) {
258
308
  num = null;
259
309
  }
260
310
  },
261
- });
311
+ }, budget);
262
312
  } catch {} // an unreadable numbering part: every list is "other", which is reported, never set
263
313
  return out;
264
314
  }
265
315
 
266
316
  /** w:styleId -> the style's display name ("heading 2" is shown as "Heading 2"); empty when styles.xml is missing or unreadable. */
267
- function styleNames(xml) {
317
+ function styleNames(xml, budget) {
268
318
  const out = new Map();
269
319
  if (xml == null) return out;
270
320
  let id = null;
@@ -277,7 +327,7 @@ function styleNames(xml) {
277
327
  onClose(name) {
278
328
  if (name === "w:style") id = null;
279
329
  },
280
- });
330
+ }, budget);
281
331
  } catch {} // a damaged styles part only costs the readable style names
282
332
  return out;
283
333
  }
@@ -287,15 +337,96 @@ const PARA_TEXT_KEEP = 2000;
287
337
 
288
338
  // ---- engine location -> durable anchor ----------------------------------------------------------------------
289
339
 
340
+ // A paragraph's normalized text, kept per paragraph object: every comment resolved against the same reading reuses it.
341
+ const NORM = new WeakMap();
342
+ const norm = (p) => {
343
+ let v = NORM.get(p);
344
+ if (v === undefined) NORM.set(p, (v = normText(p.text)));
345
+ return v;
346
+ };
347
+ /** what norm(p) is about to read: the paragraph's text the first time (src/text.js weigh), nothing once it is kept (spent from the request's budget) */
348
+ const normCost = (p) => (NORM.has(p) ? 0 : weightOf(p));
349
+ // the byte tables a search is priced by (src/text.js hitsOf): of a paragraph's normalized text, and of its text as written
350
+ const NORM_HITS = new WeakMap();
351
+ const RAW_HITS = new WeakMap();
352
+ /** a native search for `q` in norm(p), which is already kept (src/text.js searchCost) */
353
+ const normSearch = (p, q) => searchCost(norm(p).length, q, hitsOf(NORM_HITS, p, norm(p), q));
354
+
355
+ // The paragraphs of one reading by w14:paraId and by text hash, in document order, built once: every anchor resolved against the reading
356
+ // looks its paragraph up instead of scanning them all.
357
+ const INDEX = new WeakMap();
358
+ function indexOf(paragraphs) {
359
+ let ix = INDEX.get(paragraphs);
360
+ if (ix) return ix;
361
+ ix = { byId: new Map(), byHash: new Map(), byPath: new Map(), byTop: new Map(), noPath: [], tops: null };
362
+ for (const p of paragraphs) {
363
+ if (p.path) (ix.byTop.get(p.path[0]) || ix.byTop.set(p.path[0], []).get(p.path[0])).push(p);
364
+ else ix.noPath.push(p);
365
+ if (p.paraId) (ix.byId.get(p.paraId) || ix.byId.set(p.paraId, []).get(p.paraId)).push(p);
366
+ (ix.byHash.get(p.hash) || ix.byHash.set(p.hash, []).get(p.hash)).push(p);
367
+ if (p.path && !ix.byPath.has(p.path.join(","))) ix.byPath.set(p.path.join(","), p); // the first, as a scan finds it
368
+ }
369
+ INDEX.set(paragraphs, ix);
370
+ return ix;
371
+ }
372
+
373
+ // The paragraphs in the order nearest() ranks them, one level of equal distance from the body block `want` at a time (document order
374
+ // within a level), those without a body position last: the first level holding a match holds the match a scan of all of them picks.
375
+ function* byDistance(paragraphs, ix, want) {
376
+ if (!Number.isSafeInteger(want) || !ix.byTop.size) return yield paragraphs;
377
+ // the body blocks that hold paragraphs, outward from `want` (a body of many tables and few paragraphs has gaps: they are not walked)
378
+ const tops = (ix.tops ||= [...ix.byTop.keys()].sort((a, b) => a - b));
379
+ let hi = 0;
380
+ for (let lo = 0, top = tops.length; lo < top; ) {
381
+ const mid = (lo + top) >> 1;
382
+ if (tops[mid] < want) lo = mid + 1;
383
+ else top = mid;
384
+ hi = lo;
385
+ }
386
+ let lo = hi - 1;
387
+ while (lo >= 0 || hi < tops.length) {
388
+ const dl = lo >= 0 ? want - tops[lo] : Infinity;
389
+ const dh = hi < tops.length ? tops[hi] - want : Infinity;
390
+ const d = Math.min(dl, dh);
391
+ const a = dl === d ? ix.byTop.get(tops[lo--]) : null;
392
+ const b = dh === d ? ix.byTop.get(tops[hi++]) : null;
393
+ yield a && b ? a.concat(b) : a || b;
394
+ }
395
+ if (ix.noPath.length) yield ix.noPath;
396
+ }
397
+
398
+ const BIGRAMS = new WeakMap();
399
+ const bigramsOf = (p) => {
400
+ let v = BIGRAMS.get(p);
401
+ if (!v) BIGRAMS.set(p, (v = bigramCodes(p.text)));
402
+ return v;
403
+ };
404
+
290
405
  function locate(paragraphs, { path, paraId, quote, prefix, text, prevHash, nextHash, dupCount, allowPath = false }) {
291
- const q = normText(quote || "");
406
+ // The page's quote and paragraph text (each can be a whole paragraph, per comment) are paid for before they are read: the text is
407
+ // normalized up to four times below (its hash, its length, the slot's similarity, its bigrams). Past the budget a step that needs it
408
+ // is skipped, and at Send the bare position still answers.
409
+ const q = normPaid(quote) ?? "";
410
+ if (text && !(affords(4 * text.length) && spend(4 * weigh(text)))) text = "";
292
411
  const want = path ? path[0] : null;
293
412
  const dist = (p) => (want == null || !p.path ? 1e6 : Math.abs(p.path[0] - want));
294
- const nearest = (list) => list.slice().sort((a, b) => dist(a) - dist(b) || a.ordinal - b.ordinal)[0];
413
+ // the nearest of `list` (smaller distance, then smaller ordinal) in one pass, charged to the request's budget: null when it runs out
414
+ // (sorting a copy of 50,000 look-alikes for each of 500 anchors held the thread for 4 s, outside the budget)
415
+ const nearest = (list) => {
416
+ if (!spend(list.length * 3)) return null;
417
+ let best = null;
418
+ let bd = 0;
419
+ for (const p of list) {
420
+ const d = dist(p);
421
+ if (!best || d < bd || (d === bd && p.ordinal < best.ordinal)) (best = p), (bd = d);
422
+ }
423
+ return best;
424
+ };
295
425
  const nb = (p) => (prevHash && paragraphs[p.ordinal - 1]?.hash === prevHash ? 1 : 0) + (nextHash && paragraphs[p.ordinal + 1]?.hash === nextHash ? 1 : 0);
426
+ const ix = indexOf(paragraphs);
296
427
  // 1. w14:paraId
297
428
  if (paraId) {
298
- const hit = paragraphs.filter((p) => p.paraId === paraId.toUpperCase());
429
+ const hit = ix.byId.get(paraId.toUpperCase()) || [];
299
430
  if (hit.length === 1) return { p: hit[0], method: "id" };
300
431
  }
301
432
  // 2. identical text; among duplicates prefer the one whose neighbors are unchanged, then the nearest.
@@ -304,44 +435,110 @@ function locate(paragraphs, { path, paraId, quote, prefix, text, prevHash, nextH
304
435
  let ambiguousHash = null;
305
436
  if (text) {
306
437
  const h = textHash(text);
307
- const hits = paragraphs.filter((p) => p.hash === h);
438
+ const hits = ix.byHash.get(h) || [];
308
439
  const ambiguous = (dupCount != null ? dupCount > 1 : normText(text).length < 20) && (prevHash || nextHash);
309
- if (hits.length && (!ambiguous || hits.some((p) => nb(p) > 0))) return { p: nearest(hits.filter((p) => nb(p) === Math.max(...hits.map(nb)))), method: "hash" };
310
- if (hits.length) ambiguousHash = h;
440
+ // one pass over the hits for the best neighbor count, then the nearest of those (a document of 50,000 identical paragraphs and 500
441
+ // anchors held the thread for 4 s: the scans and the sort were outside the budget). Spent up front, at about what a hit costs here.
442
+ if (hits.length && spend(hits.length * 24)) {
443
+ let best = null;
444
+ let top = 0;
445
+ let bd = 0;
446
+ for (const p of hits) {
447
+ const n = nb(p);
448
+ const d = dist(p);
449
+ if (!best || n > top || (n === top && (d < bd || (d === bd && p.ordinal < best.ordinal)))) (best = p), (top = n), (bd = d);
450
+ }
451
+ if (!ambiguous || top > 0) return { p: best, method: "hash" };
452
+ ambiguousHash = h;
453
+ }
311
454
  }
312
455
  // 3. same slot: both neighbors unchanged and exactly one paragraph sits between them (edited in place)
313
456
  if (prevHash && nextHash) {
314
- const slots = paragraphs.filter((p) => nb(p) === 2);
315
- if (slots.length === 1 && (!text || similarity(slots[0].text, text) >= 0.3 || (q && normText(slots[0].text).includes(q)))) return { p: slots[0], method: "neighbors" };
457
+ const after = ix.byHash.get(prevHash) || [];
458
+ const slots = spend(after.length * 4) ? after.map((p) => paragraphs[p.ordinal + 1]).filter((p) => p && nb(p) === 2) : [];
459
+ // the slot's whole text is compared: within the request's budget (a slot of 8 million characters, 50 comments: 56 s before)
460
+ if (slots.length === 1 && (!text || (spend(2 * weightOf(slots[0]) + normCost(slots[0])) && (similarity(slots[0].text, text) >= 0.3 || (q && norm(slots[0]).includes(q)))))) return { p: slots[0], method: "neighbors" };
316
461
  }
317
- // 4. exact quote. Ambiguous when the quote also occurs elsewhere and the prefix does not disambiguate: do not guess here.
462
+ // steps 4 and 5 scan every paragraph: only within the budget of the anchors resolved together (src/text.js scanBudget), 16 units a
463
+ // paragraph, and the text each step searches character by character
464
+ // 4. exact quote. Ambiguous when the quote also occurs elsewhere and the prefix does not disambiguate: do not guess here. The
465
+ // paragraphs are read nearest first (byDistance): the answer is the nearest match, so the search stops at the first distance that
466
+ // has one (50 comments on a paragraph the agent changed, in 26,000 paragraphs, searched every paragraph 50 times).
467
+ let read = true;
318
468
  if (q) {
319
- const hits = paragraphs.filter((p) => normText(p.text).includes(q) && !(ambiguousHash && p.hash === ambiguousHash)); // look-alikes rejected in step 2 stay rejected
320
- if (hits.length) {
321
- const pre = normText(prefix || "");
322
- const better = pre ? hits.filter((p) => normText(p.text).includes(pre + (pre.endsWith(" ") ? "" : " ") + q) || normText(p.text).includes(pre + q)) : [];
323
- if (better.length) return { p: nearest(better), method: "quote" };
324
- if (hits.length === 1 || (!pre && !text)) return { p: nearest(hits), method: "quote" };
469
+ const pre = normPaid(prefix) ?? "";
470
+ const longer = pre ? [pre + (pre.endsWith(" ") ? "" : " ") + q, pre + q] : null;
471
+ const hits = [];
472
+ const better = [];
473
+ for (const level of byDistance(paragraphs, ix, want)) {
474
+ for (const p of level) {
475
+ if (!(read = spend(16 + normCost(p)) && spend(normSearch(p, q)))) break; // a native substring search (src/text.js searchCost)
476
+ if (!norm(p).includes(q) || (ambiguousHash && p.hash === ambiguousHash)) continue; // look-alikes rejected in step 2 stay rejected
477
+ hits.push(p);
478
+ // two native substring searches per hit (src/text.js searchCost, as in the loop above)
479
+ if (longer && spend(2 * normSearch(p, longer[1])) && (norm(p).includes(longer[0]) || norm(p).includes(longer[1]))) better.push(p);
480
+ }
481
+ if (!read) break;
482
+ const found = better.length ? better : !pre && !text ? hits : [];
483
+ if (found.length) {
484
+ const p = nearest(found);
485
+ if (p) return { p, method: "quote" };
486
+ }
325
487
  }
326
- }
488
+ if (read && hits.length === 1) {
489
+ const p = nearest(hits);
490
+ if (p) return { p, method: "quote" };
491
+ }
492
+ } else read = spend(paragraphs.length * 16);
327
493
  // 5. fuzzy: similarity to the remembered text (or the quote), neighbors count in favor
328
494
  const probe = text || q;
329
- if (probe) {
495
+ let spent = !read;
496
+ if (probe && read) {
330
497
  // The best candidate must clearly beat the runner-up: in templated text a deleted paragraph looks like its siblings,
331
498
  // and "moved or removed" is a better answer than attaching the comment to the wrong one.
332
- const scored = [];
499
+ // the two best candidates only, found in one pass: the answer needs the winner and the runner-up, never the order of the rest
500
+ let best = null;
501
+ let second = null;
502
+ let count = 0;
503
+ const P = bigramCodes(text || q); // the probe's bigrams once; each paragraph's are kept with it (bigramsOf)
504
+ const n = P.codes.length;
333
505
  for (const p of paragraphs) {
334
506
  if (ambiguousHash && p.hash === ambiguousHash) continue; // rejected above: a look-alike elsewhere is not this paragraph
335
- const sim = text ? similarity(p.text, text) : normText(p.text).length ? similarity(normText(p.text).slice(0, q.length + 40), q) : 0;
336
- if (sim >= 0.6) scored.push({ p, sim, score: sim + nb(p) * 0.1 - dist(p) * 1e-6 }); // distance only breaks ties
507
+ let sim = 0;
508
+ if (text) {
509
+ if (!BIGRAMS.has(p) && !spend(weightOf(p))) {
510
+ spent = true; // its bigrams are read (and sorted) once per reading: the first time costs its text
511
+ break;
512
+ }
513
+ const B = bigramsOf(p);
514
+ // 0.6 needs the shorter at least 3/7 of the longer (Dice is at most 2 min / (m + n)): skip the others without comparing
515
+ const m = B.codes.length;
516
+ if (P.short === null && B.short === null && (Math.min(m, n) * 7 < Math.max(m, n) * 3)) continue;
517
+ if (!spend(m + n)) {
518
+ spent = true; // the budget is spent: "moved or removed" (src/text.js scanBudget), or the bare position at Send
519
+ break;
520
+ }
521
+ sim = diceOf(B, P);
522
+ } else if (norm(p).length) {
523
+ if (!spend(normCost(p) + q.length + 40 + n)) {
524
+ spent = true;
525
+ break;
526
+ }
527
+ sim = diceOf(bigramCodes(norm(p).slice(0, q.length + 40)), P);
528
+ }
529
+ if (sim >= 0.6) {
530
+ const c = { p, sim, score: sim + nb(p) * 0.1 - dist(p) * 1e-6 }; // distance only breaks ties
531
+ count++;
532
+ if (!best || c.score > best.score) (second = best), (best = c);
533
+ else if (!second || c.score > second.score) second = c;
534
+ }
337
535
  }
338
- scored.sort((a, b) => b.score - a.score);
339
- if (scored.length && (scored.length === 1 || scored[0].score - scored[1].score >= 0.04)) return { p: scored[0].p, method: "fuzzy", score: Math.round(scored[0].sim * 100) / 100 };
536
+ if (!spent && best && (count === 1 || best.score - second.score >= 0.04)) return { p: best.p, method: "fuzzy", score: Math.round(best.sim * 100) / 100 };
340
537
  }
341
538
  // At Send time the browser's own index is as fresh as the bytes, so the bare position is acceptable.
342
539
  // After a reload it never is (an insert above shifts it): resolveDurable does not pass allowPath.
343
540
  if (allowPath && path) {
344
- const hit = paragraphs.find((p) => p.path && p.path.length === path.length && p.path.every((v, i) => v === path[i]));
541
+ const hit = ix.byPath.get(path.join(",")); // by lookup, not a scan: the bare position must still answer once the budget is spent
345
542
  if (hit) return { p: hit, method: "path" };
346
543
  }
347
544
  return null;
@@ -358,42 +555,205 @@ export function durableFromEngine(paragraphs, loc, ctx = {}) {
358
555
  return durableFor(paragraphs, hit.p, loc.blockPath ? loc.blockPath[0] : null, ctx, loc);
359
556
  }
360
557
 
361
- /** Offsets of every non-overlapping occurrence of `q` in `text`. */
362
- function occurrences(text, q) {
558
+ /**
559
+ * Offsets of the non-overlapping occurrences of `q` in `text` (the first `max` of them), or null when the request's budget runs out: the
560
+ * search is counted as a native substring search (src/text.js searchCost) and each occurrence found at OCCURRENCE.
561
+ */
562
+ // the complete list of a quote's occurrences in a paragraph, kept per paragraph of a reading for the last few quotes (rule: what is
563
+ // derived from the same paragraph is computed once per reading): 50 comments on one selection of a repeated text listed the same
564
+ // 395,000 occurrences 50 times. Ceiling: OCC_KEEP quotes per paragraph, the oldest dropped first.
565
+ const OCC = new WeakMap();
566
+ const OCC_KEEP = 16;
567
+ function occurrencesIn(p, q) {
568
+ let m = OCC.get(p);
569
+ const hit = m?.get(q);
570
+ if (hit) return hit;
571
+ const occ = occurrences(p, q);
572
+ if (occ) {
573
+ if (!m) OCC.set(p, (m = new Map()));
574
+ if (m.size >= OCC_KEEP) m.delete(m.keys().next().value);
575
+ m.set(q, occ);
576
+ }
577
+ return occ;
578
+ }
579
+ function occurrences(p, q, max = Infinity) {
580
+ const { text } = p;
581
+ if (!spend(searchCost(text.length, q, hitsOf(RAW_HITS, p, text, q)))) return null;
363
582
  const out = [];
364
- for (let i = text.indexOf(q); i >= 0; i = text.indexOf(q, i + q.length)) out.push(i);
583
+ for (let i = text.indexOf(q); i >= 0 && out.length < max; i = text.indexOf(q, i + q.length)) {
584
+ if (!spend(OCCURRENCE)) return null;
585
+ out.push(i);
586
+ }
365
587
  return out;
366
588
  }
367
589
 
590
+ /**
591
+ * normText(text).slice(0, n), reading only as much of the text as that needs (windows cut as endsWithNorm cuts them), each character read
592
+ * spent from the budget; null past it. A comment's paragraph text is kept to 2,000 characters, and normalizing a whole paragraph of
593
+ * 8 million characters for each of 50 comments held the thread for seconds.
594
+ */
595
+ function normHead(text, n) {
596
+ for (let size = 2 * n + 8; ; size *= 2) {
597
+ const e = cutNear(text, Math.min(text.length, size), 1);
598
+ if (e === null || !spend(weigh(text.slice(0, e)))) return null;
599
+ const head = normText(text.slice(0, e));
600
+ if (e === text.length || head.length > n) return head.slice(0, n);
601
+ }
602
+ }
603
+
604
+ // normText(text.slice(0, at)).endsWith(prefix) and normText(text.slice(from)).startsWith(suffix), reading only as much of the text as
605
+ // the prefix or suffix needs. A window is cut only where nothing on one side changes how the other side normalizes (`cuttable`), so the
606
+ // window normalizes to exactly the end (or the start) of what the whole text normalizes to, give or take one space where it was cut.
607
+ // Once the window holds at least as many normalized characters as the prefix, it answers for the whole text; else it doubles.
608
+ // Every character read (the walk to a cut and the window normalized) is spent from the budget of the request (src/text.js): null when it
609
+ // runs out. An outside review: CJK text has no character below U+0300, every window grew to the whole paragraph, 3.65 s for one anchor.
610
+ /** The nearest place at or before (dir -1) or after (dir 1) i where a window may be cut, each character walked spent; null past the budget. */
611
+ function cutNear(text, i, dir) {
612
+ let n = 0;
613
+ while (!cuttable(text, i)) {
614
+ i += dir;
615
+ if (++n % 1024 === 0 && !spend(1024)) return null;
616
+ }
617
+ return spend(n % 1024) ? i : null;
618
+ }
619
+ export function endsWithNorm(text, at, prefix) {
620
+ for (let size = 2 * prefix.length + 8; ; size *= 2) {
621
+ const w = cutNear(text, Math.max(0, at - size), -1);
622
+ if (w === null || !spend(weigh(text.slice(w, at)))) return null;
623
+ const n = normText(text.slice(w, at));
624
+ if (w === 0 || n.length >= prefix.length) return n.endsWith(prefix);
625
+ }
626
+ }
627
+ export function startsWithNorm(text, from, suffix) {
628
+ for (let size = 2 * suffix.length + 8; ; size *= 2) {
629
+ const e = cutNear(text, Math.min(text.length, from + size), 1);
630
+ if (e === null || !spend(weigh(text.slice(from, e)))) return null;
631
+ const n = normText(text.slice(from, e));
632
+ if (e === text.length || n.length >= suffix.length) return n.startsWith(suffix);
633
+ }
634
+ }
635
+
368
636
  /**
369
637
  * Which occurrence of a repeated quote a selection or a stored anchor means. Prefix and suffix are the normalized text around it;
370
- * `nth` is the occurrence index remembered at capture. Returns the offset, or -1 when the text does not occur.
638
+ * `nth` is the occurrence index remembered at capture. Returns {at: the offset, k: which occurrence it is}, or {at: -1, k: null} when the
639
+ * text does not occur or the budget ran out.
371
640
  */
372
- function pickOccurrence(text, q, { prefix = "", suffix = "", nth = null } = {}) {
373
- const occ = occurrences(text, q);
374
- if (!occ.length) return -1;
375
- const guarded = occ.filter((at) => (!prefix || normText(text.slice(0, at)).endsWith(prefix)) && (!suffix || normText(text.slice(at + q.length)).startsWith(suffix)));
641
+ /** does the text around the occurrence at `at` (of `qLen` UTF-16 units) read as prefix and suffix (normalized, as an anchor keeps them)? null: the budget ran out */
642
+ function fitsAt(text, qLen, at, { prefix = "", suffix = "" }) {
643
+ prefix = String(prefix || "").slice(-CTX); // what an anchor keeps (durableFor); a longer one from elsewhere is read as its last CTX
644
+ suffix = String(suffix || "").slice(0, CTX);
645
+ const pre = !prefix || endsWithNorm(text, at, prefix);
646
+ if (pre === null) return null;
647
+ const suf = pre && (!suffix || startsWithNorm(text, at + qLen, suffix));
648
+ return suf === null ? null : !!(pre && suf);
649
+ }
650
+ function pickOccurrence(p, q, opts = {}) {
651
+ const text = p.text;
652
+ // the remembered occurrence first, listing only the occurrences up to it: when its surroundings still fit, it is the answer every rule
653
+ // of pickFrom gives, and the others need not be listed or read (an outside review: 300,000 "a" with nth 150,000 lost its offsets
654
+ // under the budget, though the remembered one still fit)
655
+ const { nth = null } = opts;
656
+ // only an index of the occurrences is one: a selection that starts between two of them ("aa" at offset 1 of "aaaa", which the
657
+ // non-overlapping search does not list) was stored with nth -1, and occ[-1] (undefined) made the window readers loop forever on the
658
+ // server's thread, the budget never asked (a window of no characters costs nothing)
659
+ const remembered = Number.isInteger(nth) && nth >= 0 ? nth : null;
660
+ if (remembered !== null && (opts.prefix || opts.suffix)) {
661
+ const head = occurrences(p, q, remembered + 1);
662
+ if (head === null) return { at: -1, k: null };
663
+ if (head.length === remembered + 1) {
664
+ const ok = fitsAt(text, q.length, head[remembered], opts);
665
+ if (ok === null) return { at: -1, k: null };
666
+ if (ok) return { at: head[remembered], k: remembered };
667
+ }
668
+ }
669
+ const occ = occurrencesIn(p, q);
670
+ const at = occ?.length ? pickFrom(text, q.length, occ, { ...opts, nth: remembered }) : -1;
671
+ return at < 0 ? { at: -1, k: null } : { at, k: occ.indexOf(at) };
672
+ }
673
+ function pickNormOccurrence(p, q, opts = {}) {
674
+ const occ = normOccurrences(p, q);
675
+ if (!occ?.length) return { at: -1, len: 0, k: null };
676
+ let { nth = null } = opts;
677
+ if (!(Number.isInteger(nth) && nth >= 0 && nth < occ.length)) nth = null;
678
+ const prefix = String(opts.prefix || "").slice(-CTX);
679
+ const suffix = String(opts.suffix || "").slice(0, CTX);
680
+ if ((prefix || suffix) && occ.length > 1 && !spend(occ.length * 2 * (prefix.length + suffix.length + 16))) return { at: -1, len: 0, k: null };
681
+ const guarded = [];
682
+ for (let i = 0; i < occ.length; i++) {
683
+ const o = occ[i];
684
+ const ok = fitsAt(p.text, o.end - o.start, o.start, { prefix, suffix });
685
+ if (ok === null) return { at: -1, len: 0, k: null };
686
+ if (ok) guarded.push(i);
687
+ }
688
+ let idx;
689
+ if (guarded.length === 1) idx = guarded[0];
690
+ else if (nth != null && guarded.includes(nth)) idx = nth;
691
+ else if (Number.isInteger(opts.hintStart) && guarded.length) {
692
+ idx = guarded.reduce((best, i) => (Math.abs(occ[i].start - opts.hintStart) < Math.abs(occ[best].start - opts.hintStart) ? i : best), guarded[0]);
693
+ } else if (guarded.length) idx = guarded[0];
694
+ else idx = nth != null ? nth : 0;
695
+ const hit = occ[idx];
696
+ return { at: hit.start, len: hit.end - hit.start, k: idx };
697
+ }
698
+ function pickFrom(text, qLen, occ, { prefix = "", suffix = "", nth = null, hintStart = null }) {
699
+ if (!(Number.isInteger(nth) && nth >= 0 && nth < occ.length)) nth = null;
700
+ prefix = String(prefix || "").slice(-CTX);
701
+ suffix = String(suffix || "").slice(0, CTX);
702
+ const fits = (at) => fitsAt(text, qLen, at, { prefix, suffix });
703
+ // each occurrence reads about CTX characters on each side, out of the budget of the request (src/text.js): a paragraph of 100,000
704
+ // "a" read the whole text before every one of them and took 10 seconds in an outside review. Past the budget, no offset.
705
+ if ((prefix || suffix) && occ.length > 1 && !spend(occ.length * 2 * (prefix.length + suffix.length + 16))) return -1;
706
+ const guarded = [];
707
+ for (const at of occ) {
708
+ const ok = fits(at);
709
+ if (ok === null) return -1;
710
+ if (ok) guarded.push(at);
711
+ }
376
712
  if (guarded.length === 1) return guarded[0];
377
713
  if (nth != null && nth < occ.length && guarded.includes(occ[nth])) return occ[nth];
714
+ if (Number.isInteger(hintStart) && guarded.length) {
715
+ return guarded.reduce((best, at) => (Math.abs(at - hintStart) < Math.abs(best - hintStart) ? at : best), guarded[0]);
716
+ }
378
717
  if (guarded.length) return guarded[0];
379
718
  return nth != null && nth < occ.length ? occ[nth] : occ[0];
380
719
  }
381
720
 
382
721
  function durableFor(paragraphs, p, engineTop, ctx, loc = {}) {
383
- const norm = normText(p.text);
384
- const q = normText(ctx.quote || "");
385
- const prefix = normText(ctx.prefix || "").slice(-CTX);
386
- const suffix = normText(ctx.suffix || "").slice(0, CTX);
722
+ // the page's quote, prefix and suffix, paid for before they are normalized; past the budget the anchor is the paragraph alone
723
+ const q = normPaid(ctx.quote) ?? "";
724
+ const prefix = (normPaid(ctx.prefix) ?? "").slice(-CTX);
725
+ const suffix = (normPaid(ctx.suffix) ?? "").slice(0, CTX);
387
726
  let start = null;
388
727
  let end = null;
389
728
  let nth = null;
390
729
  if (q) {
391
730
  // the engine's own offsets when they point at the quote; else the occurrence whose surroundings match prefix and suffix
392
- const at = Number.isInteger(loc.start) && Number.isInteger(loc.end) && p.text.slice(loc.start, loc.end) === ctx.quote ? loc.start : pickOccurrence(p.text, ctx.quote, { prefix, suffix });
393
- if (at >= 0) {
394
- start = at;
395
- end = at + ctx.quote.length;
396
- nth = occurrences(p.text, ctx.quote).indexOf(at);
731
+ const hintStart = Number.isInteger(loc.start) ? loc.start : null;
732
+ const preferNorm = (ctx.quote !== q && ctx.quote.trim() !== q) || (q.includes(" ") && HAS_NON_PLAIN_WS.test(p.text));
733
+ const own = hintStart !== null && Number.isInteger(loc.end) && p.text.slice(loc.start, loc.end) === ctx.quote;
734
+ let picked;
735
+ if (own) {
736
+ let k = -1;
737
+ if (preferNorm) {
738
+ const nocc = normOccurrences(p, q);
739
+ if (nocc) {
740
+ k = nocc.findIndex((o) => o.start === loc.start && o.end === loc.end);
741
+ if (k < 0) k = nocc.findIndex((o) => o.start >= loc.start && o.end <= loc.end);
742
+ }
743
+ } else {
744
+ k = occurrencesIn(p, ctx.quote)?.indexOf(loc.start) ?? -1;
745
+ }
746
+ picked = { at: loc.start, k: k >= 0 ? k : null, len: ctx.quote.length };
747
+ } else if (preferNorm) {
748
+ picked = pickNormOccurrence(p, q, { prefix, suffix, hintStart });
749
+ } else {
750
+ picked = { ...pickOccurrence(p, ctx.quote, { prefix, suffix, hintStart }), len: ctx.quote.length };
751
+ if (picked.at < 0) picked = pickNormOccurrence(p, q, { prefix, suffix, hintStart });
752
+ }
753
+ if (picked.at >= 0) {
754
+ start = picked.at;
755
+ end = picked.at + picked.len;
756
+ nth = picked.k;
397
757
  }
398
758
  }
399
759
  return {
@@ -407,8 +767,8 @@ function durableFor(paragraphs, p, engineTop, ctx, loc = {}) {
407
767
  textHash: p.hash,
408
768
  prevHash: paragraphs[p.ordinal - 1]?.hash || null,
409
769
  nextHash: paragraphs[p.ordinal + 1]?.hash || null,
410
- dupCount: paragraphs.filter((x) => x.hash === p.hash).length,
411
- paraText: norm.slice(0, PARA_TEXT_KEEP),
770
+ dupCount: indexOf(paragraphs).byHash.get(p.hash)?.length ?? 0,
771
+ paraText: normHead(p.text, PARA_TEXT_KEEP) ?? "",
412
772
  quote: q,
413
773
  prefix,
414
774
  suffix,
@@ -429,10 +789,27 @@ export function resolveDurable(paragraphs, d) {
429
789
  const q = d.quote || "";
430
790
  let start = null;
431
791
  let end = null;
432
- const at = q ? pickOccurrence(p.text, q, { prefix: d.prefix, suffix: d.suffix, nth: d.nth }) : -1;
792
+ const ctxOpts = { prefix: d.prefix ?? d.before ?? "", suffix: d.suffix ?? d.after ?? "", nth: d.nth ?? d.occ ?? null };
793
+ const preferNorm = !!q && q === normText(q) && q.includes(" ") && HAS_NON_PLAIN_WS.test(p.text);
794
+ let at = -1;
795
+ let len = q.length;
796
+ let k = null;
797
+ if (q) {
798
+ if (preferNorm) {
799
+ const picked = pickNormOccurrence(p, q, ctxOpts);
800
+ if (picked.at >= 0) ({ at, len, k } = picked);
801
+ else ({ at, k } = pickOccurrence(p, q, ctxOpts));
802
+ } else {
803
+ ({ at, k } = pickOccurrence(p, q, ctxOpts));
804
+ if (at < 0 || (Number.isInteger(ctxOpts.nth) && ctxOpts.nth > 0 && k !== ctxOpts.nth)) {
805
+ const picked = pickNormOccurrence(p, q, ctxOpts);
806
+ if (picked.at >= 0) ({ at, len, k } = picked);
807
+ }
808
+ }
809
+ }
433
810
  if (at >= 0) {
434
811
  start = at;
435
- end = at + q.length;
812
+ end = at + len;
436
813
  }
437
- return { method: hit.method, score: hit.score, ordinal: p.ordinal, path: p.path, tableNo: p.tableNo ?? null, paraId: p.paraId, textHash: p.hash, protected: !!p.protected, ...(p.protected ? { protectInfo: protectInfo(p) } : {}), dupCount: paragraphs.filter((x) => x.hash === p.hash).length, nth: at >= 0 ? occurrences(p.text, q).indexOf(at) : null, prevHash: paragraphs[p.ordinal - 1]?.hash || null, nextHash: paragraphs[p.ordinal + 1]?.hash || null, text: normText(p.text).slice(0, PARA_TEXT_KEEP), start, end, engineDelta: d.engineDelta || 0 };
814
+ return { method: hit.method, score: hit.score, ordinal: p.ordinal, path: p.path, tableNo: p.tableNo ?? null, paraId: p.paraId, textHash: p.hash, protected: !!p.protected, ...(p.protected ? { protectInfo: protectInfo(p) } : {}), dupCount: indexOf(paragraphs).byHash.get(p.hash)?.length ?? 0, nth: at >= 0 ? k : null, prevHash: paragraphs[p.ordinal - 1]?.hash || null, nextHash: paragraphs[p.ordinal + 1]?.hash || null, text: normHead(p.text, PARA_TEXT_KEEP) ?? "", start, end, engineDelta: d.engineDelta || 0 };
438
815
  }