@raibitsi/tiresias 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/reader.js CHANGED
@@ -34,20 +34,176 @@ const BOILER_WORDS = new Set([
34
34
  "subscription", "sidebar", "breadcrumb", "breadcrumbs", "advert", "advertisement", "ads", "ad", "sponsored", "promo", "popup",
35
35
  "modal", "banner", "masthead", "footer", "nav", "navbar", "menu", "comments", "disqus",
36
36
  ]);
37
- function boilerplate(a) {
38
- if (QUIET_ROLES.has((a.role ?? "").toLowerCase()))
39
- return true;
37
+ // v1.2: never the page's root. A class, id or role on <html> or <body> describes the whole page, not a part of
38
+ // it: Wikipedia's <html> carries "vector-feature-main-menu-pinned-disabled", and the word "menu" there dropped the
39
+ // text of every article.
40
+ const PAGE_ROOT = new Set(["html", "body"]);
41
+ // v1.2: words that name a region of the layout. Sites also use them in the class of the wrapper AROUND the
42
+ // content ("has-sidebar", "no-footer-gap", "with-product-nav__wrapper"), so these words use local surviving
43
+ // text and semantic content to recognize wrappers (markWrappers). The other words (cookie, consent, modal,
44
+ // newsletter and so on) name the thing itself and stay boilerplate however much of a thin page they fill.
45
+ const LAYOUT_WORDS = new Set(["nav", "navbar", "menu", "sidebar", "footer", "banner", "masthead"]);
46
+ // The word that makes this element boilerplate by its class or id, or null.
47
+ function boilerWord(a) {
40
48
  const names = `${a.class ?? ""} ${a.id ?? ""}`.slice(0, 500).toLowerCase();
41
49
  for (const word of names.split(/[\s_-]+/))
42
50
  if (BOILER_WORDS.has(word))
43
- return true;
44
- return false;
51
+ return word;
52
+ return null;
53
+ }
54
+ // Every word in the class and id that names boilerplate is a layout word (so the wrapper rule may apply).
55
+ function onlyLayoutWords(a) {
56
+ const names = `${a.class ?? ""} ${a.id ?? ""}`.slice(0, 500).toLowerCase();
57
+ for (const word of names.split(/[\s_-]+/))
58
+ if (BOILER_WORDS.has(word) && !LAYOUT_WORDS.has(word))
59
+ return false;
60
+ return true;
61
+ }
62
+ function boilerplate(a, n, wrappers) {
63
+ const name = n?.name;
64
+ if (name !== undefined && PAGE_ROOT.has(name.toLowerCase()))
65
+ return false;
66
+ if (QUIET_ROLES.has((a.role ?? "").toLowerCase()))
67
+ return true;
68
+ if (boilerWord(a) === null)
69
+ return false;
70
+ return !(n !== undefined && wrappers?.has(n));
45
71
  }
46
72
  const BLOCK = new Set([
47
73
  "p", "div", "section", "article", "li", "ul", "ol", "tr", "table", "blockquote", "pre", "br", "hr", "main",
48
74
  "header", "aside", "form", "dl", "dt", "dd", "figure", "figcaption", "details", "summary", "caption",
49
75
  ]);
50
76
  const norm = (s) => s.replace(/\s+/g, " ").trim();
77
+ // ---- W-39 (v1.2): who wrote it ----
78
+ // Text a visitor typed into the page (comments, reviews, answers, forum posts), read from the page's own markup at
79
+ // three levels of confidence. Markup can only ADD the mark: nothing inside a visitor's text can take it away.
80
+ // Level 1, a formal standard: schema.org types and properties (microdata itemtype and itemprop, RDFa typeof and
81
+ // property, and the same names in JSON-LD), and the ARIA role "comment".
82
+ const VISITOR_TYPES = new Set(["comment", "review", "usercomments", "answer", "discussionforumposting", "socialmediaposting"]);
83
+ const VISITOR_PROPS = new Set(["comment", "review", "reviewbody", "commenttext", "suggestedanswer", "acceptedanswer"]);
84
+ // Level 2, a section the page names: a class or id token, or one of its parts split at - and _ (so "preview" and
85
+ // "s-prose" do not count). Forum post bodies count as whole tokens. A token that also names a form, a link, a count
86
+ // or a title is the site's own furniture around the comments ("comment-form", "comments-link", "reply-title"), and
87
+ // counts for nothing. Class names inside code are a highlighter's (class="comment") and count for nothing either.
88
+ const VISITOR_PARTS = new Set(["comments", "comment", "commentlist", "reviews", "review", "replies", "reply", "testimonials", "testimonial", "usernotes"]);
89
+ const VISITOR_TOKENS = new Set(["comment-body", "commtext", "cooked", "post-body", "postbody", "message-body", "bbwrapper", "answercell"]);
90
+ const FURNITURE_PARTS = new Set(["form", "respond", "link", "links", "count", "number", "title", "button", "btn", "icon", "notes", "closed", "policy"]);
91
+ // Level 3, a heading that names visitor text, in English only for now: its following content, up to a sibling heading
92
+ // of the same or a higher rank or the container's end, is visitor text. The whole heading must match, so
93
+ // "Our review process" and "Peer review" do not.
94
+ // A bare "Responses" is left out (API documentation uses it for status codes); "3 Responses to ..." counts.
95
+ const VISITOR_HEADING = new RegExp("^(?:(?:customer|visitor|user|reader|student|guest|member|community|product|verified|recent|latest|top|all)s?'? )?" +
96
+ "(?:comments|replies|testimonials|(?:ratings? (?:and|&) )?reviews(?: (?:and|&) ratings?)?|user contributed notes(?: \\d[\\d,.]* notes?)?)(?: \\(\\d[\\d,.]*\\))?$" +
97
+ "|^(?:\\d[\\d,.]*|one) (?:comments?|reviews?|replies|reply|responses?|thoughts?)(?: (?:on|to|for) .+)?$");
98
+ export function visitorHeading(heading) {
99
+ const text = norm(heading);
100
+ if (text.length > 200)
101
+ return false;
102
+ return VISITOR_HEADING.test(text.toLowerCase().replace(/[‘’]/g, "'").replace(/[:.]$/, ""));
103
+ }
104
+ // The last part of a schema.org name, lower case: "https://schema.org/Review", "schema:Review" and "Review" are one.
105
+ function schemaName(t) {
106
+ let end = t.length;
107
+ while (end > 0 && t[end - 1] === "/")
108
+ end--;
109
+ let start = end;
110
+ while (start > 0 && t[start - 1] !== "/" && t[start - 1] !== "#" && t[start - 1] !== ":")
111
+ start--;
112
+ return t.slice(start, end).toLowerCase();
113
+ }
114
+ // Scan all attribute tokens, without allocating an array proportional to a hostile attribute's word count.
115
+ function* words(v) {
116
+ if (v)
117
+ for (const m of v.matchAll(/\S+/g))
118
+ yield m[0];
119
+ }
120
+ // The level an element's own attributes give its text (1 or 2), or 0.
121
+ function visitorMark(a, inCode) {
122
+ for (const key of ["itemtype", "typeof"])
123
+ for (const t of words(a[key]))
124
+ if (VISITOR_TYPES.has(schemaName(t)))
125
+ return 1;
126
+ for (const key of ["itemprop", "property"])
127
+ for (const p of words(a[key]))
128
+ if (VISITOR_PROPS.has(schemaName(p)))
129
+ return 1;
130
+ for (const r of words(a.role))
131
+ if (r.toLowerCase() === "comment")
132
+ return 1;
133
+ if (inCode)
134
+ return 0;
135
+ for (const key of ["class", "id"])
136
+ for (const token of words(a[key])) {
137
+ const t = token.toLowerCase();
138
+ if (t.length <= 32 && VISITOR_TOKENS.has(t.replace(/_/g, "-")))
139
+ return 2;
140
+ let visitor = false;
141
+ let furniture = false;
142
+ for (const m of t.matchAll(/[^-_]+/g)) {
143
+ if (FURNITURE_PARTS.has(m[0])) {
144
+ furniture = true;
145
+ break;
146
+ }
147
+ if (VISITOR_PARTS.has(m[0]))
148
+ visitor = true;
149
+ }
150
+ if (visitor && !furniture)
151
+ return 2;
152
+ }
153
+ return 0;
154
+ }
155
+ // Source ancestry survives the bounded DOM's forced closes, including code's highlighter exception.
156
+ function elementMark(a, inCode, scopes) {
157
+ let inherited = scopes.get(a);
158
+ if (!inherited)
159
+ scopes.set(a, inherited = { visitor: 0, code: false });
160
+ const key = inCode || inherited.code ? "codeMark" : "normalMark";
161
+ // A flattened parent can have thousands of children. Its long attributes are scanned once per code context.
162
+ const own = inherited[key] ?? (inherited[key] = visitorMark(a, key === "codeMark"));
163
+ return strongest(inherited.visitor, own);
164
+ }
165
+ // Any visitor value anywhere in parsed JSON (walked without recursion; a 5 MB document is the most it can be).
166
+ function anyVisitorValue(root) {
167
+ const stack = [[root, undefined]];
168
+ while (stack.length) {
169
+ const [v, key] = stack.pop();
170
+ if (visitorValue(v, key))
171
+ return true;
172
+ if (Array.isArray(v))
173
+ for (const c of v)
174
+ stack.push([c, undefined]);
175
+ else if (v && typeof v === "object")
176
+ for (const [k, c] of Object.entries(v))
177
+ stack.push([c, k]);
178
+ }
179
+ return false;
180
+ }
181
+ // The stronger of two levels (1 is the strongest; 0 is none).
182
+ const strongest = (x, y) => (x && y ? Math.min(x, y) : x || y);
183
+ const asLevel = (n) => (n === 1 || n === 2 || n === 3 ? n : undefined);
184
+ // A JSON-LD value a visitor wrote: one under a visitor property, or an object of a visitor type.
185
+ function visitorValue(v, key) {
186
+ if (key !== undefined && VISITOR_PROPS.has(schemaName(key)))
187
+ return true;
188
+ return !!v && typeof v === "object" && !Array.isArray(v) && typeOf(v).some((t) => VISITOR_TYPES.has(schemaName(t)));
189
+ }
190
+ // v1.2: extraction and scoring agree about destinations, independently of the link-list cap.
191
+ // Keep the overlong case separate so extraction can still report omitted URLs.
192
+ function linkDestination(href, base) {
193
+ if (!href)
194
+ return null;
195
+ let u;
196
+ try {
197
+ u = new URL(href.trim(), base);
198
+ }
199
+ catch {
200
+ return null;
201
+ }
202
+ if (u.protocol !== "http:" && u.protocol !== "https:")
203
+ return null;
204
+ u.hash = "";
205
+ return u.href.length > MAX_URL ? { tooLong: true } : { url: u.href };
206
+ }
51
207
  // Collapse whitespace while tracking where link markers land in the result.
52
208
  function collapse(parts) {
53
209
  let text = "";
@@ -93,7 +249,7 @@ function splitBlock(b) {
93
249
  end = splitEnd(b.text, end);
94
250
  const lo = start;
95
251
  const marks = b.marks.filter(([pos]) => (pos > lo || lo === 0) && pos <= end).map(([pos, n]) => [pos - lo, n]);
96
- out.push({ text: b.text.slice(start, end), marks });
252
+ out.push({ text: b.text.slice(start, end), marks, ...(b.visitor ? { visitor: b.visitor } : {}) });
97
253
  start = end;
98
254
  while (b.text[start] === " ")
99
255
  start++;
@@ -126,10 +282,26 @@ class Builder {
126
282
  main = null; // W-22: the main content's element and its ancestors, when one was found
127
283
  inMain = new Set(); // every node inside the main content element
128
284
  parentOf = new Map();
285
+ wrappers = new Set(); // v1.2: layout-named elements kept by local surviving text (markWrappers)
286
+ // W-39: the level of the visitor markup around the node being read (0: none), the strongest level among the text
287
+ // waiting in `parts`, the source heading regions, and code depth.
288
+ visitor = 0;
289
+ partsVisitor = 0;
290
+ headingVisitors = new WeakSet(); // v1.2: source heading scopes, independent of main-content selection
291
+ visitorScopes = new WeakMap();
292
+ code = 0;
129
293
  constructor(shield, base) {
130
294
  this.shield = shield;
131
295
  this.base = base;
132
296
  }
297
+ // W-39: the level a block made now gets: its own text's, or the heading region's.
298
+ by(own) {
299
+ return asLevel(own);
300
+ }
301
+ take() {
302
+ this.parts = [];
303
+ this.partsVisitor = 0;
304
+ }
133
305
  // A preformatted block, kept exactly, fenced.
134
306
  // The pending marker of a list item, taken to lead a block written directly (a <pre>: round 79).
135
307
  // Outside an item, or for an item already led, the continuation indent (round 80).
@@ -138,7 +310,7 @@ class Builder {
138
310
  if (!this.loneMarker())
139
311
  return item?.led ? item.indent : "";
140
312
  const m = this.parts.filter((p) => typeof p === "string").join("").replace(/\s+$/, ""); // nesting indent kept
141
- this.parts = [];
313
+ this.take();
142
314
  this.marker = false;
143
315
  if (item)
144
316
  item.led = true;
@@ -146,7 +318,7 @@ class Builder {
146
318
  }
147
319
  // `indent` goes in front of EVERY line of the block, its fences included, so a <pre> inside a list item
148
320
  // stays inside it (Codex round 81); the first chunk starts with `lead` (the item's marker) instead.
149
- pre(raw, marks = [], lead = "", indent = "") {
321
+ pre(raw, marks = [], lead = "", indent = "", visitor = this.visitor) {
150
322
  if (!this.cur)
151
323
  this.startTop();
152
324
  for (let i = 0; i < raw.length;) {
@@ -166,7 +338,8 @@ class Builder {
166
338
  const lines = indent ? piece.slice(0, p).split("\n").length : 0; // lines up to the mark, each indented
167
339
  return [head.length + fence.length + 1 + p + indent.length * lines, n];
168
340
  });
169
- this.cur.blocks.push({ text: `${head}${fence}\n${body}\n${indent}${fence}`, marks: m });
341
+ const by = this.by(visitor);
342
+ this.cur.blocks.push({ text: `${head}${fence}\n${body}\n${indent}${fence}`, marks: m, ...(by ? { visitor: by } : {}) });
170
343
  this.shield.scan(piece, "visible text");
171
344
  i = end;
172
345
  }
@@ -177,14 +350,18 @@ class Builder {
177
350
  this.start(0, "(top of page)");
178
351
  this.cur.generated = true;
179
352
  }
180
- start(level, heading) {
353
+ start(level, heading, visitor = 0) {
181
354
  this.shield.scan(heading, "heading"); // headings are returned text too
182
355
  // The heading shown in the outline is bounded. The full text is not lost:
183
356
  // an overlong heading also becomes the section's first block, so it can
184
357
  // be expanded, searched and quoted.
185
358
  this.cur = { n: this.sections.length + 1, level, heading: heading.slice(0, MAX_HEADING), blocks: [] };
359
+ if (visitor)
360
+ this.cur.visitor = asLevel(visitor); // the heading itself, independently of the following blocks
361
+ // W-39: a visitor's overlong heading stays the visitor's.
362
+ const by = this.by(visitor);
186
363
  if (heading.length > MAX_HEADING)
187
- this.cur.blocks.push(...splitBlock({ text: heading, marks: [] }));
364
+ this.cur.blocks.push(...splitBlock({ text: heading, marks: [], ...(by ? { visitor: by } : {}) }));
188
365
  this.sections.push(this.cur);
189
366
  }
190
367
  marker = false; // a list item's marker was pushed and has not yet led any content
@@ -193,7 +370,7 @@ class Builder {
193
370
  }
194
371
  dropLoneMarker() {
195
372
  if (this.loneMarker())
196
- this.parts = [];
373
+ this.take();
197
374
  this.marker = false;
198
375
  }
199
376
  items = []; // open list items: their continuation indent (round 79)
@@ -216,7 +393,11 @@ class Builder {
216
393
  b.text = indent + b.text;
217
394
  b.marks = b.marks.map(([pos, n]) => [pos + indent.length, n]);
218
395
  }
219
- this.parts = [];
396
+ // W-39: a block holding any visitor text is the visitor's (a mixed paragraph errs toward the visitor).
397
+ const by = this.by(this.partsVisitor);
398
+ if (by)
399
+ b.visitor = by;
400
+ this.take();
220
401
  if (!b.text)
221
402
  return;
222
403
  if (!this.cur)
@@ -225,12 +406,12 @@ class Builder {
225
406
  // first get the item's continuation indent.
226
407
  const pieces = splitBlock(b);
227
408
  const cont = item ? item.indent : "";
228
- this.cur.blocks.push(...pieces.map((pc, k) => k > 0 && cont ? { text: cont + pc.text, marks: pc.marks.map(([pos, n]) => [pos + cont.length, n]) } : pc));
409
+ this.cur.blocks.push(...pieces.map((pc, k) => k > 0 && cont ? { ...pc, text: cont + pc.text, marks: pc.marks.map(([pos, n]) => [pos + cont.length, n]) } : pc));
229
410
  this.shield.scan(b.text, "visible text");
230
411
  }
231
412
  takeHeading() {
232
413
  const t = collapse(this.parts.filter((p) => typeof p === "string")).text;
233
- this.parts = [];
414
+ this.take();
234
415
  // A heading that took a pending list marker leads its item (round 83).
235
416
  if (this.marker) {
236
417
  this.marker = false;
@@ -246,7 +427,7 @@ class Builder {
246
427
  if (!this.loneMarker())
247
428
  return;
248
429
  const m = this.parts.filter((p) => typeof p === "string").join("").replace(/\s+$/, "");
249
- this.parts = [];
430
+ this.take();
250
431
  this.marker = false;
251
432
  const item = this.items[this.items.length - 1];
252
433
  if (item)
@@ -257,23 +438,15 @@ class Builder {
257
438
  }
258
439
  // Returns the link number, or 0 if it is not a followable link or the cap is hit.
259
440
  link(href, text) {
260
- if (!href)
261
- return 0;
262
- let u;
263
- try {
264
- u = new URL(href.trim(), this.base);
265
- }
266
- catch {
267
- return 0;
268
- }
269
- if (u.protocol !== "http:" && u.protocol !== "https:")
441
+ const destination = linkDestination(href, this.base);
442
+ if (!destination)
270
443
  return 0;
271
- u.hash = "";
272
- if (u.href.length > MAX_URL) {
444
+ if ("tooLong" in destination) {
273
445
  this.omitted++; // a shortened URL is a different destination, so it is left out and counted
274
446
  return 0;
275
447
  }
276
- const known = this.byUrl.get(u.href);
448
+ const url = destination.url;
449
+ const known = this.byUrl.get(url);
277
450
  if (known)
278
451
  return known;
279
452
  if (this.links.length >= MAX_LINKS)
@@ -281,8 +454,8 @@ class Builder {
281
454
  const n = this.links.length + 1;
282
455
  const label = snippet(text, 0, 80);
283
456
  this.shield.scan(label, "link text");
284
- this.links.push({ n, url: u.href, text: label });
285
- this.byUrl.set(u.href, n);
457
+ this.links.push({ n, url, text: label });
458
+ this.byUrl.set(url, n);
286
459
  return n;
287
460
  }
288
461
  }
@@ -308,7 +481,8 @@ function visibleText(root, cap = 300, sheet) {
308
481
  // Links inside it are collected like anywhere else and get their [n] marker (Codex round 76).
309
482
  function preText(root, b, cap, marks = []) {
310
483
  let out = "";
311
- const stack = [...(root.children ?? [])].reverse().map((n) => ({ n, quiet: false }));
484
+ let visitor = 0;
485
+ const stack = [...(root.children ?? [])].reverse().map((n) => ({ n, quiet: false, visitor: b.visitor }));
312
486
  let seen = 0;
313
487
  while (stack.length && out.length < cap) {
314
488
  const item = stack.pop();
@@ -323,8 +497,11 @@ function preText(root, b, cap, marks = []) {
323
497
  const { n } = item;
324
498
  let quiet = item.quiet;
325
499
  if (n.type === "text") {
326
- if (!quiet)
500
+ if (!quiet) {
327
501
  out += n.data ?? "";
502
+ if (/\S/.test(n.data ?? ""))
503
+ visitor = strongest(visitor, strongest(item.visitor, b.headingVisitors.has(n) ? 3 : 0));
504
+ }
328
505
  continue;
329
506
  }
330
507
  if (n.type === "comment") {
@@ -343,7 +520,7 @@ function preText(root, b, cap, marks = []) {
343
520
  b.shield.hidden(why, txt);
344
521
  continue;
345
522
  }
346
- if (QUIET.has(n.name) || boilerplate(attribs))
523
+ if (QUIET.has(n.name) || boilerplate(attribs, n, b.wrappers))
347
524
  quiet = true;
348
525
  if (n.name === "br" && !quiet)
349
526
  out += "\n";
@@ -352,10 +529,11 @@ function preText(root, b, cap, marks = []) {
352
529
  if (num && !quiet)
353
530
  stack.push({ mark: num });
354
531
  }
532
+ const mark = strongest(item.visitor, elementMark(attribs, true, b.visitorScopes));
355
533
  for (let i = (n.children ?? []).length - 1; i >= 0; i--)
356
- stack.push({ n: n.children[i], quiet });
534
+ stack.push({ n: n.children[i], quiet, visitor: mark });
357
535
  }
358
- return out;
536
+ return { text: out, visitor };
359
537
  }
360
538
  function rawText(root, cap = 5000) {
361
539
  let out = "";
@@ -374,9 +552,9 @@ function rawText(root, cap = 5000) {
374
552
  function pushChildren(stack, n, quiet) {
375
553
  const ch = n.children ?? [];
376
554
  for (let i = ch.length - 1; i >= 0; i--)
377
- stack.push({ n: ch[i], quiet });
555
+ stack.push({ n: ch[i], quiet, parent: n });
378
556
  }
379
- function walk(root, b) {
557
+ function walk(root, b, headings) {
380
558
  const stack = [{ n: root, quiet: false }];
381
559
  let count = 0;
382
560
  while (stack.length) {
@@ -391,8 +569,11 @@ function walk(root, b) {
391
569
  let quiet = t.quiet;
392
570
  if (n.type === "text") {
393
571
  // A "|" inside a table cell is escaped, so only Tiresias's own separators split cells (round 78).
394
- if (!quiet)
572
+ if (!quiet) {
395
573
  b.parts.push(b.rowDepth > 0 ? (n.data ?? "").replace(/\\/g, "\\\\").replace(/\|/g, "\\|") : n.data ?? "");
574
+ if (/\S/.test(n.data ?? ""))
575
+ b.partsVisitor = strongest(b.partsVisitor, strongest(b.visitor, b.headingVisitors.has(n) ? 3 : 0)); // W-39
576
+ }
396
577
  continue;
397
578
  }
398
579
  if (n.type === "comment") {
@@ -417,8 +598,20 @@ function walk(root, b) {
417
598
  b.shield.scan(attribs[a], `${a} attribute`);
418
599
  if (name === "script" || SKIP.has(name))
419
600
  continue;
420
- if (QUIET.has(name) || boilerplate(attribs) || (b.main !== null && !b.main.has(n) && !b.inMain.has(n)))
601
+ if (QUIET.has(name) || boilerplate(attribs, n, b.wrappers) || (b.main !== null && !b.main.has(n) && !b.inMain.has(n)))
421
602
  quiet = true;
603
+ // W-39: this element's own visitor mark holds for everything inside it, until its end (the restore runs after
604
+ // its children and their own closing steps). Class names on or inside code and pre are a highlighter's.
605
+ if (name === "code" || name === "pre") {
606
+ b.code++;
607
+ stack.push(() => void b.code--);
608
+ }
609
+ const mark = elementMark(attribs, b.code > 0, b.visitorScopes);
610
+ if (mark && strongest(b.visitor, mark) !== b.visitor) {
611
+ const outer = b.visitor;
612
+ b.visitor = strongest(outer, mark);
613
+ stack.push(() => void (b.visitor = outer));
614
+ }
422
615
  // Inside a table row, block elements (paragraphs, lists, headings, pre, nested rows) do not end the
423
616
  // row: their text joins the cell (Codex round 76).
424
617
  const inRow = b.rowDepth > 0;
@@ -429,12 +622,12 @@ function walk(root, b) {
429
622
  const marks = [];
430
623
  const full = preText(n, b, Infinity, marks);
431
624
  let lead = 0;
432
- while (full.charCodeAt(lead) === 10)
625
+ while (full.text.charCodeAt(lead) === 10)
433
626
  lead++; // leading newlines, linearly (Codex round 75: the old regex was quadratic)
434
- const raw = full.slice(lead).trimEnd();
627
+ const raw = full.text.slice(lead).trimEnd();
435
628
  const itemIndent = b.items.length ? b.items[b.items.length - 1].indent : "";
436
629
  if (raw.trim())
437
- b.pre(raw, marks.map(([pos, num]) => [Math.min(pos - lead, raw.length), num]), b.takeLoneMarker(), itemIndent);
630
+ b.pre(raw, marks.map(([pos, num]) => [Math.min(pos - lead, raw.length), num]), b.takeLoneMarker(), itemIndent, strongest(b.visitor, full.visitor));
438
631
  continue;
439
632
  }
440
633
  if (!quiet && (name === "ul" || name === "ol")) {
@@ -509,9 +702,11 @@ function walk(root, b) {
509
702
  b.flush();
510
703
  const level = Number(name[1]);
511
704
  stack.push(() => {
705
+ const by = b.partsVisitor;
512
706
  const heading = b.takeHeading();
707
+ headings?.set(n, { text: heading, visitor: by });
513
708
  if (heading)
514
- b.start(level, heading);
709
+ b.start(level, heading, by);
515
710
  });
516
711
  pushChildren(stack, n, quiet);
517
712
  continue;
@@ -536,6 +731,19 @@ function walk(root, b) {
536
731
  pushChildren(stack, n, quiet);
537
732
  }
538
733
  }
734
+ // v1.2: use the extraction walk before main selection can quiet a source heading. Both ends of a
735
+ // region use its joined, normalised text and visible descendants' attribution. The separate builder
736
+ // and shield discard this pass's body and flags; only the heading summaries survive into the final walk.
737
+ function sourceHeadings(root, b) {
738
+ const shield = new Shield();
739
+ shield.sheet = b.shield.sheet;
740
+ const reader = new Builder(shield, b.base);
741
+ reader.visitorScopes = b.visitorScopes;
742
+ reader.wrappers = b.wrappers;
743
+ const headings = new WeakMap();
744
+ walk(root, reader, headings);
745
+ return headings;
746
+ }
539
747
  // W-22 (v0.5.1): main-content scoring, in-house (no library). Every element's visible text and link text
540
748
  // are summed in one pass from the leaves up. The main content is a <main> (or role=main) element with real
541
749
  // text; else the <article> with the most text; else the container whose text, less its link text, is
@@ -544,29 +752,94 @@ function walk(root, b) {
544
752
  // What the scoring never counts (Codex round 69): what is not rendered, what is hidden (the same rules as
545
753
  // the walk, the page's own style rules included), and boilerplate. So invisible filler cannot win.
546
754
  const unscored = (n, b) => (n.type === "tag" || n.type === "script" || n.type === "style") &&
547
- (SKIP.has(n.name) || n.name === "script" || QUIET.has(n.name) || boilerplate(n.attribs ?? {}) || hiddenReason(n.attribs ?? {}, b.shield.sheet) !== null);
548
- function chooseMain(root, b) {
549
- const text = new Map();
550
- const link = new Map();
755
+ (SKIP.has(n.name) || n.name === "script" || QUIET.has(n.name) || boilerplate(n.attribs ?? {}, n, b.wrappers) || hiddenReason(n.attribs ?? {}, b.shield.sheet) !== null);
756
+ // v1.2: a bounded preorder for both scoring passes. Link membership belongs to text nodes and is inherited
757
+ // from any enclosing destination anchor, including one outside the element being scored.
758
+ function textOrder(root, b, dead, parent) {
551
759
  const order = [];
552
- const stack = [root];
760
+ const linked = new Set();
761
+ const stack = [{ n: root, inLink: false }];
553
762
  let count = 0;
554
763
  while (stack.length) {
555
- const n = stack.pop();
764
+ const { n, inLink } = stack.pop();
556
765
  if (++count > MAX_NODES)
557
766
  throw new ComplexityError();
558
767
  order.push(n);
559
- if (unscored(n, b))
768
+ if (dead(n))
560
769
  continue;
770
+ if (n.type === "text" && inLink)
771
+ linked.add(n);
772
+ const destination = n.name === "a" ? linkDestination(n.attribs?.href, b.base) : null;
773
+ const nextLink = inLink || (destination !== null && "url" in destination);
561
774
  for (const c of n.children ?? []) {
562
- b.parentOf.set(c, n);
563
- stack.push(c);
775
+ parent?.set(c, n);
776
+ stack.push({ n: c, inLink: nextLink });
777
+ }
778
+ }
779
+ return { order, linked };
780
+ }
781
+ // v1.2: layout-named wrappers are decided locally, children before parents. Rejected children contribute
782
+ // nothing. A semantic main, a substantial article, or a wrapper containing substantial semantic content is
783
+ // kept even inside one big link; otherwise substantial text must be at most half destination-link text.
784
+ export const WRAPPER_MIN = 200;
785
+ export const WRAPPER_MAX_LINK_SHARE = 0.5; // a candidate whose text is mostly link text is the navigation itself
786
+ function markWrappers(root, b) {
787
+ // Counted as nothing, and not entered: what the reader never shows (skipped, script, hidden), and what it always
788
+ // quiets whatever this pass decides: nav, footer, aside, the quiet roles (Codex round 1, LOW), and an element
789
+ // named by a word that is not a layout word, such as a cookie box (Codex round 3: its text is dropped later, so
790
+ // counting it here would let it dilute a menu's share of links).
791
+ const dead = (n) => (n.type === "tag" || n.type === "script" || n.type === "style") &&
792
+ (SKIP.has(n.name) ||
793
+ n.name === "script" ||
794
+ QUIET.has(n.name) ||
795
+ (!PAGE_ROOT.has(n.name) && (QUIET_ROLES.has((n.attribs?.role ?? "").toLowerCase()) || !onlyLayoutWords(n.attribs ?? {}))) ||
796
+ hiddenReason(n.attribs ?? {}, b.shield.sheet) !== null);
797
+ const main = (n) => n.name === "main" || (n.attribs?.role ?? "").toLowerCase() === "main";
798
+ const semantic = (n) => main(n) || n.name === "article";
799
+ const { order, linked } = textOrder(root, b, dead);
800
+ const text = new Map();
801
+ const link = new Map(); // of that text, how much sits inside links
802
+ const holdsMain = new Set(); // contains surviving semantic content with at least WRAPPER_MIN text
803
+ for (let i = order.length - 1; i >= 0; i--) {
804
+ const n = order[i];
805
+ let t = n.type === "text" ? norm(n.data ?? "").length : 0;
806
+ let l = linked.has(n) ? t : 0;
807
+ let holds = false;
808
+ if (!dead(n)) {
809
+ for (const c of n.children ?? []) {
810
+ t += text.get(c) ?? 0;
811
+ l += link.get(c) ?? 0;
812
+ if (holdsMain.has(c))
813
+ holds = true;
814
+ }
564
815
  }
816
+ else
817
+ t = 0;
818
+ if (n.type === "tag" && !dead(n) && !PAGE_ROOT.has(n.name) && boilerWord(n.attribs ?? {}) !== null) {
819
+ const itself = main(n) || (n.name === "article" && t >= WRAPPER_MIN);
820
+ const bySize = t >= WRAPPER_MIN && l <= t * WRAPPER_MAX_LINK_SHARE;
821
+ if (itself || holds || bySize)
822
+ b.wrappers.add(n);
823
+ else {
824
+ t = 0;
825
+ l = 0;
826
+ holds = false;
827
+ }
828
+ }
829
+ text.set(n, t);
830
+ link.set(n, l);
831
+ if (holds || (semantic(n) && t >= WRAPPER_MIN))
832
+ holdsMain.add(n);
565
833
  }
834
+ }
835
+ function chooseMain(root, b) {
836
+ const text = new Map();
837
+ const link = new Map();
838
+ const { order, linked } = textOrder(root, b, (n) => unscored(n, b), b.parentOf);
566
839
  for (let i = order.length - 1; i >= 0; i--) {
567
840
  const n = order[i];
568
841
  let t = n.type === "text" ? norm(n.data ?? "").length : 0;
569
- let l = 0;
842
+ let l = linked.has(n) ? t : 0;
570
843
  if (unscored(n, b))
571
844
  t = 0;
572
845
  else
@@ -574,8 +847,6 @@ function chooseMain(root, b) {
574
847
  t += text.get(c) ?? 0;
575
848
  l += link.get(c) ?? 0;
576
849
  }
577
- if (n.name === "a")
578
- l = t;
579
850
  text.set(n, t);
580
851
  link.set(n, l);
581
852
  }
@@ -641,18 +912,38 @@ function chooseMain(root, b) {
641
912
  st.push(c);
642
913
  }
643
914
  }
644
- function scanHead(root) {
645
- const found = { ld: [], links: [], metas: [] };
646
- const stack = [root];
915
+ function scanHead(root, scopes = new WeakMap(), source = new WeakMap(), sheet, headings) {
916
+ const found = { ld: [], links: [], metas: [], headingVisitors: new WeakSet() };
917
+ const stack = [{ n: root, visitor: 0, code: false, headingHidden: false }];
918
+ let region;
647
919
  let count = 0;
648
920
  while (stack.length) {
649
- const n = stack.pop();
921
+ const item = stack.pop();
922
+ const { n } = item;
650
923
  if (++count > MAX_NODES)
651
924
  throw new ComplexityError();
925
+ if (region && (n.startIndex ?? -1) >= region.end)
926
+ region = undefined;
927
+ const code = item.code || n.name === "code" || n.name === "pre";
928
+ const visitor = strongest(item.visitor, elementMark(n.attribs ?? {}, code, scopes));
929
+ const headingHidden = item.headingHidden || SKIP.has(n.name ?? "") || hiddenReason(n.attribs ?? {}, sheet) !== null;
930
+ // v1.2: source headings identify visitor text before main-content selection can quiet them.
931
+ // The same regions apply to ordinary text and to JSON-LD, including flattened source containers.
932
+ const heading = headings?.get(n);
933
+ if (!headingHidden && !visitor && !code && heading?.text && !heading.visitor) {
934
+ const level = Number(n.name[1]);
935
+ const parent = source.get(n.attribs ?? {})?.parent ?? root;
936
+ if (region?.container === parent && level <= region.level)
937
+ region = undefined;
938
+ if (!region && visitorHeading(heading.text))
939
+ region = { level, container: parent, start: source.get(n.attribs ?? {})?.end ?? Infinity, end: source.get(parent)?.end ?? Infinity };
940
+ }
941
+ if (region && (n.startIndex ?? -1) >= region.start)
942
+ found.headingVisitors.add(n);
652
943
  if (n.name === "title" && !found.title)
653
944
  found.title = n;
654
945
  else if (n.name === "script" && /ld\+json/i.test(n.attribs?.type ?? ""))
655
- found.ld.push(n);
946
+ found.ld.push({ n, visitor: strongest(visitor, region ? 3 : 0) });
656
947
  else if (n.name === "base" && !found.base && n.attribs?.href !== undefined)
657
948
  found.base = n;
658
949
  else if (n.name === "link")
@@ -667,7 +958,7 @@ function scanHead(root) {
667
958
  found.byline = n;
668
959
  if (n.children)
669
960
  for (let i = n.children.length - 1; i >= 0; i--)
670
- stack.push(n.children[i]);
961
+ stack.push({ n: n.children[i], visitor, code, headingHidden });
671
962
  }
672
963
  return found;
673
964
  }
@@ -725,7 +1016,19 @@ function finish(b, base) {
725
1016
  // of the line), so a change still only moves the blocks around it. A line longer than MAX_BLOCK is cut.
726
1017
  // Each block is an exact piece of the text, so a quote of it is a real substring.
727
1018
  const MAX_LINE_BLOCKS = 5000;
728
- export function lineBlocks(text) {
1019
+ // W-39: `spans` (sorted, not overlapping: boundedPretty's) marks every block that touches one as written by a visitor.
1020
+ export function lineBlocks(text, spans) {
1021
+ const blocks = lineSlices(text);
1022
+ if (!spans?.length)
1023
+ return blocks.map(([a, b]) => ({ text: text.slice(a, b), marks: [] }));
1024
+ let i = 0;
1025
+ return blocks.map(([a, b]) => {
1026
+ while (i < spans.length && spans[i][1] <= a)
1027
+ i++;
1028
+ return { text: text.slice(a, b), marks: [], ...(i < spans.length && spans[i][0] < b ? { visitor: 1 } : {}) };
1029
+ });
1030
+ }
1031
+ function lineSlices(text) {
729
1032
  // Lines as [start, end) offsets into the text, so every block is an exact slice of it (Codex round 75:
730
1033
  // joining kept lines with "\n" dropped the blank lines between them and made quotes that are not in
731
1034
  // the text).
@@ -743,7 +1046,7 @@ export function lineBlocks(text) {
743
1046
  i = j + 1;
744
1047
  }
745
1048
  if (lines.length <= MAX_LINE_BLOCKS)
746
- return lines.map(([a, b]) => ({ text: text.slice(a, b), marks: [] }));
1049
+ return lines;
747
1050
  const out = [];
748
1051
  let start = -1;
749
1052
  let end = -1;
@@ -757,17 +1060,17 @@ export function lineBlocks(text) {
757
1060
  if (start === -1)
758
1061
  start = a;
759
1062
  else if (b - start > MAX_BLOCK) {
760
- out.push({ text: text.slice(start, end), marks: [] }); // a run never grows past MAX_BLOCK
1063
+ out.push([start, end]); // a run never grows past MAX_BLOCK
761
1064
  start = a;
762
1065
  }
763
1066
  end = b;
764
1067
  if (cut(text.slice(a, b))) {
765
- out.push({ text: text.slice(start, end), marks: [] });
1068
+ out.push([start, end]);
766
1069
  start = -1;
767
1070
  }
768
1071
  }
769
1072
  if (start !== -1)
770
- out.push({ text: text.slice(start, end), marks: [] });
1073
+ out.push([start, end]);
771
1074
  return out;
772
1075
  }
773
1076
  function rawBlocks(text) {
@@ -787,20 +1090,22 @@ export function parseHtml(html, baseUrl, keep, rendered) {
787
1090
  const doc = tree.doc;
788
1091
  const shield = new Shield();
789
1092
  shield.sheet = pageSheet(doc);
1093
+ const visitorScopes = new WeakMap();
790
1094
  if (rendered && /^data-t-[0-9a-f]{16}$/.test(rendered.mark))
791
1095
  shield.sheet.rendered = { mark: rendered.mark, complete: rendered.complete === true };
792
1096
  if (tree.flattened) {
793
1097
  shield.sheet.inherited = new WeakMap();
794
1098
  const reasons = new WeakMap();
795
- for (const [attrs, parent] of tree.inherited) {
1099
+ for (const [attrs, parent, code] of tree.inherited) {
796
1100
  let why = reasons.get(parent);
797
1101
  if (why === undefined)
798
1102
  reasons.set(parent, why = hiddenReason(parent, shield.sheet));
799
1103
  if (why)
800
1104
  shield.sheet.inherited.set(attrs, why);
1105
+ visitorScopes.set(attrs, { visitor: elementMark(parent, code, visitorScopes), code });
801
1106
  }
802
1107
  }
803
- const found = scanHead(doc);
1108
+ let found = scanHead(doc, visitorScopes, tree.source, shield.sheet);
804
1109
  let linkBase = baseUrl;
805
1110
  try {
806
1111
  const base = new URL(found.base?.attribs?.href ?? "", baseUrl);
@@ -811,6 +1116,10 @@ export function parseHtml(html, baseUrl, keep, rendered) {
811
1116
  // An unusable first base leaves links relative to the fetched page.
812
1117
  }
813
1118
  const b = new Builder(shield, linkBase);
1119
+ b.visitorScopes = visitorScopes;
1120
+ markWrappers(doc, b);
1121
+ found = scanHead(doc, visitorScopes, tree.source, shield.sheet, sourceHeadings(doc, b));
1122
+ b.headingVisitors = found.headingVisitors;
814
1123
  chooseMain(doc, b);
815
1124
  walk(doc, b);
816
1125
  b.flush();
@@ -850,10 +1159,12 @@ export function parseHtml(html, baseUrl, keep, rendered) {
850
1159
  // parsed, never executed, and phrases inside it are flagged.
851
1160
  const types = new Set();
852
1161
  const rawParts = [];
1162
+ const partSpans = [];
1163
+ const partVisitors = [];
853
1164
  let used = 0;
854
1165
  const skipped = { invalid: 0, too_large: 0 };
855
1166
  for (const s of found.ld) {
856
- const raw = rawText(s, MAX_STRUCTURED).trim();
1167
+ const raw = rawText(s.n, MAX_STRUCTURED).trim();
857
1168
  if (!raw)
858
1169
  continue;
859
1170
  if (used + raw.length > MAX_STRUCTURED) {
@@ -870,15 +1181,23 @@ export function parseHtml(html, baseUrl, keep, rendered) {
870
1181
  // The bounded printer stops at the remaining budget (Codex round 78: printing whole first could use
871
1182
  // far more memory than the cap); a print that does not fit whole keeps the raw text.
872
1183
  let pretty = raw;
1184
+ // W-39: where the visitor values (reviews, comments, answers) are in the text kept. Raw text kept whole is
1185
+ // the visitor's throughout when any value in it is.
1186
+ let spans = [];
1187
+ let printed = false;
873
1188
  try {
874
- const p = boundedPretty(data, Math.max(0, MAX_STRUCTURED - used));
1189
+ const p = boundedPretty(data, Math.max(0, MAX_STRUCTURED - used), spans);
875
1190
  if (!p.omitted && p.exact)
876
- pretty = p.text;
1191
+ (pretty = p.text), (printed = true);
877
1192
  }
878
1193
  catch {
879
1194
  // keep raw
880
1195
  }
1196
+ if (!printed)
1197
+ spans = anyVisitorValue(data) ? [[0, raw.length]] : [];
881
1198
  rawParts.push(pretty);
1199
+ partSpans.push(spans);
1200
+ partVisitors.push(s.visitor);
882
1201
  used += pretty.length;
883
1202
  }
884
1203
  catch {
@@ -890,7 +1209,10 @@ export function parseHtml(html, baseUrl, keep, rendered) {
890
1209
  page.searchable_chars += used;
891
1210
  page.structured_skipped = skipped.invalid || skipped.too_large ? skipped : undefined;
892
1211
  if (rawParts.length) {
893
- page.sections.push({ n: page.sections.length + 1, level: 0, heading: "(structured data)", generated: true, blocks: rawParts.flatMap(lineBlocks) });
1212
+ page.sections.push({ n: page.sections.length + 1, level: 0, heading: STRUCTURED_HEADING, generated: true, blocks: rawParts.flatMap((part, i) => lineBlocks(part, partSpans[i]).map((b) => {
1213
+ const visitor = asLevel(strongest(b.visitor ?? 0, partVisitors[i]));
1214
+ return visitor ? { ...b, visitor } : b;
1215
+ })) });
894
1216
  }
895
1217
  // W-04: the page's forms, as fields. Labels and option texts are returned text, so the shield reads them.
896
1218
  const fx = extractForms(doc, baseUrl, tree.flattened);
@@ -1247,7 +1569,9 @@ export const MAX_PRETTY_DEPTH = 100_000;
1247
1569
  // keeps counting the length of what it does not write, so the omitted count is exact and costs
1248
1570
  // arithmetic, not memory. Returns { text, omitted, exact }: `exact` is false only when the nesting
1249
1571
  // passed MAX_PRETTY_DEPTH, and then `omitted` is a lower bound.
1250
- export function boundedPretty(root, max = MAX_EXPANDED) {
1572
+ // W-39: with `spans`, it also records where each visitor value (visitorValue) starts and ends in the text, outermost
1573
+ // only: a review's whole object, its author included, is the visitor's.
1574
+ export function boundedPretty(root, max = MAX_EXPANDED, spans) {
1251
1575
  const out = [];
1252
1576
  let used = 0; // characters kept
1253
1577
  let total = 0; // characters the full output would have
@@ -1271,22 +1595,32 @@ export function boundedPretty(root, max = MAX_EXPANDED) {
1271
1595
  };
1272
1596
  const stack = [];
1273
1597
  let exact = true;
1274
- const value = (v, depth) => {
1598
+ let open = 0; // W-39: visitor values being printed
1599
+ // A value with no children: written whole, so its span (if it starts one) closes here too.
1600
+ const leaf = (s, span) => {
1601
+ put(s);
1602
+ if (span !== undefined)
1603
+ spans.push([span, used]);
1604
+ };
1605
+ const value = (v, depth, key) => {
1606
+ const span = spans && open === 0 && visitorValue(v, key) ? used : undefined;
1275
1607
  if (typeof v !== "object" || v === null)
1276
- return put(JSON.stringify(v) ?? "null");
1608
+ return leaf(JSON.stringify(v) ?? "null", span);
1277
1609
  if (Array.isArray(v)) {
1278
1610
  if (v.length === 0)
1279
- return put("[]");
1611
+ return leaf("[]", span);
1280
1612
  put("[");
1281
- stack.push({ arr: v, i: 0, n: v.length, depth });
1613
+ stack.push(span === undefined ? { arr: v, i: 0, n: v.length, depth } : { arr: v, i: 0, n: v.length, depth, span });
1282
1614
  }
1283
1615
  else {
1284
1616
  const keys = Object.keys(v);
1285
1617
  if (keys.length === 0)
1286
- return put("{}");
1618
+ return leaf("{}", span);
1287
1619
  put("{");
1288
- stack.push({ obj: v, keys, i: 0, n: keys.length, depth });
1620
+ stack.push(span === undefined ? { obj: v, keys, i: 0, n: keys.length, depth } : { obj: v, keys, i: 0, n: keys.length, depth, span });
1289
1621
  }
1622
+ if (span !== undefined)
1623
+ open++;
1290
1624
  };
1291
1625
  value(root, 0);
1292
1626
  while (stack.length) {
@@ -1300,20 +1634,25 @@ export function boundedPretty(root, max = MAX_EXPANDED) {
1300
1634
  put(",");
1301
1635
  line(f.depth + 1);
1302
1636
  let child;
1637
+ let key;
1303
1638
  if (f.arr)
1304
1639
  child = f.arr[f.i];
1305
1640
  else {
1306
- const k = f.keys[f.i];
1307
- put(JSON.stringify(k) + ": ");
1308
- child = f.obj[k];
1641
+ key = f.keys[f.i];
1642
+ put(JSON.stringify(key) + ": ");
1643
+ child = f.obj[key];
1309
1644
  }
1310
1645
  f.i++;
1311
- value(child, f.depth + 1);
1646
+ value(child, f.depth + 1, key);
1312
1647
  }
1313
1648
  else {
1314
1649
  stack.pop();
1315
1650
  line(f.depth);
1316
1651
  put(f.arr ? "]" : "}");
1652
+ if (f.span !== undefined) {
1653
+ spans.push([f.span, used]);
1654
+ open--;
1655
+ }
1317
1656
  }
1318
1657
  }
1319
1658
  return { text: out.join(""), omitted: Math.max(0, total - max), exact };
@@ -1428,14 +1767,23 @@ export const tokens = (s) => {
1428
1767
  }
1429
1768
  return out;
1430
1769
  };
1770
+ // The label of the section Tiresias makes from a page's JSON-LD.
1771
+ export const STRUCTURED_HEADING = "(structured data)";
1772
+ // A line of structured data longer than this is a copy of the page's text (an article's whole body, often as
1773
+ // escaped HTML), not a fact such as a price or a date. A long line holds many words, so by word overlap alone it
1774
+ // outranked the article's own paragraphs (field list, 2026-11-14). Such a line is ranked after every other
1775
+ // passage: it is still returned when little else matches.
1776
+ export const STRUCTURED_COPY = 500;
1431
1777
  // Plain keyword overlap. Honest and boring on purpose: no AI, no surprises.
1432
1778
  export function topPassages(p, question, k = 5) {
1433
1779
  const q = new Set(tokens(question));
1434
1780
  if (!q.size)
1435
1781
  return [];
1436
1782
  const scored = [];
1783
+ const copies = new Set();
1437
1784
  for (const s of p.sections) {
1438
1785
  const headTokens = new Set(tokens(s.heading));
1786
+ const structured = !!s.generated && s.heading === STRUCTURED_HEADING;
1439
1787
  for (const block of s.blocks) {
1440
1788
  const bt = new Set(tokens(block.text));
1441
1789
  let hits = 0;
@@ -1445,9 +1793,15 @@ export function topPassages(p, question, k = 5) {
1445
1793
  for (const t of q)
1446
1794
  if (headTokens.has(t))
1447
1795
  hits += 0.5; // a matching heading helps a little
1448
- if (hits > 0)
1449
- scored.push({ section: s.n, heading: s.heading, text: block.text, score: Math.round((hits / q.size) * 100) / 100 });
1796
+ if (hits > 0) {
1797
+ const visitor = asLevel(strongest(s.visitor ?? 0, block.visitor ?? 0));
1798
+ const passage = { section: s.n, heading: s.heading, text: block.text, score: Math.round((hits / q.size) * 100) / 100, ...(visitor ? { visitor } : {}) };
1799
+ scored.push(passage);
1800
+ if (structured && block.text.length > STRUCTURED_COPY)
1801
+ copies.add(passage);
1802
+ }
1450
1803
  }
1451
1804
  }
1452
- return scored.sort((a, b) => b.score - a.score || a.section - b.section).slice(0, k);
1805
+ // The page's own text comes before a long line of its structured data (see STRUCTURED_COPY).
1806
+ return scored.sort((a, b) => Number(copies.has(a)) - Number(copies.has(b)) || b.score - a.score || a.section - b.section).slice(0, k);
1453
1807
  }