@raibitsi/tiresias 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +27 -11
- package/data/README.md +33 -0
- package/data/guard-ordinary-urls.txt +102 -0
- package/data/guard-starter.jsonl +424 -0
- package/dist/aimed.js +23 -0
- package/dist/anchor.js +26 -13
- package/dist/browsecli.js +16 -13
- package/dist/cleanup.js +3 -1
- package/dist/cli.js +144 -49
- package/dist/config.js +60 -1
- package/dist/confirm.js +15 -4
- package/dist/dom.js +13 -8
- package/dist/doors.js +1 -1
- package/dist/engine.js +6 -1
- package/dist/envelope.js +10 -1
- package/dist/fetcher.js +24 -2
- package/dist/forms.js +80 -8
- package/dist/getactions.js +28 -8
- package/dist/guard.js +1240 -0
- package/dist/guardclient.js +245 -0
- package/dist/guardnetwork.js +22 -0
- package/dist/history.js +4 -1
- package/dist/hosts.js +24 -17
- package/dist/local.js +537 -0
- package/dist/mcp.js +46 -15
- package/dist/pipeline.js +106 -24
- package/dist/preformatted.js +12 -0
- package/dist/queue.js +20 -18
- package/dist/reader.js +441 -87
- package/dist/receipts.js +5 -0
- package/dist/render.js +65 -15
- package/dist/renderactions.js +69 -0
- package/dist/rules.js +14 -2
- package/dist/safety.js +6 -4
- package/dist/settings.js +5 -2
- package/dist/shield.js +15 -0
- package/dist/textview.js +8 -2
- package/dist/types.js +9 -2
- package/dist/work.js +6 -0
- package/docs/LOCAL-MODELS.md +271 -0
- package/package.json +5 -2
package/dist/reader.js
CHANGED
|
@@ -34,20 +34,176 @@ const BOILER_WORDS = new Set([
|
|
|
34
34
|
"subscription", "sidebar", "breadcrumb", "breadcrumbs", "advert", "advertisement", "ads", "ad", "sponsored", "promo", "popup",
|
|
35
35
|
"modal", "banner", "masthead", "footer", "nav", "navbar", "menu", "comments", "disqus",
|
|
36
36
|
]);
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
// v1.2: never the page's root. A class, id or role on <html> or <body> describes the whole page, not a part of
|
|
38
|
+
// it: Wikipedia's <html> carries "vector-feature-main-menu-pinned-disabled", and the word "menu" there dropped the
|
|
39
|
+
// text of every article.
|
|
40
|
+
const PAGE_ROOT = new Set(["html", "body"]);
|
|
41
|
+
// v1.2: words that name a region of the layout. Sites also use them in the class of the wrapper AROUND the
|
|
42
|
+
// content ("has-sidebar", "no-footer-gap", "with-product-nav__wrapper"), so these words use local surviving
|
|
43
|
+
// text and semantic content to recognize wrappers (markWrappers). The other words (cookie, consent, modal,
|
|
44
|
+
// newsletter and so on) name the thing itself and stay boilerplate however much of a thin page they fill.
|
|
45
|
+
const LAYOUT_WORDS = new Set(["nav", "navbar", "menu", "sidebar", "footer", "banner", "masthead"]);
|
|
46
|
+
// The word that makes this element boilerplate by its class or id, or null.
|
|
47
|
+
function boilerWord(a) {
|
|
40
48
|
const names = `${a.class ?? ""} ${a.id ?? ""}`.slice(0, 500).toLowerCase();
|
|
41
49
|
for (const word of names.split(/[\s_-]+/))
|
|
42
50
|
if (BOILER_WORDS.has(word))
|
|
43
|
-
return
|
|
44
|
-
return
|
|
51
|
+
return word;
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
// Every word in the class and id that names boilerplate is a layout word (so the wrapper rule may apply).
|
|
55
|
+
function onlyLayoutWords(a) {
|
|
56
|
+
const names = `${a.class ?? ""} ${a.id ?? ""}`.slice(0, 500).toLowerCase();
|
|
57
|
+
for (const word of names.split(/[\s_-]+/))
|
|
58
|
+
if (BOILER_WORDS.has(word) && !LAYOUT_WORDS.has(word))
|
|
59
|
+
return false;
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
function boilerplate(a, n, wrappers) {
|
|
63
|
+
const name = n?.name;
|
|
64
|
+
if (name !== undefined && PAGE_ROOT.has(name.toLowerCase()))
|
|
65
|
+
return false;
|
|
66
|
+
if (QUIET_ROLES.has((a.role ?? "").toLowerCase()))
|
|
67
|
+
return true;
|
|
68
|
+
if (boilerWord(a) === null)
|
|
69
|
+
return false;
|
|
70
|
+
return !(n !== undefined && wrappers?.has(n));
|
|
45
71
|
}
|
|
46
72
|
const BLOCK = new Set([
|
|
47
73
|
"p", "div", "section", "article", "li", "ul", "ol", "tr", "table", "blockquote", "pre", "br", "hr", "main",
|
|
48
74
|
"header", "aside", "form", "dl", "dt", "dd", "figure", "figcaption", "details", "summary", "caption",
|
|
49
75
|
]);
|
|
50
76
|
const norm = (s) => s.replace(/\s+/g, " ").trim();
|
|
77
|
+
// ---- W-39 (v1.2): who wrote it ----
|
|
78
|
+
// Text a visitor typed into the page (comments, reviews, answers, forum posts), read from the page's own markup at
|
|
79
|
+
// three levels of confidence. Markup can only ADD the mark: nothing inside a visitor's text can take it away.
|
|
80
|
+
// Level 1, a formal standard: schema.org types and properties (microdata itemtype and itemprop, RDFa typeof and
|
|
81
|
+
// property, and the same names in JSON-LD), and the ARIA role "comment".
|
|
82
|
+
const VISITOR_TYPES = new Set(["comment", "review", "usercomments", "answer", "discussionforumposting", "socialmediaposting"]);
|
|
83
|
+
const VISITOR_PROPS = new Set(["comment", "review", "reviewbody", "commenttext", "suggestedanswer", "acceptedanswer"]);
|
|
84
|
+
// Level 2, a section the page names: a class or id token, or one of its parts split at - and _ (so "preview" and
|
|
85
|
+
// "s-prose" do not count). Forum post bodies count as whole tokens. A token that also names a form, a link, a count
|
|
86
|
+
// or a title is the site's own furniture around the comments ("comment-form", "comments-link", "reply-title"), and
|
|
87
|
+
// counts for nothing. Class names inside code are a highlighter's (class="comment") and count for nothing either.
|
|
88
|
+
const VISITOR_PARTS = new Set(["comments", "comment", "commentlist", "reviews", "review", "replies", "reply", "testimonials", "testimonial", "usernotes"]);
|
|
89
|
+
const VISITOR_TOKENS = new Set(["comment-body", "commtext", "cooked", "post-body", "postbody", "message-body", "bbwrapper", "answercell"]);
|
|
90
|
+
const FURNITURE_PARTS = new Set(["form", "respond", "link", "links", "count", "number", "title", "button", "btn", "icon", "notes", "closed", "policy"]);
|
|
91
|
+
// Level 3, a heading that names visitor text, in English only for now: its following content, up to a sibling heading
|
|
92
|
+
// of the same or a higher rank or the container's end, is visitor text. The whole heading must match, so
|
|
93
|
+
// "Our review process" and "Peer review" do not.
|
|
94
|
+
// A bare "Responses" is left out (API documentation uses it for status codes); "3 Responses to ..." counts.
|
|
95
|
+
const VISITOR_HEADING = new RegExp("^(?:(?:customer|visitor|user|reader|student|guest|member|community|product|verified|recent|latest|top|all)s?'? )?" +
|
|
96
|
+
"(?:comments|replies|testimonials|(?:ratings? (?:and|&) )?reviews(?: (?:and|&) ratings?)?|user contributed notes(?: \\d[\\d,.]* notes?)?)(?: \\(\\d[\\d,.]*\\))?$" +
|
|
97
|
+
"|^(?:\\d[\\d,.]*|one) (?:comments?|reviews?|replies|reply|responses?|thoughts?)(?: (?:on|to|for) .+)?$");
|
|
98
|
+
export function visitorHeading(heading) {
|
|
99
|
+
const text = norm(heading);
|
|
100
|
+
if (text.length > 200)
|
|
101
|
+
return false;
|
|
102
|
+
return VISITOR_HEADING.test(text.toLowerCase().replace(/[‘’]/g, "'").replace(/[:.]$/, ""));
|
|
103
|
+
}
|
|
104
|
+
// The last part of a schema.org name, lower case: "https://schema.org/Review", "schema:Review" and "Review" are one.
|
|
105
|
+
function schemaName(t) {
|
|
106
|
+
let end = t.length;
|
|
107
|
+
while (end > 0 && t[end - 1] === "/")
|
|
108
|
+
end--;
|
|
109
|
+
let start = end;
|
|
110
|
+
while (start > 0 && t[start - 1] !== "/" && t[start - 1] !== "#" && t[start - 1] !== ":")
|
|
111
|
+
start--;
|
|
112
|
+
return t.slice(start, end).toLowerCase();
|
|
113
|
+
}
|
|
114
|
+
// Scan all attribute tokens, without allocating an array proportional to a hostile attribute's word count.
|
|
115
|
+
function* words(v) {
|
|
116
|
+
if (v)
|
|
117
|
+
for (const m of v.matchAll(/\S+/g))
|
|
118
|
+
yield m[0];
|
|
119
|
+
}
|
|
120
|
+
// The level an element's own attributes give its text (1 or 2), or 0.
|
|
121
|
+
function visitorMark(a, inCode) {
|
|
122
|
+
for (const key of ["itemtype", "typeof"])
|
|
123
|
+
for (const t of words(a[key]))
|
|
124
|
+
if (VISITOR_TYPES.has(schemaName(t)))
|
|
125
|
+
return 1;
|
|
126
|
+
for (const key of ["itemprop", "property"])
|
|
127
|
+
for (const p of words(a[key]))
|
|
128
|
+
if (VISITOR_PROPS.has(schemaName(p)))
|
|
129
|
+
return 1;
|
|
130
|
+
for (const r of words(a.role))
|
|
131
|
+
if (r.toLowerCase() === "comment")
|
|
132
|
+
return 1;
|
|
133
|
+
if (inCode)
|
|
134
|
+
return 0;
|
|
135
|
+
for (const key of ["class", "id"])
|
|
136
|
+
for (const token of words(a[key])) {
|
|
137
|
+
const t = token.toLowerCase();
|
|
138
|
+
if (t.length <= 32 && VISITOR_TOKENS.has(t.replace(/_/g, "-")))
|
|
139
|
+
return 2;
|
|
140
|
+
let visitor = false;
|
|
141
|
+
let furniture = false;
|
|
142
|
+
for (const m of t.matchAll(/[^-_]+/g)) {
|
|
143
|
+
if (FURNITURE_PARTS.has(m[0])) {
|
|
144
|
+
furniture = true;
|
|
145
|
+
break;
|
|
146
|
+
}
|
|
147
|
+
if (VISITOR_PARTS.has(m[0]))
|
|
148
|
+
visitor = true;
|
|
149
|
+
}
|
|
150
|
+
if (visitor && !furniture)
|
|
151
|
+
return 2;
|
|
152
|
+
}
|
|
153
|
+
return 0;
|
|
154
|
+
}
|
|
155
|
+
// Source ancestry survives the bounded DOM's forced closes, including code's highlighter exception.
|
|
156
|
+
function elementMark(a, inCode, scopes) {
|
|
157
|
+
let inherited = scopes.get(a);
|
|
158
|
+
if (!inherited)
|
|
159
|
+
scopes.set(a, inherited = { visitor: 0, code: false });
|
|
160
|
+
const key = inCode || inherited.code ? "codeMark" : "normalMark";
|
|
161
|
+
// A flattened parent can have thousands of children. Its long attributes are scanned once per code context.
|
|
162
|
+
const own = inherited[key] ?? (inherited[key] = visitorMark(a, key === "codeMark"));
|
|
163
|
+
return strongest(inherited.visitor, own);
|
|
164
|
+
}
|
|
165
|
+
// Any visitor value anywhere in parsed JSON (walked without recursion; a 5 MB document is the most it can be).
|
|
166
|
+
function anyVisitorValue(root) {
|
|
167
|
+
const stack = [[root, undefined]];
|
|
168
|
+
while (stack.length) {
|
|
169
|
+
const [v, key] = stack.pop();
|
|
170
|
+
if (visitorValue(v, key))
|
|
171
|
+
return true;
|
|
172
|
+
if (Array.isArray(v))
|
|
173
|
+
for (const c of v)
|
|
174
|
+
stack.push([c, undefined]);
|
|
175
|
+
else if (v && typeof v === "object")
|
|
176
|
+
for (const [k, c] of Object.entries(v))
|
|
177
|
+
stack.push([c, k]);
|
|
178
|
+
}
|
|
179
|
+
return false;
|
|
180
|
+
}
|
|
181
|
+
// The stronger of two levels (1 is the strongest; 0 is none).
|
|
182
|
+
const strongest = (x, y) => (x && y ? Math.min(x, y) : x || y);
|
|
183
|
+
const asLevel = (n) => (n === 1 || n === 2 || n === 3 ? n : undefined);
|
|
184
|
+
// A JSON-LD value a visitor wrote: one under a visitor property, or an object of a visitor type.
|
|
185
|
+
function visitorValue(v, key) {
|
|
186
|
+
if (key !== undefined && VISITOR_PROPS.has(schemaName(key)))
|
|
187
|
+
return true;
|
|
188
|
+
return !!v && typeof v === "object" && !Array.isArray(v) && typeOf(v).some((t) => VISITOR_TYPES.has(schemaName(t)));
|
|
189
|
+
}
|
|
190
|
+
// v1.2: extraction and scoring agree about destinations, independently of the link-list cap.
|
|
191
|
+
// Keep the overlong case separate so extraction can still report omitted URLs.
|
|
192
|
+
function linkDestination(href, base) {
|
|
193
|
+
if (!href)
|
|
194
|
+
return null;
|
|
195
|
+
let u;
|
|
196
|
+
try {
|
|
197
|
+
u = new URL(href.trim(), base);
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
return null;
|
|
201
|
+
}
|
|
202
|
+
if (u.protocol !== "http:" && u.protocol !== "https:")
|
|
203
|
+
return null;
|
|
204
|
+
u.hash = "";
|
|
205
|
+
return u.href.length > MAX_URL ? { tooLong: true } : { url: u.href };
|
|
206
|
+
}
|
|
51
207
|
// Collapse whitespace while tracking where link markers land in the result.
|
|
52
208
|
function collapse(parts) {
|
|
53
209
|
let text = "";
|
|
@@ -93,7 +249,7 @@ function splitBlock(b) {
|
|
|
93
249
|
end = splitEnd(b.text, end);
|
|
94
250
|
const lo = start;
|
|
95
251
|
const marks = b.marks.filter(([pos]) => (pos > lo || lo === 0) && pos <= end).map(([pos, n]) => [pos - lo, n]);
|
|
96
|
-
out.push({ text: b.text.slice(start, end), marks });
|
|
252
|
+
out.push({ text: b.text.slice(start, end), marks, ...(b.visitor ? { visitor: b.visitor } : {}) });
|
|
97
253
|
start = end;
|
|
98
254
|
while (b.text[start] === " ")
|
|
99
255
|
start++;
|
|
@@ -126,10 +282,26 @@ class Builder {
|
|
|
126
282
|
main = null; // W-22: the main content's element and its ancestors, when one was found
|
|
127
283
|
inMain = new Set(); // every node inside the main content element
|
|
128
284
|
parentOf = new Map();
|
|
285
|
+
wrappers = new Set(); // v1.2: layout-named elements kept by local surviving text (markWrappers)
|
|
286
|
+
// W-39: the level of the visitor markup around the node being read (0: none), the strongest level among the text
|
|
287
|
+
// waiting in `parts`, the source heading regions, and code depth.
|
|
288
|
+
visitor = 0;
|
|
289
|
+
partsVisitor = 0;
|
|
290
|
+
headingVisitors = new WeakSet(); // v1.2: source heading scopes, independent of main-content selection
|
|
291
|
+
visitorScopes = new WeakMap();
|
|
292
|
+
code = 0;
|
|
129
293
|
constructor(shield, base) {
|
|
130
294
|
this.shield = shield;
|
|
131
295
|
this.base = base;
|
|
132
296
|
}
|
|
297
|
+
// W-39: the level a block made now gets: its own text's, or the heading region's.
|
|
298
|
+
by(own) {
|
|
299
|
+
return asLevel(own);
|
|
300
|
+
}
|
|
301
|
+
take() {
|
|
302
|
+
this.parts = [];
|
|
303
|
+
this.partsVisitor = 0;
|
|
304
|
+
}
|
|
133
305
|
// A preformatted block, kept exactly, fenced.
|
|
134
306
|
// The pending marker of a list item, taken to lead a block written directly (a <pre>: round 79).
|
|
135
307
|
// Outside an item, or for an item already led, the continuation indent (round 80).
|
|
@@ -138,7 +310,7 @@ class Builder {
|
|
|
138
310
|
if (!this.loneMarker())
|
|
139
311
|
return item?.led ? item.indent : "";
|
|
140
312
|
const m = this.parts.filter((p) => typeof p === "string").join("").replace(/\s+$/, ""); // nesting indent kept
|
|
141
|
-
this.
|
|
313
|
+
this.take();
|
|
142
314
|
this.marker = false;
|
|
143
315
|
if (item)
|
|
144
316
|
item.led = true;
|
|
@@ -146,7 +318,7 @@ class Builder {
|
|
|
146
318
|
}
|
|
147
319
|
// `indent` goes in front of EVERY line of the block, its fences included, so a <pre> inside a list item
|
|
148
320
|
// stays inside it (Codex round 81); the first chunk starts with `lead` (the item's marker) instead.
|
|
149
|
-
pre(raw, marks = [], lead = "", indent = "") {
|
|
321
|
+
pre(raw, marks = [], lead = "", indent = "", visitor = this.visitor) {
|
|
150
322
|
if (!this.cur)
|
|
151
323
|
this.startTop();
|
|
152
324
|
for (let i = 0; i < raw.length;) {
|
|
@@ -166,7 +338,8 @@ class Builder {
|
|
|
166
338
|
const lines = indent ? piece.slice(0, p).split("\n").length : 0; // lines up to the mark, each indented
|
|
167
339
|
return [head.length + fence.length + 1 + p + indent.length * lines, n];
|
|
168
340
|
});
|
|
169
|
-
this.
|
|
341
|
+
const by = this.by(visitor);
|
|
342
|
+
this.cur.blocks.push({ text: `${head}${fence}\n${body}\n${indent}${fence}`, marks: m, ...(by ? { visitor: by } : {}) });
|
|
170
343
|
this.shield.scan(piece, "visible text");
|
|
171
344
|
i = end;
|
|
172
345
|
}
|
|
@@ -177,14 +350,18 @@ class Builder {
|
|
|
177
350
|
this.start(0, "(top of page)");
|
|
178
351
|
this.cur.generated = true;
|
|
179
352
|
}
|
|
180
|
-
start(level, heading) {
|
|
353
|
+
start(level, heading, visitor = 0) {
|
|
181
354
|
this.shield.scan(heading, "heading"); // headings are returned text too
|
|
182
355
|
// The heading shown in the outline is bounded. The full text is not lost:
|
|
183
356
|
// an overlong heading also becomes the section's first block, so it can
|
|
184
357
|
// be expanded, searched and quoted.
|
|
185
358
|
this.cur = { n: this.sections.length + 1, level, heading: heading.slice(0, MAX_HEADING), blocks: [] };
|
|
359
|
+
if (visitor)
|
|
360
|
+
this.cur.visitor = asLevel(visitor); // the heading itself, independently of the following blocks
|
|
361
|
+
// W-39: a visitor's overlong heading stays the visitor's.
|
|
362
|
+
const by = this.by(visitor);
|
|
186
363
|
if (heading.length > MAX_HEADING)
|
|
187
|
-
this.cur.blocks.push(...splitBlock({ text: heading, marks: [] }));
|
|
364
|
+
this.cur.blocks.push(...splitBlock({ text: heading, marks: [], ...(by ? { visitor: by } : {}) }));
|
|
188
365
|
this.sections.push(this.cur);
|
|
189
366
|
}
|
|
190
367
|
marker = false; // a list item's marker was pushed and has not yet led any content
|
|
@@ -193,7 +370,7 @@ class Builder {
|
|
|
193
370
|
}
|
|
194
371
|
dropLoneMarker() {
|
|
195
372
|
if (this.loneMarker())
|
|
196
|
-
this.
|
|
373
|
+
this.take();
|
|
197
374
|
this.marker = false;
|
|
198
375
|
}
|
|
199
376
|
items = []; // open list items: their continuation indent (round 79)
|
|
@@ -216,7 +393,11 @@ class Builder {
|
|
|
216
393
|
b.text = indent + b.text;
|
|
217
394
|
b.marks = b.marks.map(([pos, n]) => [pos + indent.length, n]);
|
|
218
395
|
}
|
|
219
|
-
|
|
396
|
+
// W-39: a block holding any visitor text is the visitor's (a mixed paragraph errs toward the visitor).
|
|
397
|
+
const by = this.by(this.partsVisitor);
|
|
398
|
+
if (by)
|
|
399
|
+
b.visitor = by;
|
|
400
|
+
this.take();
|
|
220
401
|
if (!b.text)
|
|
221
402
|
return;
|
|
222
403
|
if (!this.cur)
|
|
@@ -225,12 +406,12 @@ class Builder {
|
|
|
225
406
|
// first get the item's continuation indent.
|
|
226
407
|
const pieces = splitBlock(b);
|
|
227
408
|
const cont = item ? item.indent : "";
|
|
228
|
-
this.cur.blocks.push(...pieces.map((pc, k) => k > 0 && cont ? { text: cont + pc.text, marks: pc.marks.map(([pos, n]) => [pos + cont.length, n]) } : pc));
|
|
409
|
+
this.cur.blocks.push(...pieces.map((pc, k) => k > 0 && cont ? { ...pc, text: cont + pc.text, marks: pc.marks.map(([pos, n]) => [pos + cont.length, n]) } : pc));
|
|
229
410
|
this.shield.scan(b.text, "visible text");
|
|
230
411
|
}
|
|
231
412
|
takeHeading() {
|
|
232
413
|
const t = collapse(this.parts.filter((p) => typeof p === "string")).text;
|
|
233
|
-
this.
|
|
414
|
+
this.take();
|
|
234
415
|
// A heading that took a pending list marker leads its item (round 83).
|
|
235
416
|
if (this.marker) {
|
|
236
417
|
this.marker = false;
|
|
@@ -246,7 +427,7 @@ class Builder {
|
|
|
246
427
|
if (!this.loneMarker())
|
|
247
428
|
return;
|
|
248
429
|
const m = this.parts.filter((p) => typeof p === "string").join("").replace(/\s+$/, "");
|
|
249
|
-
this.
|
|
430
|
+
this.take();
|
|
250
431
|
this.marker = false;
|
|
251
432
|
const item = this.items[this.items.length - 1];
|
|
252
433
|
if (item)
|
|
@@ -257,23 +438,15 @@ class Builder {
|
|
|
257
438
|
}
|
|
258
439
|
// Returns the link number, or 0 if it is not a followable link or the cap is hit.
|
|
259
440
|
link(href, text) {
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
let u;
|
|
263
|
-
try {
|
|
264
|
-
u = new URL(href.trim(), this.base);
|
|
265
|
-
}
|
|
266
|
-
catch {
|
|
267
|
-
return 0;
|
|
268
|
-
}
|
|
269
|
-
if (u.protocol !== "http:" && u.protocol !== "https:")
|
|
441
|
+
const destination = linkDestination(href, this.base);
|
|
442
|
+
if (!destination)
|
|
270
443
|
return 0;
|
|
271
|
-
|
|
272
|
-
if (u.href.length > MAX_URL) {
|
|
444
|
+
if ("tooLong" in destination) {
|
|
273
445
|
this.omitted++; // a shortened URL is a different destination, so it is left out and counted
|
|
274
446
|
return 0;
|
|
275
447
|
}
|
|
276
|
-
const
|
|
448
|
+
const url = destination.url;
|
|
449
|
+
const known = this.byUrl.get(url);
|
|
277
450
|
if (known)
|
|
278
451
|
return known;
|
|
279
452
|
if (this.links.length >= MAX_LINKS)
|
|
@@ -281,8 +454,8 @@ class Builder {
|
|
|
281
454
|
const n = this.links.length + 1;
|
|
282
455
|
const label = snippet(text, 0, 80);
|
|
283
456
|
this.shield.scan(label, "link text");
|
|
284
|
-
this.links.push({ n, url
|
|
285
|
-
this.byUrl.set(
|
|
457
|
+
this.links.push({ n, url, text: label });
|
|
458
|
+
this.byUrl.set(url, n);
|
|
286
459
|
return n;
|
|
287
460
|
}
|
|
288
461
|
}
|
|
@@ -308,7 +481,8 @@ function visibleText(root, cap = 300, sheet) {
|
|
|
308
481
|
// Links inside it are collected like anywhere else and get their [n] marker (Codex round 76).
|
|
309
482
|
function preText(root, b, cap, marks = []) {
|
|
310
483
|
let out = "";
|
|
311
|
-
|
|
484
|
+
let visitor = 0;
|
|
485
|
+
const stack = [...(root.children ?? [])].reverse().map((n) => ({ n, quiet: false, visitor: b.visitor }));
|
|
312
486
|
let seen = 0;
|
|
313
487
|
while (stack.length && out.length < cap) {
|
|
314
488
|
const item = stack.pop();
|
|
@@ -323,8 +497,11 @@ function preText(root, b, cap, marks = []) {
|
|
|
323
497
|
const { n } = item;
|
|
324
498
|
let quiet = item.quiet;
|
|
325
499
|
if (n.type === "text") {
|
|
326
|
-
if (!quiet)
|
|
500
|
+
if (!quiet) {
|
|
327
501
|
out += n.data ?? "";
|
|
502
|
+
if (/\S/.test(n.data ?? ""))
|
|
503
|
+
visitor = strongest(visitor, strongest(item.visitor, b.headingVisitors.has(n) ? 3 : 0));
|
|
504
|
+
}
|
|
328
505
|
continue;
|
|
329
506
|
}
|
|
330
507
|
if (n.type === "comment") {
|
|
@@ -343,7 +520,7 @@ function preText(root, b, cap, marks = []) {
|
|
|
343
520
|
b.shield.hidden(why, txt);
|
|
344
521
|
continue;
|
|
345
522
|
}
|
|
346
|
-
if (QUIET.has(n.name) || boilerplate(attribs))
|
|
523
|
+
if (QUIET.has(n.name) || boilerplate(attribs, n, b.wrappers))
|
|
347
524
|
quiet = true;
|
|
348
525
|
if (n.name === "br" && !quiet)
|
|
349
526
|
out += "\n";
|
|
@@ -352,10 +529,11 @@ function preText(root, b, cap, marks = []) {
|
|
|
352
529
|
if (num && !quiet)
|
|
353
530
|
stack.push({ mark: num });
|
|
354
531
|
}
|
|
532
|
+
const mark = strongest(item.visitor, elementMark(attribs, true, b.visitorScopes));
|
|
355
533
|
for (let i = (n.children ?? []).length - 1; i >= 0; i--)
|
|
356
|
-
stack.push({ n: n.children[i], quiet });
|
|
534
|
+
stack.push({ n: n.children[i], quiet, visitor: mark });
|
|
357
535
|
}
|
|
358
|
-
return out;
|
|
536
|
+
return { text: out, visitor };
|
|
359
537
|
}
|
|
360
538
|
function rawText(root, cap = 5000) {
|
|
361
539
|
let out = "";
|
|
@@ -374,9 +552,9 @@ function rawText(root, cap = 5000) {
|
|
|
374
552
|
function pushChildren(stack, n, quiet) {
|
|
375
553
|
const ch = n.children ?? [];
|
|
376
554
|
for (let i = ch.length - 1; i >= 0; i--)
|
|
377
|
-
stack.push({ n: ch[i], quiet });
|
|
555
|
+
stack.push({ n: ch[i], quiet, parent: n });
|
|
378
556
|
}
|
|
379
|
-
function walk(root, b) {
|
|
557
|
+
function walk(root, b, headings) {
|
|
380
558
|
const stack = [{ n: root, quiet: false }];
|
|
381
559
|
let count = 0;
|
|
382
560
|
while (stack.length) {
|
|
@@ -391,8 +569,11 @@ function walk(root, b) {
|
|
|
391
569
|
let quiet = t.quiet;
|
|
392
570
|
if (n.type === "text") {
|
|
393
571
|
// A "|" inside a table cell is escaped, so only Tiresias's own separators split cells (round 78).
|
|
394
|
-
if (!quiet)
|
|
572
|
+
if (!quiet) {
|
|
395
573
|
b.parts.push(b.rowDepth > 0 ? (n.data ?? "").replace(/\\/g, "\\\\").replace(/\|/g, "\\|") : n.data ?? "");
|
|
574
|
+
if (/\S/.test(n.data ?? ""))
|
|
575
|
+
b.partsVisitor = strongest(b.partsVisitor, strongest(b.visitor, b.headingVisitors.has(n) ? 3 : 0)); // W-39
|
|
576
|
+
}
|
|
396
577
|
continue;
|
|
397
578
|
}
|
|
398
579
|
if (n.type === "comment") {
|
|
@@ -417,8 +598,20 @@ function walk(root, b) {
|
|
|
417
598
|
b.shield.scan(attribs[a], `${a} attribute`);
|
|
418
599
|
if (name === "script" || SKIP.has(name))
|
|
419
600
|
continue;
|
|
420
|
-
if (QUIET.has(name) || boilerplate(attribs) || (b.main !== null && !b.main.has(n) && !b.inMain.has(n)))
|
|
601
|
+
if (QUIET.has(name) || boilerplate(attribs, n, b.wrappers) || (b.main !== null && !b.main.has(n) && !b.inMain.has(n)))
|
|
421
602
|
quiet = true;
|
|
603
|
+
// W-39: this element's own visitor mark holds for everything inside it, until its end (the restore runs after
|
|
604
|
+
// its children and their own closing steps). Class names on or inside code and pre are a highlighter's.
|
|
605
|
+
if (name === "code" || name === "pre") {
|
|
606
|
+
b.code++;
|
|
607
|
+
stack.push(() => void b.code--);
|
|
608
|
+
}
|
|
609
|
+
const mark = elementMark(attribs, b.code > 0, b.visitorScopes);
|
|
610
|
+
if (mark && strongest(b.visitor, mark) !== b.visitor) {
|
|
611
|
+
const outer = b.visitor;
|
|
612
|
+
b.visitor = strongest(outer, mark);
|
|
613
|
+
stack.push(() => void (b.visitor = outer));
|
|
614
|
+
}
|
|
422
615
|
// Inside a table row, block elements (paragraphs, lists, headings, pre, nested rows) do not end the
|
|
423
616
|
// row: their text joins the cell (Codex round 76).
|
|
424
617
|
const inRow = b.rowDepth > 0;
|
|
@@ -429,12 +622,12 @@ function walk(root, b) {
|
|
|
429
622
|
const marks = [];
|
|
430
623
|
const full = preText(n, b, Infinity, marks);
|
|
431
624
|
let lead = 0;
|
|
432
|
-
while (full.charCodeAt(lead) === 10)
|
|
625
|
+
while (full.text.charCodeAt(lead) === 10)
|
|
433
626
|
lead++; // leading newlines, linearly (Codex round 75: the old regex was quadratic)
|
|
434
|
-
const raw = full.slice(lead).trimEnd();
|
|
627
|
+
const raw = full.text.slice(lead).trimEnd();
|
|
435
628
|
const itemIndent = b.items.length ? b.items[b.items.length - 1].indent : "";
|
|
436
629
|
if (raw.trim())
|
|
437
|
-
b.pre(raw, marks.map(([pos, num]) => [Math.min(pos - lead, raw.length), num]), b.takeLoneMarker(), itemIndent);
|
|
630
|
+
b.pre(raw, marks.map(([pos, num]) => [Math.min(pos - lead, raw.length), num]), b.takeLoneMarker(), itemIndent, strongest(b.visitor, full.visitor));
|
|
438
631
|
continue;
|
|
439
632
|
}
|
|
440
633
|
if (!quiet && (name === "ul" || name === "ol")) {
|
|
@@ -509,9 +702,11 @@ function walk(root, b) {
|
|
|
509
702
|
b.flush();
|
|
510
703
|
const level = Number(name[1]);
|
|
511
704
|
stack.push(() => {
|
|
705
|
+
const by = b.partsVisitor;
|
|
512
706
|
const heading = b.takeHeading();
|
|
707
|
+
headings?.set(n, { text: heading, visitor: by });
|
|
513
708
|
if (heading)
|
|
514
|
-
b.start(level, heading);
|
|
709
|
+
b.start(level, heading, by);
|
|
515
710
|
});
|
|
516
711
|
pushChildren(stack, n, quiet);
|
|
517
712
|
continue;
|
|
@@ -536,6 +731,19 @@ function walk(root, b) {
|
|
|
536
731
|
pushChildren(stack, n, quiet);
|
|
537
732
|
}
|
|
538
733
|
}
|
|
734
|
+
// v1.2: use the extraction walk before main selection can quiet a source heading. Both ends of a
|
|
735
|
+
// region use its joined, normalised text and visible descendants' attribution. The separate builder
|
|
736
|
+
// and shield discard this pass's body and flags; only the heading summaries survive into the final walk.
|
|
737
|
+
function sourceHeadings(root, b) {
|
|
738
|
+
const shield = new Shield();
|
|
739
|
+
shield.sheet = b.shield.sheet;
|
|
740
|
+
const reader = new Builder(shield, b.base);
|
|
741
|
+
reader.visitorScopes = b.visitorScopes;
|
|
742
|
+
reader.wrappers = b.wrappers;
|
|
743
|
+
const headings = new WeakMap();
|
|
744
|
+
walk(root, reader, headings);
|
|
745
|
+
return headings;
|
|
746
|
+
}
|
|
539
747
|
// W-22 (v0.5.1): main-content scoring, in-house (no library). Every element's visible text and link text
|
|
540
748
|
// are summed in one pass from the leaves up. The main content is a <main> (or role=main) element with real
|
|
541
749
|
// text; else the <article> with the most text; else the container whose text, less its link text, is
|
|
@@ -544,29 +752,94 @@ function walk(root, b) {
|
|
|
544
752
|
// What the scoring never counts (Codex round 69): what is not rendered, what is hidden (the same rules as
|
|
545
753
|
// the walk, the page's own style rules included), and boilerplate. So invisible filler cannot win.
|
|
546
754
|
const unscored = (n, b) => (n.type === "tag" || n.type === "script" || n.type === "style") &&
|
|
547
|
-
(SKIP.has(n.name) || n.name === "script" || QUIET.has(n.name) || boilerplate(n.attribs ?? {}) || hiddenReason(n.attribs ?? {}, b.shield.sheet) !== null);
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
755
|
+
(SKIP.has(n.name) || n.name === "script" || QUIET.has(n.name) || boilerplate(n.attribs ?? {}, n, b.wrappers) || hiddenReason(n.attribs ?? {}, b.shield.sheet) !== null);
|
|
756
|
+
// v1.2: a bounded preorder for both scoring passes. Link membership belongs to text nodes and is inherited
|
|
757
|
+
// from any enclosing destination anchor, including one outside the element being scored.
|
|
758
|
+
function textOrder(root, b, dead, parent) {
|
|
551
759
|
const order = [];
|
|
552
|
-
const
|
|
760
|
+
const linked = new Set();
|
|
761
|
+
const stack = [{ n: root, inLink: false }];
|
|
553
762
|
let count = 0;
|
|
554
763
|
while (stack.length) {
|
|
555
|
-
const n = stack.pop();
|
|
764
|
+
const { n, inLink } = stack.pop();
|
|
556
765
|
if (++count > MAX_NODES)
|
|
557
766
|
throw new ComplexityError();
|
|
558
767
|
order.push(n);
|
|
559
|
-
if (
|
|
768
|
+
if (dead(n))
|
|
560
769
|
continue;
|
|
770
|
+
if (n.type === "text" && inLink)
|
|
771
|
+
linked.add(n);
|
|
772
|
+
const destination = n.name === "a" ? linkDestination(n.attribs?.href, b.base) : null;
|
|
773
|
+
const nextLink = inLink || (destination !== null && "url" in destination);
|
|
561
774
|
for (const c of n.children ?? []) {
|
|
562
|
-
|
|
563
|
-
stack.push(c);
|
|
775
|
+
parent?.set(c, n);
|
|
776
|
+
stack.push({ n: c, inLink: nextLink });
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
return { order, linked };
|
|
780
|
+
}
|
|
781
|
+
// v1.2: layout-named wrappers are decided locally, children before parents. Rejected children contribute
|
|
782
|
+
// nothing. A semantic main, a substantial article, or a wrapper containing substantial semantic content is
|
|
783
|
+
// kept even inside one big link; otherwise substantial text must be at most half destination-link text.
|
|
784
|
+
export const WRAPPER_MIN = 200;
|
|
785
|
+
export const WRAPPER_MAX_LINK_SHARE = 0.5; // a candidate whose text is mostly link text is the navigation itself
|
|
786
|
+
function markWrappers(root, b) {
|
|
787
|
+
// Counted as nothing, and not entered: what the reader never shows (skipped, script, hidden), and what it always
|
|
788
|
+
// quiets whatever this pass decides: nav, footer, aside, the quiet roles (Codex round 1, LOW), and an element
|
|
789
|
+
// named by a word that is not a layout word, such as a cookie box (Codex round 3: its text is dropped later, so
|
|
790
|
+
// counting it here would let it dilute a menu's share of links).
|
|
791
|
+
const dead = (n) => (n.type === "tag" || n.type === "script" || n.type === "style") &&
|
|
792
|
+
(SKIP.has(n.name) ||
|
|
793
|
+
n.name === "script" ||
|
|
794
|
+
QUIET.has(n.name) ||
|
|
795
|
+
(!PAGE_ROOT.has(n.name) && (QUIET_ROLES.has((n.attribs?.role ?? "").toLowerCase()) || !onlyLayoutWords(n.attribs ?? {}))) ||
|
|
796
|
+
hiddenReason(n.attribs ?? {}, b.shield.sheet) !== null);
|
|
797
|
+
const main = (n) => n.name === "main" || (n.attribs?.role ?? "").toLowerCase() === "main";
|
|
798
|
+
const semantic = (n) => main(n) || n.name === "article";
|
|
799
|
+
const { order, linked } = textOrder(root, b, dead);
|
|
800
|
+
const text = new Map();
|
|
801
|
+
const link = new Map(); // of that text, how much sits inside links
|
|
802
|
+
const holdsMain = new Set(); // contains surviving semantic content with at least WRAPPER_MIN text
|
|
803
|
+
for (let i = order.length - 1; i >= 0; i--) {
|
|
804
|
+
const n = order[i];
|
|
805
|
+
let t = n.type === "text" ? norm(n.data ?? "").length : 0;
|
|
806
|
+
let l = linked.has(n) ? t : 0;
|
|
807
|
+
let holds = false;
|
|
808
|
+
if (!dead(n)) {
|
|
809
|
+
for (const c of n.children ?? []) {
|
|
810
|
+
t += text.get(c) ?? 0;
|
|
811
|
+
l += link.get(c) ?? 0;
|
|
812
|
+
if (holdsMain.has(c))
|
|
813
|
+
holds = true;
|
|
814
|
+
}
|
|
564
815
|
}
|
|
816
|
+
else
|
|
817
|
+
t = 0;
|
|
818
|
+
if (n.type === "tag" && !dead(n) && !PAGE_ROOT.has(n.name) && boilerWord(n.attribs ?? {}) !== null) {
|
|
819
|
+
const itself = main(n) || (n.name === "article" && t >= WRAPPER_MIN);
|
|
820
|
+
const bySize = t >= WRAPPER_MIN && l <= t * WRAPPER_MAX_LINK_SHARE;
|
|
821
|
+
if (itself || holds || bySize)
|
|
822
|
+
b.wrappers.add(n);
|
|
823
|
+
else {
|
|
824
|
+
t = 0;
|
|
825
|
+
l = 0;
|
|
826
|
+
holds = false;
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
text.set(n, t);
|
|
830
|
+
link.set(n, l);
|
|
831
|
+
if (holds || (semantic(n) && t >= WRAPPER_MIN))
|
|
832
|
+
holdsMain.add(n);
|
|
565
833
|
}
|
|
834
|
+
}
|
|
835
|
+
function chooseMain(root, b) {
|
|
836
|
+
const text = new Map();
|
|
837
|
+
const link = new Map();
|
|
838
|
+
const { order, linked } = textOrder(root, b, (n) => unscored(n, b), b.parentOf);
|
|
566
839
|
for (let i = order.length - 1; i >= 0; i--) {
|
|
567
840
|
const n = order[i];
|
|
568
841
|
let t = n.type === "text" ? norm(n.data ?? "").length : 0;
|
|
569
|
-
let l = 0;
|
|
842
|
+
let l = linked.has(n) ? t : 0;
|
|
570
843
|
if (unscored(n, b))
|
|
571
844
|
t = 0;
|
|
572
845
|
else
|
|
@@ -574,8 +847,6 @@ function chooseMain(root, b) {
|
|
|
574
847
|
t += text.get(c) ?? 0;
|
|
575
848
|
l += link.get(c) ?? 0;
|
|
576
849
|
}
|
|
577
|
-
if (n.name === "a")
|
|
578
|
-
l = t;
|
|
579
850
|
text.set(n, t);
|
|
580
851
|
link.set(n, l);
|
|
581
852
|
}
|
|
@@ -641,18 +912,38 @@ function chooseMain(root, b) {
|
|
|
641
912
|
st.push(c);
|
|
642
913
|
}
|
|
643
914
|
}
|
|
644
|
-
function scanHead(root) {
|
|
645
|
-
const found = { ld: [], links: [], metas: [] };
|
|
646
|
-
const stack = [root];
|
|
915
|
+
function scanHead(root, scopes = new WeakMap(), source = new WeakMap(), sheet, headings) {
|
|
916
|
+
const found = { ld: [], links: [], metas: [], headingVisitors: new WeakSet() };
|
|
917
|
+
const stack = [{ n: root, visitor: 0, code: false, headingHidden: false }];
|
|
918
|
+
let region;
|
|
647
919
|
let count = 0;
|
|
648
920
|
while (stack.length) {
|
|
649
|
-
const
|
|
921
|
+
const item = stack.pop();
|
|
922
|
+
const { n } = item;
|
|
650
923
|
if (++count > MAX_NODES)
|
|
651
924
|
throw new ComplexityError();
|
|
925
|
+
if (region && (n.startIndex ?? -1) >= region.end)
|
|
926
|
+
region = undefined;
|
|
927
|
+
const code = item.code || n.name === "code" || n.name === "pre";
|
|
928
|
+
const visitor = strongest(item.visitor, elementMark(n.attribs ?? {}, code, scopes));
|
|
929
|
+
const headingHidden = item.headingHidden || SKIP.has(n.name ?? "") || hiddenReason(n.attribs ?? {}, sheet) !== null;
|
|
930
|
+
// v1.2: source headings identify visitor text before main-content selection can quiet them.
|
|
931
|
+
// The same regions apply to ordinary text and to JSON-LD, including flattened source containers.
|
|
932
|
+
const heading = headings?.get(n);
|
|
933
|
+
if (!headingHidden && !visitor && !code && heading?.text && !heading.visitor) {
|
|
934
|
+
const level = Number(n.name[1]);
|
|
935
|
+
const parent = source.get(n.attribs ?? {})?.parent ?? root;
|
|
936
|
+
if (region?.container === parent && level <= region.level)
|
|
937
|
+
region = undefined;
|
|
938
|
+
if (!region && visitorHeading(heading.text))
|
|
939
|
+
region = { level, container: parent, start: source.get(n.attribs ?? {})?.end ?? Infinity, end: source.get(parent)?.end ?? Infinity };
|
|
940
|
+
}
|
|
941
|
+
if (region && (n.startIndex ?? -1) >= region.start)
|
|
942
|
+
found.headingVisitors.add(n);
|
|
652
943
|
if (n.name === "title" && !found.title)
|
|
653
944
|
found.title = n;
|
|
654
945
|
else if (n.name === "script" && /ld\+json/i.test(n.attribs?.type ?? ""))
|
|
655
|
-
found.ld.push(n);
|
|
946
|
+
found.ld.push({ n, visitor: strongest(visitor, region ? 3 : 0) });
|
|
656
947
|
else if (n.name === "base" && !found.base && n.attribs?.href !== undefined)
|
|
657
948
|
found.base = n;
|
|
658
949
|
else if (n.name === "link")
|
|
@@ -667,7 +958,7 @@ function scanHead(root) {
|
|
|
667
958
|
found.byline = n;
|
|
668
959
|
if (n.children)
|
|
669
960
|
for (let i = n.children.length - 1; i >= 0; i--)
|
|
670
|
-
stack.push(n.children[i]);
|
|
961
|
+
stack.push({ n: n.children[i], visitor, code, headingHidden });
|
|
671
962
|
}
|
|
672
963
|
return found;
|
|
673
964
|
}
|
|
@@ -725,7 +1016,19 @@ function finish(b, base) {
|
|
|
725
1016
|
// of the line), so a change still only moves the blocks around it. A line longer than MAX_BLOCK is cut.
|
|
726
1017
|
// Each block is an exact piece of the text, so a quote of it is a real substring.
|
|
727
1018
|
const MAX_LINE_BLOCKS = 5000;
|
|
728
|
-
|
|
1019
|
+
// W-39: `spans` (sorted, not overlapping: boundedPretty's) marks every block that touches one as written by a visitor.
|
|
1020
|
+
export function lineBlocks(text, spans) {
|
|
1021
|
+
const blocks = lineSlices(text);
|
|
1022
|
+
if (!spans?.length)
|
|
1023
|
+
return blocks.map(([a, b]) => ({ text: text.slice(a, b), marks: [] }));
|
|
1024
|
+
let i = 0;
|
|
1025
|
+
return blocks.map(([a, b]) => {
|
|
1026
|
+
while (i < spans.length && spans[i][1] <= a)
|
|
1027
|
+
i++;
|
|
1028
|
+
return { text: text.slice(a, b), marks: [], ...(i < spans.length && spans[i][0] < b ? { visitor: 1 } : {}) };
|
|
1029
|
+
});
|
|
1030
|
+
}
|
|
1031
|
+
function lineSlices(text) {
|
|
729
1032
|
// Lines as [start, end) offsets into the text, so every block is an exact slice of it (Codex round 75:
|
|
730
1033
|
// joining kept lines with "\n" dropped the blank lines between them and made quotes that are not in
|
|
731
1034
|
// the text).
|
|
@@ -743,7 +1046,7 @@ export function lineBlocks(text) {
|
|
|
743
1046
|
i = j + 1;
|
|
744
1047
|
}
|
|
745
1048
|
if (lines.length <= MAX_LINE_BLOCKS)
|
|
746
|
-
return lines
|
|
1049
|
+
return lines;
|
|
747
1050
|
const out = [];
|
|
748
1051
|
let start = -1;
|
|
749
1052
|
let end = -1;
|
|
@@ -757,17 +1060,17 @@ export function lineBlocks(text) {
|
|
|
757
1060
|
if (start === -1)
|
|
758
1061
|
start = a;
|
|
759
1062
|
else if (b - start > MAX_BLOCK) {
|
|
760
|
-
out.push(
|
|
1063
|
+
out.push([start, end]); // a run never grows past MAX_BLOCK
|
|
761
1064
|
start = a;
|
|
762
1065
|
}
|
|
763
1066
|
end = b;
|
|
764
1067
|
if (cut(text.slice(a, b))) {
|
|
765
|
-
out.push(
|
|
1068
|
+
out.push([start, end]);
|
|
766
1069
|
start = -1;
|
|
767
1070
|
}
|
|
768
1071
|
}
|
|
769
1072
|
if (start !== -1)
|
|
770
|
-
out.push(
|
|
1073
|
+
out.push([start, end]);
|
|
771
1074
|
return out;
|
|
772
1075
|
}
|
|
773
1076
|
function rawBlocks(text) {
|
|
@@ -787,20 +1090,22 @@ export function parseHtml(html, baseUrl, keep, rendered) {
|
|
|
787
1090
|
const doc = tree.doc;
|
|
788
1091
|
const shield = new Shield();
|
|
789
1092
|
shield.sheet = pageSheet(doc);
|
|
1093
|
+
const visitorScopes = new WeakMap();
|
|
790
1094
|
if (rendered && /^data-t-[0-9a-f]{16}$/.test(rendered.mark))
|
|
791
1095
|
shield.sheet.rendered = { mark: rendered.mark, complete: rendered.complete === true };
|
|
792
1096
|
if (tree.flattened) {
|
|
793
1097
|
shield.sheet.inherited = new WeakMap();
|
|
794
1098
|
const reasons = new WeakMap();
|
|
795
|
-
for (const [attrs, parent] of tree.inherited) {
|
|
1099
|
+
for (const [attrs, parent, code] of tree.inherited) {
|
|
796
1100
|
let why = reasons.get(parent);
|
|
797
1101
|
if (why === undefined)
|
|
798
1102
|
reasons.set(parent, why = hiddenReason(parent, shield.sheet));
|
|
799
1103
|
if (why)
|
|
800
1104
|
shield.sheet.inherited.set(attrs, why);
|
|
1105
|
+
visitorScopes.set(attrs, { visitor: elementMark(parent, code, visitorScopes), code });
|
|
801
1106
|
}
|
|
802
1107
|
}
|
|
803
|
-
|
|
1108
|
+
let found = scanHead(doc, visitorScopes, tree.source, shield.sheet);
|
|
804
1109
|
let linkBase = baseUrl;
|
|
805
1110
|
try {
|
|
806
1111
|
const base = new URL(found.base?.attribs?.href ?? "", baseUrl);
|
|
@@ -811,6 +1116,10 @@ export function parseHtml(html, baseUrl, keep, rendered) {
|
|
|
811
1116
|
// An unusable first base leaves links relative to the fetched page.
|
|
812
1117
|
}
|
|
813
1118
|
const b = new Builder(shield, linkBase);
|
|
1119
|
+
b.visitorScopes = visitorScopes;
|
|
1120
|
+
markWrappers(doc, b);
|
|
1121
|
+
found = scanHead(doc, visitorScopes, tree.source, shield.sheet, sourceHeadings(doc, b));
|
|
1122
|
+
b.headingVisitors = found.headingVisitors;
|
|
814
1123
|
chooseMain(doc, b);
|
|
815
1124
|
walk(doc, b);
|
|
816
1125
|
b.flush();
|
|
@@ -850,10 +1159,12 @@ export function parseHtml(html, baseUrl, keep, rendered) {
|
|
|
850
1159
|
// parsed, never executed, and phrases inside it are flagged.
|
|
851
1160
|
const types = new Set();
|
|
852
1161
|
const rawParts = [];
|
|
1162
|
+
const partSpans = [];
|
|
1163
|
+
const partVisitors = [];
|
|
853
1164
|
let used = 0;
|
|
854
1165
|
const skipped = { invalid: 0, too_large: 0 };
|
|
855
1166
|
for (const s of found.ld) {
|
|
856
|
-
const raw = rawText(s, MAX_STRUCTURED).trim();
|
|
1167
|
+
const raw = rawText(s.n, MAX_STRUCTURED).trim();
|
|
857
1168
|
if (!raw)
|
|
858
1169
|
continue;
|
|
859
1170
|
if (used + raw.length > MAX_STRUCTURED) {
|
|
@@ -870,15 +1181,23 @@ export function parseHtml(html, baseUrl, keep, rendered) {
|
|
|
870
1181
|
// The bounded printer stops at the remaining budget (Codex round 78: printing whole first could use
|
|
871
1182
|
// far more memory than the cap); a print that does not fit whole keeps the raw text.
|
|
872
1183
|
let pretty = raw;
|
|
1184
|
+
// W-39: where the visitor values (reviews, comments, answers) are in the text kept. Raw text kept whole is
|
|
1185
|
+
// the visitor's throughout when any value in it is.
|
|
1186
|
+
let spans = [];
|
|
1187
|
+
let printed = false;
|
|
873
1188
|
try {
|
|
874
|
-
const p = boundedPretty(data, Math.max(0, MAX_STRUCTURED - used));
|
|
1189
|
+
const p = boundedPretty(data, Math.max(0, MAX_STRUCTURED - used), spans);
|
|
875
1190
|
if (!p.omitted && p.exact)
|
|
876
|
-
pretty = p.text;
|
|
1191
|
+
(pretty = p.text), (printed = true);
|
|
877
1192
|
}
|
|
878
1193
|
catch {
|
|
879
1194
|
// keep raw
|
|
880
1195
|
}
|
|
1196
|
+
if (!printed)
|
|
1197
|
+
spans = anyVisitorValue(data) ? [[0, raw.length]] : [];
|
|
881
1198
|
rawParts.push(pretty);
|
|
1199
|
+
partSpans.push(spans);
|
|
1200
|
+
partVisitors.push(s.visitor);
|
|
882
1201
|
used += pretty.length;
|
|
883
1202
|
}
|
|
884
1203
|
catch {
|
|
@@ -890,7 +1209,10 @@ export function parseHtml(html, baseUrl, keep, rendered) {
|
|
|
890
1209
|
page.searchable_chars += used;
|
|
891
1210
|
page.structured_skipped = skipped.invalid || skipped.too_large ? skipped : undefined;
|
|
892
1211
|
if (rawParts.length) {
|
|
893
|
-
page.sections.push({ n: page.sections.length + 1, level: 0, heading:
|
|
1212
|
+
page.sections.push({ n: page.sections.length + 1, level: 0, heading: STRUCTURED_HEADING, generated: true, blocks: rawParts.flatMap((part, i) => lineBlocks(part, partSpans[i]).map((b) => {
|
|
1213
|
+
const visitor = asLevel(strongest(b.visitor ?? 0, partVisitors[i]));
|
|
1214
|
+
return visitor ? { ...b, visitor } : b;
|
|
1215
|
+
})) });
|
|
894
1216
|
}
|
|
895
1217
|
// W-04: the page's forms, as fields. Labels and option texts are returned text, so the shield reads them.
|
|
896
1218
|
const fx = extractForms(doc, baseUrl, tree.flattened);
|
|
@@ -1247,7 +1569,9 @@ export const MAX_PRETTY_DEPTH = 100_000;
|
|
|
1247
1569
|
// keeps counting the length of what it does not write, so the omitted count is exact and costs
|
|
1248
1570
|
// arithmetic, not memory. Returns { text, omitted, exact }: `exact` is false only when the nesting
|
|
1249
1571
|
// passed MAX_PRETTY_DEPTH, and then `omitted` is a lower bound.
|
|
1250
|
-
|
|
1572
|
+
// W-39: with `spans`, it also records where each visitor value (visitorValue) starts and ends in the text, outermost
|
|
1573
|
+
// only: a review's whole object, its author included, is the visitor's.
|
|
1574
|
+
export function boundedPretty(root, max = MAX_EXPANDED, spans) {
|
|
1251
1575
|
const out = [];
|
|
1252
1576
|
let used = 0; // characters kept
|
|
1253
1577
|
let total = 0; // characters the full output would have
|
|
@@ -1271,22 +1595,32 @@ export function boundedPretty(root, max = MAX_EXPANDED) {
|
|
|
1271
1595
|
};
|
|
1272
1596
|
const stack = [];
|
|
1273
1597
|
let exact = true;
|
|
1274
|
-
|
|
1598
|
+
let open = 0; // W-39: visitor values being printed
|
|
1599
|
+
// A value with no children: written whole, so its span (if it starts one) closes here too.
|
|
1600
|
+
const leaf = (s, span) => {
|
|
1601
|
+
put(s);
|
|
1602
|
+
if (span !== undefined)
|
|
1603
|
+
spans.push([span, used]);
|
|
1604
|
+
};
|
|
1605
|
+
const value = (v, depth, key) => {
|
|
1606
|
+
const span = spans && open === 0 && visitorValue(v, key) ? used : undefined;
|
|
1275
1607
|
if (typeof v !== "object" || v === null)
|
|
1276
|
-
return
|
|
1608
|
+
return leaf(JSON.stringify(v) ?? "null", span);
|
|
1277
1609
|
if (Array.isArray(v)) {
|
|
1278
1610
|
if (v.length === 0)
|
|
1279
|
-
return
|
|
1611
|
+
return leaf("[]", span);
|
|
1280
1612
|
put("[");
|
|
1281
|
-
stack.push({ arr: v, i: 0, n: v.length, depth });
|
|
1613
|
+
stack.push(span === undefined ? { arr: v, i: 0, n: v.length, depth } : { arr: v, i: 0, n: v.length, depth, span });
|
|
1282
1614
|
}
|
|
1283
1615
|
else {
|
|
1284
1616
|
const keys = Object.keys(v);
|
|
1285
1617
|
if (keys.length === 0)
|
|
1286
|
-
return
|
|
1618
|
+
return leaf("{}", span);
|
|
1287
1619
|
put("{");
|
|
1288
|
-
stack.push({ obj: v, keys, i: 0, n: keys.length, depth });
|
|
1620
|
+
stack.push(span === undefined ? { obj: v, keys, i: 0, n: keys.length, depth } : { obj: v, keys, i: 0, n: keys.length, depth, span });
|
|
1289
1621
|
}
|
|
1622
|
+
if (span !== undefined)
|
|
1623
|
+
open++;
|
|
1290
1624
|
};
|
|
1291
1625
|
value(root, 0);
|
|
1292
1626
|
while (stack.length) {
|
|
@@ -1300,20 +1634,25 @@ export function boundedPretty(root, max = MAX_EXPANDED) {
|
|
|
1300
1634
|
put(",");
|
|
1301
1635
|
line(f.depth + 1);
|
|
1302
1636
|
let child;
|
|
1637
|
+
let key;
|
|
1303
1638
|
if (f.arr)
|
|
1304
1639
|
child = f.arr[f.i];
|
|
1305
1640
|
else {
|
|
1306
|
-
|
|
1307
|
-
put(JSON.stringify(
|
|
1308
|
-
child = f.obj[
|
|
1641
|
+
key = f.keys[f.i];
|
|
1642
|
+
put(JSON.stringify(key) + ": ");
|
|
1643
|
+
child = f.obj[key];
|
|
1309
1644
|
}
|
|
1310
1645
|
f.i++;
|
|
1311
|
-
value(child, f.depth + 1);
|
|
1646
|
+
value(child, f.depth + 1, key);
|
|
1312
1647
|
}
|
|
1313
1648
|
else {
|
|
1314
1649
|
stack.pop();
|
|
1315
1650
|
line(f.depth);
|
|
1316
1651
|
put(f.arr ? "]" : "}");
|
|
1652
|
+
if (f.span !== undefined) {
|
|
1653
|
+
spans.push([f.span, used]);
|
|
1654
|
+
open--;
|
|
1655
|
+
}
|
|
1317
1656
|
}
|
|
1318
1657
|
}
|
|
1319
1658
|
return { text: out.join(""), omitted: Math.max(0, total - max), exact };
|
|
@@ -1428,14 +1767,23 @@ export const tokens = (s) => {
|
|
|
1428
1767
|
}
|
|
1429
1768
|
return out;
|
|
1430
1769
|
};
|
|
1770
|
+
// The label of the section Tiresias makes from a page's JSON-LD.
|
|
1771
|
+
export const STRUCTURED_HEADING = "(structured data)";
|
|
1772
|
+
// A line of structured data longer than this is a copy of the page's text (an article's whole body, often as
|
|
1773
|
+
// escaped HTML), not a fact such as a price or a date. A long line holds many words, so by word overlap alone it
|
|
1774
|
+
// outranked the article's own paragraphs (field list, 2026-11-14). Such a line is ranked after every other
|
|
1775
|
+
// passage: it is still returned when little else matches.
|
|
1776
|
+
export const STRUCTURED_COPY = 500;
|
|
1431
1777
|
// Plain keyword overlap. Honest and boring on purpose: no AI, no surprises.
|
|
1432
1778
|
export function topPassages(p, question, k = 5) {
|
|
1433
1779
|
const q = new Set(tokens(question));
|
|
1434
1780
|
if (!q.size)
|
|
1435
1781
|
return [];
|
|
1436
1782
|
const scored = [];
|
|
1783
|
+
const copies = new Set();
|
|
1437
1784
|
for (const s of p.sections) {
|
|
1438
1785
|
const headTokens = new Set(tokens(s.heading));
|
|
1786
|
+
const structured = !!s.generated && s.heading === STRUCTURED_HEADING;
|
|
1439
1787
|
for (const block of s.blocks) {
|
|
1440
1788
|
const bt = new Set(tokens(block.text));
|
|
1441
1789
|
let hits = 0;
|
|
@@ -1445,9 +1793,15 @@ export function topPassages(p, question, k = 5) {
|
|
|
1445
1793
|
for (const t of q)
|
|
1446
1794
|
if (headTokens.has(t))
|
|
1447
1795
|
hits += 0.5; // a matching heading helps a little
|
|
1448
|
-
if (hits > 0)
|
|
1449
|
-
|
|
1796
|
+
if (hits > 0) {
|
|
1797
|
+
const visitor = asLevel(strongest(s.visitor ?? 0, block.visitor ?? 0));
|
|
1798
|
+
const passage = { section: s.n, heading: s.heading, text: block.text, score: Math.round((hits / q.size) * 100) / 100, ...(visitor ? { visitor } : {}) };
|
|
1799
|
+
scored.push(passage);
|
|
1800
|
+
if (structured && block.text.length > STRUCTURED_COPY)
|
|
1801
|
+
copies.add(passage);
|
|
1802
|
+
}
|
|
1450
1803
|
}
|
|
1451
1804
|
}
|
|
1452
|
-
|
|
1805
|
+
// The page's own text comes before a long line of its structured data (see STRUCTURED_COPY).
|
|
1806
|
+
return scored.sort((a, b) => Number(copies.has(a)) - Number(copies.has(b)) || b.score - a.score || a.section - b.section).slice(0, k);
|
|
1453
1807
|
}
|