@quanta-intellect/vessel-browser 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +377 -0
- package/bin/vessel-browser.js +25 -0
- package/out/main/index.js +13004 -0
- package/out/preload/content-script.js +2836 -0
- package/out/preload/index.js +172 -0
- package/out/renderer/assets/index-BYA528aQ.js +4883 -0
- package/out/renderer/assets/index-Bz1EMkt-.css +3354 -0
- package/out/renderer/assets/vessel-logo-transparent-IT25qr-Z.png +0 -0
- package/out/renderer/index.html +13 -0
- package/package.json +72 -0
- package/resources/vessel-icon.png +0 -0
|
@@ -0,0 +1,2836 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
const electron = require("electron");
|
|
3
|
+
var Readability = { exports: {} };
|
|
4
|
+
var hasRequiredReadability$1;
|
|
5
|
+
function requireReadability$1() {
|
|
6
|
+
if (hasRequiredReadability$1) return Readability.exports;
|
|
7
|
+
hasRequiredReadability$1 = 1;
|
|
8
|
+
(function(module) {
|
|
9
|
+
function Readability2(doc, options) {
|
|
10
|
+
if (options && options.documentElement) {
|
|
11
|
+
doc = options;
|
|
12
|
+
options = arguments[2];
|
|
13
|
+
} else if (!doc || !doc.documentElement) {
|
|
14
|
+
throw new Error(
|
|
15
|
+
"First argument to Readability constructor should be a document object."
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
options = options || {};
|
|
19
|
+
this._doc = doc;
|
|
20
|
+
this._docJSDOMParser = this._doc.firstChild.__JSDOMParser__;
|
|
21
|
+
this._articleTitle = null;
|
|
22
|
+
this._articleByline = null;
|
|
23
|
+
this._articleDir = null;
|
|
24
|
+
this._articleSiteName = null;
|
|
25
|
+
this._attempts = [];
|
|
26
|
+
this._metadata = {};
|
|
27
|
+
this._debug = !!options.debug;
|
|
28
|
+
this._maxElemsToParse = options.maxElemsToParse || this.DEFAULT_MAX_ELEMS_TO_PARSE;
|
|
29
|
+
this._nbTopCandidates = options.nbTopCandidates || this.DEFAULT_N_TOP_CANDIDATES;
|
|
30
|
+
this._charThreshold = options.charThreshold || this.DEFAULT_CHAR_THRESHOLD;
|
|
31
|
+
this._classesToPreserve = this.CLASSES_TO_PRESERVE.concat(
|
|
32
|
+
options.classesToPreserve || []
|
|
33
|
+
);
|
|
34
|
+
this._keepClasses = !!options.keepClasses;
|
|
35
|
+
this._serializer = options.serializer || function(el) {
|
|
36
|
+
return el.innerHTML;
|
|
37
|
+
};
|
|
38
|
+
this._disableJSONLD = !!options.disableJSONLD;
|
|
39
|
+
this._allowedVideoRegex = options.allowedVideoRegex || this.REGEXPS.videos;
|
|
40
|
+
this._linkDensityModifier = options.linkDensityModifier || 0;
|
|
41
|
+
this._flags = this.FLAG_STRIP_UNLIKELYS | this.FLAG_WEIGHT_CLASSES | this.FLAG_CLEAN_CONDITIONALLY;
|
|
42
|
+
if (this._debug) {
|
|
43
|
+
let logNode = function(node) {
|
|
44
|
+
if (node.nodeType == node.TEXT_NODE) {
|
|
45
|
+
return `${node.nodeName} ("${node.textContent}")`;
|
|
46
|
+
}
|
|
47
|
+
let attrPairs = Array.from(node.attributes || [], function(attr) {
|
|
48
|
+
return `${attr.name}="${attr.value}"`;
|
|
49
|
+
}).join(" ");
|
|
50
|
+
return `<${node.localName} ${attrPairs}>`;
|
|
51
|
+
};
|
|
52
|
+
this.log = function() {
|
|
53
|
+
if (typeof console !== "undefined") {
|
|
54
|
+
let args = Array.from(arguments, (arg) => {
|
|
55
|
+
if (arg && arg.nodeType == this.ELEMENT_NODE) {
|
|
56
|
+
return logNode(arg);
|
|
57
|
+
}
|
|
58
|
+
return arg;
|
|
59
|
+
});
|
|
60
|
+
args.unshift("Reader: (Readability)");
|
|
61
|
+
console.log(...args);
|
|
62
|
+
} else if (typeof dump !== "undefined") {
|
|
63
|
+
var msg = Array.prototype.map.call(arguments, function(x) {
|
|
64
|
+
return x && x.nodeName ? logNode(x) : x;
|
|
65
|
+
}).join(" ");
|
|
66
|
+
dump("Reader: (Readability) " + msg + "\n");
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
} else {
|
|
70
|
+
this.log = function() {
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
Readability2.prototype = {
|
|
75
|
+
FLAG_STRIP_UNLIKELYS: 1,
|
|
76
|
+
FLAG_WEIGHT_CLASSES: 2,
|
|
77
|
+
FLAG_CLEAN_CONDITIONALLY: 4,
|
|
78
|
+
// https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeType
|
|
79
|
+
ELEMENT_NODE: 1,
|
|
80
|
+
TEXT_NODE: 3,
|
|
81
|
+
// Max number of nodes supported by this parser. Default: 0 (no limit)
|
|
82
|
+
DEFAULT_MAX_ELEMS_TO_PARSE: 0,
|
|
83
|
+
// The number of top candidates to consider when analysing how
|
|
84
|
+
// tight the competition is among candidates.
|
|
85
|
+
DEFAULT_N_TOP_CANDIDATES: 5,
|
|
86
|
+
// Element tags to score by default.
|
|
87
|
+
DEFAULT_TAGS_TO_SCORE: "section,h2,h3,h4,h5,h6,p,td,pre".toUpperCase().split(","),
|
|
88
|
+
// The default number of chars an article must have in order to return a result
|
|
89
|
+
DEFAULT_CHAR_THRESHOLD: 500,
|
|
90
|
+
// All of the regular expressions in use within readability.
|
|
91
|
+
// Defined up here so we don't instantiate them repeatedly in loops.
|
|
92
|
+
REGEXPS: {
|
|
93
|
+
// NOTE: These two regular expressions are duplicated in
|
|
94
|
+
// Readability-readerable.js. Please keep both copies in sync.
|
|
95
|
+
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
|
|
96
|
+
okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i,
|
|
97
|
+
positive: /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,
|
|
98
|
+
negative: /-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget/i,
|
|
99
|
+
extraneous: /print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,
|
|
100
|
+
byline: /byline|author|dateline|writtenby|p-author/i,
|
|
101
|
+
replaceFonts: /<(\/?)font[^>]*>/gi,
|
|
102
|
+
normalize: /\s{2,}/g,
|
|
103
|
+
videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,
|
|
104
|
+
shareElements: /(\b|_)(share|sharedaddy)(\b|_)/i,
|
|
105
|
+
nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,
|
|
106
|
+
prevLink: /(prev|earl|old|new|<|«)/i,
|
|
107
|
+
tokenize: /\W+/g,
|
|
108
|
+
whitespace: /^\s*$/,
|
|
109
|
+
hasContent: /\S$/,
|
|
110
|
+
hashUrl: /^#.+/,
|
|
111
|
+
srcsetUrl: /(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))/g,
|
|
112
|
+
b64DataUrl: /^data:\s*([^\s;,]+)\s*;\s*base64\s*,/i,
|
|
113
|
+
// Commas as used in Latin, Sindhi, Chinese and various other scripts.
|
|
114
|
+
// see: https://en.wikipedia.org/wiki/Comma#Comma_variants
|
|
115
|
+
commas: /\u002C|\u060C|\uFE50|\uFE10|\uFE11|\u2E41|\u2E34|\u2E32|\uFF0C/g,
|
|
116
|
+
// See: https://schema.org/Article
|
|
117
|
+
jsonLdArticleTypes: /^Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|AskPublicNewsArticle|BackgroundNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|SatiricalArticle|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference$/,
|
|
118
|
+
// used to see if a node's content matches words commonly used for ad blocks or loading indicators
|
|
119
|
+
adWords: /^(ad(vertising|vertisement)?|pub(licité)?|werb(ung)?|广告|Реклама|Anuncio)$/iu,
|
|
120
|
+
loadingWords: /^((loading|正在加载|Загрузка|chargement|cargando)(…|\.\.\.)?)$/iu
|
|
121
|
+
},
|
|
122
|
+
UNLIKELY_ROLES: [
|
|
123
|
+
"menu",
|
|
124
|
+
"menubar",
|
|
125
|
+
"complementary",
|
|
126
|
+
"navigation",
|
|
127
|
+
"alert",
|
|
128
|
+
"alertdialog",
|
|
129
|
+
"dialog"
|
|
130
|
+
],
|
|
131
|
+
DIV_TO_P_ELEMS: /* @__PURE__ */ new Set([
|
|
132
|
+
"BLOCKQUOTE",
|
|
133
|
+
"DL",
|
|
134
|
+
"DIV",
|
|
135
|
+
"IMG",
|
|
136
|
+
"OL",
|
|
137
|
+
"P",
|
|
138
|
+
"PRE",
|
|
139
|
+
"TABLE",
|
|
140
|
+
"UL"
|
|
141
|
+
]),
|
|
142
|
+
ALTER_TO_DIV_EXCEPTIONS: ["DIV", "ARTICLE", "SECTION", "P", "OL", "UL"],
|
|
143
|
+
PRESENTATIONAL_ATTRIBUTES: [
|
|
144
|
+
"align",
|
|
145
|
+
"background",
|
|
146
|
+
"bgcolor",
|
|
147
|
+
"border",
|
|
148
|
+
"cellpadding",
|
|
149
|
+
"cellspacing",
|
|
150
|
+
"frame",
|
|
151
|
+
"hspace",
|
|
152
|
+
"rules",
|
|
153
|
+
"style",
|
|
154
|
+
"valign",
|
|
155
|
+
"vspace"
|
|
156
|
+
],
|
|
157
|
+
DEPRECATED_SIZE_ATTRIBUTE_ELEMS: ["TABLE", "TH", "TD", "HR", "PRE"],
|
|
158
|
+
// The commented out elements qualify as phrasing content but tend to be
|
|
159
|
+
// removed by readability when put into paragraphs, so we ignore them here.
|
|
160
|
+
PHRASING_ELEMS: [
|
|
161
|
+
// "CANVAS", "IFRAME", "SVG", "VIDEO",
|
|
162
|
+
"ABBR",
|
|
163
|
+
"AUDIO",
|
|
164
|
+
"B",
|
|
165
|
+
"BDO",
|
|
166
|
+
"BR",
|
|
167
|
+
"BUTTON",
|
|
168
|
+
"CITE",
|
|
169
|
+
"CODE",
|
|
170
|
+
"DATA",
|
|
171
|
+
"DATALIST",
|
|
172
|
+
"DFN",
|
|
173
|
+
"EM",
|
|
174
|
+
"EMBED",
|
|
175
|
+
"I",
|
|
176
|
+
"IMG",
|
|
177
|
+
"INPUT",
|
|
178
|
+
"KBD",
|
|
179
|
+
"LABEL",
|
|
180
|
+
"MARK",
|
|
181
|
+
"MATH",
|
|
182
|
+
"METER",
|
|
183
|
+
"NOSCRIPT",
|
|
184
|
+
"OBJECT",
|
|
185
|
+
"OUTPUT",
|
|
186
|
+
"PROGRESS",
|
|
187
|
+
"Q",
|
|
188
|
+
"RUBY",
|
|
189
|
+
"SAMP",
|
|
190
|
+
"SCRIPT",
|
|
191
|
+
"SELECT",
|
|
192
|
+
"SMALL",
|
|
193
|
+
"SPAN",
|
|
194
|
+
"STRONG",
|
|
195
|
+
"SUB",
|
|
196
|
+
"SUP",
|
|
197
|
+
"TEXTAREA",
|
|
198
|
+
"TIME",
|
|
199
|
+
"VAR",
|
|
200
|
+
"WBR"
|
|
201
|
+
],
|
|
202
|
+
// These are the classes that readability sets itself.
|
|
203
|
+
CLASSES_TO_PRESERVE: ["page"],
|
|
204
|
+
// These are the list of HTML entities that need to be escaped.
|
|
205
|
+
HTML_ESCAPE_MAP: {
|
|
206
|
+
lt: "<",
|
|
207
|
+
gt: ">",
|
|
208
|
+
amp: "&",
|
|
209
|
+
quot: '"',
|
|
210
|
+
apos: "'"
|
|
211
|
+
},
|
|
212
|
+
/**
|
|
213
|
+
* Run any post-process modifications to article content as necessary.
|
|
214
|
+
*
|
|
215
|
+
* @param Element
|
|
216
|
+
* @return void
|
|
217
|
+
**/
|
|
218
|
+
_postProcessContent(articleContent) {
|
|
219
|
+
this._fixRelativeUris(articleContent);
|
|
220
|
+
this._simplifyNestedElements(articleContent);
|
|
221
|
+
if (!this._keepClasses) {
|
|
222
|
+
this._cleanClasses(articleContent);
|
|
223
|
+
}
|
|
224
|
+
},
|
|
225
|
+
/**
|
|
226
|
+
* Iterates over a NodeList, calls `filterFn` for each node and removes node
|
|
227
|
+
* if function returned `true`.
|
|
228
|
+
*
|
|
229
|
+
* If function is not passed, removes all the nodes in node list.
|
|
230
|
+
*
|
|
231
|
+
* @param NodeList nodeList The nodes to operate on
|
|
232
|
+
* @param Function filterFn the function to use as a filter
|
|
233
|
+
* @return void
|
|
234
|
+
*/
|
|
235
|
+
_removeNodes(nodeList, filterFn) {
|
|
236
|
+
if (this._docJSDOMParser && nodeList._isLiveNodeList) {
|
|
237
|
+
throw new Error("Do not pass live node lists to _removeNodes");
|
|
238
|
+
}
|
|
239
|
+
for (var i = nodeList.length - 1; i >= 0; i--) {
|
|
240
|
+
var node = nodeList[i];
|
|
241
|
+
var parentNode = node.parentNode;
|
|
242
|
+
if (parentNode) {
|
|
243
|
+
if (!filterFn || filterFn.call(this, node, i, nodeList)) {
|
|
244
|
+
parentNode.removeChild(node);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
},
|
|
249
|
+
/**
|
|
250
|
+
* Iterates over a NodeList, and calls _setNodeTag for each node.
|
|
251
|
+
*
|
|
252
|
+
* @param NodeList nodeList The nodes to operate on
|
|
253
|
+
* @param String newTagName the new tag name to use
|
|
254
|
+
* @return void
|
|
255
|
+
*/
|
|
256
|
+
_replaceNodeTags(nodeList, newTagName) {
|
|
257
|
+
if (this._docJSDOMParser && nodeList._isLiveNodeList) {
|
|
258
|
+
throw new Error("Do not pass live node lists to _replaceNodeTags");
|
|
259
|
+
}
|
|
260
|
+
for (const node of nodeList) {
|
|
261
|
+
this._setNodeTag(node, newTagName);
|
|
262
|
+
}
|
|
263
|
+
},
|
|
264
|
+
/**
|
|
265
|
+
* Iterate over a NodeList, which doesn't natively fully implement the Array
|
|
266
|
+
* interface.
|
|
267
|
+
*
|
|
268
|
+
* For convenience, the current object context is applied to the provided
|
|
269
|
+
* iterate function.
|
|
270
|
+
*
|
|
271
|
+
* @param NodeList nodeList The NodeList.
|
|
272
|
+
* @param Function fn The iterate function.
|
|
273
|
+
* @return void
|
|
274
|
+
*/
|
|
275
|
+
_forEachNode(nodeList, fn) {
|
|
276
|
+
Array.prototype.forEach.call(nodeList, fn, this);
|
|
277
|
+
},
|
|
278
|
+
/**
|
|
279
|
+
* Iterate over a NodeList, and return the first node that passes
|
|
280
|
+
* the supplied test function
|
|
281
|
+
*
|
|
282
|
+
* For convenience, the current object context is applied to the provided
|
|
283
|
+
* test function.
|
|
284
|
+
*
|
|
285
|
+
* @param NodeList nodeList The NodeList.
|
|
286
|
+
* @param Function fn The test function.
|
|
287
|
+
* @return void
|
|
288
|
+
*/
|
|
289
|
+
_findNode(nodeList, fn) {
|
|
290
|
+
return Array.prototype.find.call(nodeList, fn, this);
|
|
291
|
+
},
|
|
292
|
+
/**
|
|
293
|
+
* Iterate over a NodeList, return true if any of the provided iterate
|
|
294
|
+
* function calls returns true, false otherwise.
|
|
295
|
+
*
|
|
296
|
+
* For convenience, the current object context is applied to the
|
|
297
|
+
* provided iterate function.
|
|
298
|
+
*
|
|
299
|
+
* @param NodeList nodeList The NodeList.
|
|
300
|
+
* @param Function fn The iterate function.
|
|
301
|
+
* @return Boolean
|
|
302
|
+
*/
|
|
303
|
+
_someNode(nodeList, fn) {
|
|
304
|
+
return Array.prototype.some.call(nodeList, fn, this);
|
|
305
|
+
},
|
|
306
|
+
/**
|
|
307
|
+
* Iterate over a NodeList, return true if all of the provided iterate
|
|
308
|
+
* function calls return true, false otherwise.
|
|
309
|
+
*
|
|
310
|
+
* For convenience, the current object context is applied to the
|
|
311
|
+
* provided iterate function.
|
|
312
|
+
*
|
|
313
|
+
* @param NodeList nodeList The NodeList.
|
|
314
|
+
* @param Function fn The iterate function.
|
|
315
|
+
* @return Boolean
|
|
316
|
+
*/
|
|
317
|
+
_everyNode(nodeList, fn) {
|
|
318
|
+
return Array.prototype.every.call(nodeList, fn, this);
|
|
319
|
+
},
|
|
320
|
+
_getAllNodesWithTag(node, tagNames) {
|
|
321
|
+
if (node.querySelectorAll) {
|
|
322
|
+
return node.querySelectorAll(tagNames.join(","));
|
|
323
|
+
}
|
|
324
|
+
return [].concat.apply(
|
|
325
|
+
[],
|
|
326
|
+
tagNames.map(function(tag) {
|
|
327
|
+
var collection = node.getElementsByTagName(tag);
|
|
328
|
+
return Array.isArray(collection) ? collection : Array.from(collection);
|
|
329
|
+
})
|
|
330
|
+
);
|
|
331
|
+
},
|
|
332
|
+
/**
|
|
333
|
+
* Removes the class="" attribute from every element in the given
|
|
334
|
+
* subtree, except those that match CLASSES_TO_PRESERVE and
|
|
335
|
+
* the classesToPreserve array from the options object.
|
|
336
|
+
*
|
|
337
|
+
* @param Element
|
|
338
|
+
* @return void
|
|
339
|
+
*/
|
|
340
|
+
_cleanClasses(node) {
|
|
341
|
+
var classesToPreserve = this._classesToPreserve;
|
|
342
|
+
var className = (node.getAttribute("class") || "").split(/\s+/).filter((cls) => classesToPreserve.includes(cls)).join(" ");
|
|
343
|
+
if (className) {
|
|
344
|
+
node.setAttribute("class", className);
|
|
345
|
+
} else {
|
|
346
|
+
node.removeAttribute("class");
|
|
347
|
+
}
|
|
348
|
+
for (node = node.firstElementChild; node; node = node.nextElementSibling) {
|
|
349
|
+
this._cleanClasses(node);
|
|
350
|
+
}
|
|
351
|
+
},
|
|
352
|
+
/**
|
|
353
|
+
* Tests whether a string is a URL or not.
|
|
354
|
+
*
|
|
355
|
+
* @param {string} str The string to test
|
|
356
|
+
* @return {boolean} true if str is a URL, false if not
|
|
357
|
+
*/
|
|
358
|
+
_isUrl(str) {
|
|
359
|
+
try {
|
|
360
|
+
new URL(str);
|
|
361
|
+
return true;
|
|
362
|
+
} catch {
|
|
363
|
+
return false;
|
|
364
|
+
}
|
|
365
|
+
},
|
|
366
|
+
/**
|
|
367
|
+
* Converts each <a> and <img> uri in the given element to an absolute URI,
|
|
368
|
+
* ignoring #ref URIs.
|
|
369
|
+
*
|
|
370
|
+
* @param Element
|
|
371
|
+
* @return void
|
|
372
|
+
*/
|
|
373
|
+
_fixRelativeUris(articleContent) {
|
|
374
|
+
var baseURI = this._doc.baseURI;
|
|
375
|
+
var documentURI = this._doc.documentURI;
|
|
376
|
+
function toAbsoluteURI(uri) {
|
|
377
|
+
if (baseURI == documentURI && uri.charAt(0) == "#") {
|
|
378
|
+
return uri;
|
|
379
|
+
}
|
|
380
|
+
try {
|
|
381
|
+
return new URL(uri, baseURI).href;
|
|
382
|
+
} catch (ex) {
|
|
383
|
+
}
|
|
384
|
+
return uri;
|
|
385
|
+
}
|
|
386
|
+
var links = this._getAllNodesWithTag(articleContent, ["a"]);
|
|
387
|
+
this._forEachNode(links, function(link) {
|
|
388
|
+
var href = link.getAttribute("href");
|
|
389
|
+
if (href) {
|
|
390
|
+
if (href.indexOf("javascript:") === 0) {
|
|
391
|
+
if (link.childNodes.length === 1 && link.childNodes[0].nodeType === this.TEXT_NODE) {
|
|
392
|
+
var text = this._doc.createTextNode(link.textContent);
|
|
393
|
+
link.parentNode.replaceChild(text, link);
|
|
394
|
+
} else {
|
|
395
|
+
var container = this._doc.createElement("span");
|
|
396
|
+
while (link.firstChild) {
|
|
397
|
+
container.appendChild(link.firstChild);
|
|
398
|
+
}
|
|
399
|
+
link.parentNode.replaceChild(container, link);
|
|
400
|
+
}
|
|
401
|
+
} else {
|
|
402
|
+
link.setAttribute("href", toAbsoluteURI(href));
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
});
|
|
406
|
+
var medias = this._getAllNodesWithTag(articleContent, [
|
|
407
|
+
"img",
|
|
408
|
+
"picture",
|
|
409
|
+
"figure",
|
|
410
|
+
"video",
|
|
411
|
+
"audio",
|
|
412
|
+
"source"
|
|
413
|
+
]);
|
|
414
|
+
this._forEachNode(medias, function(media) {
|
|
415
|
+
var src = media.getAttribute("src");
|
|
416
|
+
var poster = media.getAttribute("poster");
|
|
417
|
+
var srcset = media.getAttribute("srcset");
|
|
418
|
+
if (src) {
|
|
419
|
+
media.setAttribute("src", toAbsoluteURI(src));
|
|
420
|
+
}
|
|
421
|
+
if (poster) {
|
|
422
|
+
media.setAttribute("poster", toAbsoluteURI(poster));
|
|
423
|
+
}
|
|
424
|
+
if (srcset) {
|
|
425
|
+
var newSrcset = srcset.replace(
|
|
426
|
+
this.REGEXPS.srcsetUrl,
|
|
427
|
+
function(_, p1, p2, p3) {
|
|
428
|
+
return toAbsoluteURI(p1) + (p2 || "") + p3;
|
|
429
|
+
}
|
|
430
|
+
);
|
|
431
|
+
media.setAttribute("srcset", newSrcset);
|
|
432
|
+
}
|
|
433
|
+
});
|
|
434
|
+
},
|
|
435
|
+
_simplifyNestedElements(articleContent) {
|
|
436
|
+
var node = articleContent;
|
|
437
|
+
while (node) {
|
|
438
|
+
if (node.parentNode && ["DIV", "SECTION"].includes(node.tagName) && !(node.id && node.id.startsWith("readability"))) {
|
|
439
|
+
if (this._isElementWithoutContent(node)) {
|
|
440
|
+
node = this._removeAndGetNext(node);
|
|
441
|
+
continue;
|
|
442
|
+
} else if (this._hasSingleTagInsideElement(node, "DIV") || this._hasSingleTagInsideElement(node, "SECTION")) {
|
|
443
|
+
var child = node.children[0];
|
|
444
|
+
for (var i = 0; i < node.attributes.length; i++) {
|
|
445
|
+
child.setAttributeNode(node.attributes[i].cloneNode());
|
|
446
|
+
}
|
|
447
|
+
node.parentNode.replaceChild(child, node);
|
|
448
|
+
node = child;
|
|
449
|
+
continue;
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
node = this._getNextNode(node);
|
|
453
|
+
}
|
|
454
|
+
},
|
|
455
|
+
/**
|
|
456
|
+
* Get the article title as an H1.
|
|
457
|
+
*
|
|
458
|
+
* @return string
|
|
459
|
+
**/
|
|
460
|
+
_getArticleTitle() {
|
|
461
|
+
var doc = this._doc;
|
|
462
|
+
var curTitle = "";
|
|
463
|
+
var origTitle = "";
|
|
464
|
+
try {
|
|
465
|
+
curTitle = origTitle = doc.title.trim();
|
|
466
|
+
if (typeof curTitle !== "string") {
|
|
467
|
+
curTitle = origTitle = this._getInnerText(
|
|
468
|
+
doc.getElementsByTagName("title")[0]
|
|
469
|
+
);
|
|
470
|
+
}
|
|
471
|
+
} catch (e) {
|
|
472
|
+
}
|
|
473
|
+
var titleHadHierarchicalSeparators = false;
|
|
474
|
+
function wordCount(str) {
|
|
475
|
+
return str.split(/\s+/).length;
|
|
476
|
+
}
|
|
477
|
+
if (/ [\|\-\\\/>»] /.test(curTitle)) {
|
|
478
|
+
titleHadHierarchicalSeparators = / [\\\/>»] /.test(curTitle);
|
|
479
|
+
let allSeparators = Array.from(origTitle.matchAll(/ [\|\-\\\/>»] /gi));
|
|
480
|
+
curTitle = origTitle.substring(0, allSeparators.pop().index);
|
|
481
|
+
if (wordCount(curTitle) < 3) {
|
|
482
|
+
curTitle = origTitle.replace(/^[^\|\-\\\/>»]*[\|\-\\\/>»]/gi, "");
|
|
483
|
+
}
|
|
484
|
+
} else if (curTitle.includes(": ")) {
|
|
485
|
+
var headings = this._getAllNodesWithTag(doc, ["h1", "h2"]);
|
|
486
|
+
var trimmedTitle = curTitle.trim();
|
|
487
|
+
var match = this._someNode(headings, function(heading) {
|
|
488
|
+
return heading.textContent.trim() === trimmedTitle;
|
|
489
|
+
});
|
|
490
|
+
if (!match) {
|
|
491
|
+
curTitle = origTitle.substring(origTitle.lastIndexOf(":") + 1);
|
|
492
|
+
if (wordCount(curTitle) < 3) {
|
|
493
|
+
curTitle = origTitle.substring(origTitle.indexOf(":") + 1);
|
|
494
|
+
} else if (wordCount(origTitle.substr(0, origTitle.indexOf(":"))) > 5) {
|
|
495
|
+
curTitle = origTitle;
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
} else if (curTitle.length > 150 || curTitle.length < 15) {
|
|
499
|
+
var hOnes = doc.getElementsByTagName("h1");
|
|
500
|
+
if (hOnes.length === 1) {
|
|
501
|
+
curTitle = this._getInnerText(hOnes[0]);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
curTitle = curTitle.trim().replace(this.REGEXPS.normalize, " ");
|
|
505
|
+
var curTitleWordCount = wordCount(curTitle);
|
|
506
|
+
if (curTitleWordCount <= 4 && (!titleHadHierarchicalSeparators || curTitleWordCount != wordCount(origTitle.replace(/[\|\-\\\/>»]+/g, "")) - 1)) {
|
|
507
|
+
curTitle = origTitle;
|
|
508
|
+
}
|
|
509
|
+
return curTitle;
|
|
510
|
+
},
|
|
511
|
+
/**
|
|
512
|
+
* Prepare the HTML document for readability to scrape it.
|
|
513
|
+
* This includes things like stripping javascript, CSS, and handling terrible markup.
|
|
514
|
+
*
|
|
515
|
+
* @return void
|
|
516
|
+
**/
|
|
517
|
+
_prepDocument() {
|
|
518
|
+
var doc = this._doc;
|
|
519
|
+
this._removeNodes(this._getAllNodesWithTag(doc, ["style"]));
|
|
520
|
+
if (doc.body) {
|
|
521
|
+
this._replaceBrs(doc.body);
|
|
522
|
+
}
|
|
523
|
+
this._replaceNodeTags(this._getAllNodesWithTag(doc, ["font"]), "SPAN");
|
|
524
|
+
},
|
|
525
|
+
/**
|
|
526
|
+
* Finds the next node, starting from the given node, and ignoring
|
|
527
|
+
* whitespace in between. If the given node is an element, the same node is
|
|
528
|
+
* returned.
|
|
529
|
+
*/
|
|
530
|
+
_nextNode(node) {
|
|
531
|
+
var next = node;
|
|
532
|
+
while (next && next.nodeType != this.ELEMENT_NODE && this.REGEXPS.whitespace.test(next.textContent)) {
|
|
533
|
+
next = next.nextSibling;
|
|
534
|
+
}
|
|
535
|
+
return next;
|
|
536
|
+
},
|
|
537
|
+
/**
|
|
538
|
+
* Replaces 2 or more successive <br> elements with a single <p>.
|
|
539
|
+
* Whitespace between <br> elements are ignored. For example:
|
|
540
|
+
* <div>foo<br>bar<br> <br><br>abc</div>
|
|
541
|
+
* will become:
|
|
542
|
+
* <div>foo<br>bar<p>abc</p></div>
|
|
543
|
+
*/
|
|
544
|
+
_replaceBrs(elem) {
|
|
545
|
+
this._forEachNode(this._getAllNodesWithTag(elem, ["br"]), function(br) {
|
|
546
|
+
var next = br.nextSibling;
|
|
547
|
+
var replaced = false;
|
|
548
|
+
while ((next = this._nextNode(next)) && next.tagName == "BR") {
|
|
549
|
+
replaced = true;
|
|
550
|
+
var brSibling = next.nextSibling;
|
|
551
|
+
next.remove();
|
|
552
|
+
next = brSibling;
|
|
553
|
+
}
|
|
554
|
+
if (replaced) {
|
|
555
|
+
var p = this._doc.createElement("p");
|
|
556
|
+
br.parentNode.replaceChild(p, br);
|
|
557
|
+
next = p.nextSibling;
|
|
558
|
+
while (next) {
|
|
559
|
+
if (next.tagName == "BR") {
|
|
560
|
+
var nextElem = this._nextNode(next.nextSibling);
|
|
561
|
+
if (nextElem && nextElem.tagName == "BR") {
|
|
562
|
+
break;
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
if (!this._isPhrasingContent(next)) {
|
|
566
|
+
break;
|
|
567
|
+
}
|
|
568
|
+
var sibling = next.nextSibling;
|
|
569
|
+
p.appendChild(next);
|
|
570
|
+
next = sibling;
|
|
571
|
+
}
|
|
572
|
+
while (p.lastChild && this._isWhitespace(p.lastChild)) {
|
|
573
|
+
p.lastChild.remove();
|
|
574
|
+
}
|
|
575
|
+
if (p.parentNode.tagName === "P") {
|
|
576
|
+
this._setNodeTag(p.parentNode, "DIV");
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
});
|
|
580
|
+
},
|
|
581
|
+
_setNodeTag(node, tag) {
|
|
582
|
+
this.log("_setNodeTag", node, tag);
|
|
583
|
+
if (this._docJSDOMParser) {
|
|
584
|
+
node.localName = tag.toLowerCase();
|
|
585
|
+
node.tagName = tag.toUpperCase();
|
|
586
|
+
return node;
|
|
587
|
+
}
|
|
588
|
+
var replacement = node.ownerDocument.createElement(tag);
|
|
589
|
+
while (node.firstChild) {
|
|
590
|
+
replacement.appendChild(node.firstChild);
|
|
591
|
+
}
|
|
592
|
+
node.parentNode.replaceChild(replacement, node);
|
|
593
|
+
if (node.readability) {
|
|
594
|
+
replacement.readability = node.readability;
|
|
595
|
+
}
|
|
596
|
+
for (var i = 0; i < node.attributes.length; i++) {
|
|
597
|
+
replacement.setAttributeNode(node.attributes[i].cloneNode());
|
|
598
|
+
}
|
|
599
|
+
return replacement;
|
|
600
|
+
},
|
|
601
|
+
/**
|
|
602
|
+
* Prepare the article node for display. Clean out any inline styles,
|
|
603
|
+
* iframes, forms, strip extraneous <p> tags, etc.
|
|
604
|
+
*
|
|
605
|
+
* @param Element
|
|
606
|
+
* @return void
|
|
607
|
+
**/
|
|
608
|
+
_prepArticle(articleContent) {
|
|
609
|
+
this._cleanStyles(articleContent);
|
|
610
|
+
this._markDataTables(articleContent);
|
|
611
|
+
this._fixLazyImages(articleContent);
|
|
612
|
+
this._cleanConditionally(articleContent, "form");
|
|
613
|
+
this._cleanConditionally(articleContent, "fieldset");
|
|
614
|
+
this._clean(articleContent, "object");
|
|
615
|
+
this._clean(articleContent, "embed");
|
|
616
|
+
this._clean(articleContent, "footer");
|
|
617
|
+
this._clean(articleContent, "link");
|
|
618
|
+
this._clean(articleContent, "aside");
|
|
619
|
+
var shareElementThreshold = this.DEFAULT_CHAR_THRESHOLD;
|
|
620
|
+
this._forEachNode(articleContent.children, function(topCandidate) {
|
|
621
|
+
this._cleanMatchedNodes(topCandidate, function(node, matchString) {
|
|
622
|
+
return this.REGEXPS.shareElements.test(matchString) && node.textContent.length < shareElementThreshold;
|
|
623
|
+
});
|
|
624
|
+
});
|
|
625
|
+
this._clean(articleContent, "iframe");
|
|
626
|
+
this._clean(articleContent, "input");
|
|
627
|
+
this._clean(articleContent, "textarea");
|
|
628
|
+
this._clean(articleContent, "select");
|
|
629
|
+
this._clean(articleContent, "button");
|
|
630
|
+
this._cleanHeaders(articleContent);
|
|
631
|
+
this._cleanConditionally(articleContent, "table");
|
|
632
|
+
this._cleanConditionally(articleContent, "ul");
|
|
633
|
+
this._cleanConditionally(articleContent, "div");
|
|
634
|
+
this._replaceNodeTags(
|
|
635
|
+
this._getAllNodesWithTag(articleContent, ["h1"]),
|
|
636
|
+
"h2"
|
|
637
|
+
);
|
|
638
|
+
this._removeNodes(
|
|
639
|
+
this._getAllNodesWithTag(articleContent, ["p"]),
|
|
640
|
+
function(paragraph) {
|
|
641
|
+
var contentElementCount = this._getAllNodesWithTag(paragraph, [
|
|
642
|
+
"img",
|
|
643
|
+
"embed",
|
|
644
|
+
"object",
|
|
645
|
+
"iframe"
|
|
646
|
+
]).length;
|
|
647
|
+
return contentElementCount === 0 && !this._getInnerText(paragraph, false);
|
|
648
|
+
}
|
|
649
|
+
);
|
|
650
|
+
this._forEachNode(
|
|
651
|
+
this._getAllNodesWithTag(articleContent, ["br"]),
|
|
652
|
+
function(br) {
|
|
653
|
+
var next = this._nextNode(br.nextSibling);
|
|
654
|
+
if (next && next.tagName == "P") {
|
|
655
|
+
br.remove();
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
);
|
|
659
|
+
this._forEachNode(
|
|
660
|
+
this._getAllNodesWithTag(articleContent, ["table"]),
|
|
661
|
+
function(table) {
|
|
662
|
+
var tbody = this._hasSingleTagInsideElement(table, "TBODY") ? table.firstElementChild : table;
|
|
663
|
+
if (this._hasSingleTagInsideElement(tbody, "TR")) {
|
|
664
|
+
var row = tbody.firstElementChild;
|
|
665
|
+
if (this._hasSingleTagInsideElement(row, "TD")) {
|
|
666
|
+
var cell = row.firstElementChild;
|
|
667
|
+
cell = this._setNodeTag(
|
|
668
|
+
cell,
|
|
669
|
+
this._everyNode(cell.childNodes, this._isPhrasingContent) ? "P" : "DIV"
|
|
670
|
+
);
|
|
671
|
+
table.parentNode.replaceChild(cell, table);
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
);
|
|
676
|
+
},
|
|
677
|
+
/**
|
|
678
|
+
* Initialize a node with the readability object. Also checks the
|
|
679
|
+
* className/id for special names to add to its score.
|
|
680
|
+
*
|
|
681
|
+
* @param Element
|
|
682
|
+
* @return void
|
|
683
|
+
**/
|
|
684
|
+
_initializeNode(node) {
|
|
685
|
+
node.readability = { contentScore: 0 };
|
|
686
|
+
switch (node.tagName) {
|
|
687
|
+
case "DIV":
|
|
688
|
+
node.readability.contentScore += 5;
|
|
689
|
+
break;
|
|
690
|
+
case "PRE":
|
|
691
|
+
case "TD":
|
|
692
|
+
case "BLOCKQUOTE":
|
|
693
|
+
node.readability.contentScore += 3;
|
|
694
|
+
break;
|
|
695
|
+
case "ADDRESS":
|
|
696
|
+
case "OL":
|
|
697
|
+
case "UL":
|
|
698
|
+
case "DL":
|
|
699
|
+
case "DD":
|
|
700
|
+
case "DT":
|
|
701
|
+
case "LI":
|
|
702
|
+
case "FORM":
|
|
703
|
+
node.readability.contentScore -= 3;
|
|
704
|
+
break;
|
|
705
|
+
case "H1":
|
|
706
|
+
case "H2":
|
|
707
|
+
case "H3":
|
|
708
|
+
case "H4":
|
|
709
|
+
case "H5":
|
|
710
|
+
case "H6":
|
|
711
|
+
case "TH":
|
|
712
|
+
node.readability.contentScore -= 5;
|
|
713
|
+
break;
|
|
714
|
+
}
|
|
715
|
+
node.readability.contentScore += this._getClassWeight(node);
|
|
716
|
+
},
|
|
717
|
+
_removeAndGetNext(node) {
|
|
718
|
+
var nextNode = this._getNextNode(node, true);
|
|
719
|
+
node.remove();
|
|
720
|
+
return nextNode;
|
|
721
|
+
},
|
|
722
|
+
/**
|
|
723
|
+
* Traverse the DOM from node to node, starting at the node passed in.
|
|
724
|
+
* Pass true for the second parameter to indicate this node itself
|
|
725
|
+
* (and its kids) are going away, and we want the next node over.
|
|
726
|
+
*
|
|
727
|
+
* Calling this in a loop will traverse the DOM depth-first.
|
|
728
|
+
*
|
|
729
|
+
* @param {Element} node
|
|
730
|
+
* @param {boolean} ignoreSelfAndKids
|
|
731
|
+
* @return {Element}
|
|
732
|
+
*/
|
|
733
|
+
_getNextNode(node, ignoreSelfAndKids) {
|
|
734
|
+
if (!ignoreSelfAndKids && node.firstElementChild) {
|
|
735
|
+
return node.firstElementChild;
|
|
736
|
+
}
|
|
737
|
+
if (node.nextElementSibling) {
|
|
738
|
+
return node.nextElementSibling;
|
|
739
|
+
}
|
|
740
|
+
do {
|
|
741
|
+
node = node.parentNode;
|
|
742
|
+
} while (node && !node.nextElementSibling);
|
|
743
|
+
return node && node.nextElementSibling;
|
|
744
|
+
},
|
|
745
|
+
// compares second text to first one
|
|
746
|
+
// 1 = same text, 0 = completely different text
|
|
747
|
+
// works the way that it splits both texts into words and then finds words that are unique in second text
|
|
748
|
+
// the result is given by the lower length of unique parts
|
|
749
|
+
_textSimilarity(textA, textB) {
|
|
750
|
+
var tokensA = textA.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean);
|
|
751
|
+
var tokensB = textB.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean);
|
|
752
|
+
if (!tokensA.length || !tokensB.length) {
|
|
753
|
+
return 0;
|
|
754
|
+
}
|
|
755
|
+
var uniqTokensB = tokensB.filter((token) => !tokensA.includes(token));
|
|
756
|
+
var distanceB = uniqTokensB.join(" ").length / tokensB.join(" ").length;
|
|
757
|
+
return 1 - distanceB;
|
|
758
|
+
},
|
|
759
|
+
/**
|
|
760
|
+
* Checks whether an element node contains a valid byline
|
|
761
|
+
*
|
|
762
|
+
* @param node {Element}
|
|
763
|
+
* @param matchString {string}
|
|
764
|
+
* @return boolean
|
|
765
|
+
*/
|
|
766
|
+
_isValidByline(node, matchString) {
|
|
767
|
+
var rel = node.getAttribute("rel");
|
|
768
|
+
var itemprop = node.getAttribute("itemprop");
|
|
769
|
+
var bylineLength = node.textContent.trim().length;
|
|
770
|
+
return (rel === "author" || itemprop && itemprop.includes("author") || this.REGEXPS.byline.test(matchString)) && !!bylineLength && bylineLength < 100;
|
|
771
|
+
},
|
|
772
|
+
_getNodeAncestors(node, maxDepth) {
|
|
773
|
+
maxDepth = maxDepth || 0;
|
|
774
|
+
var i = 0, ancestors = [];
|
|
775
|
+
while (node.parentNode) {
|
|
776
|
+
ancestors.push(node.parentNode);
|
|
777
|
+
if (maxDepth && ++i === maxDepth) {
|
|
778
|
+
break;
|
|
779
|
+
}
|
|
780
|
+
node = node.parentNode;
|
|
781
|
+
}
|
|
782
|
+
return ancestors;
|
|
783
|
+
},
|
|
784
|
+
/***
|
|
785
|
+
* grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
|
|
786
|
+
* most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
|
|
787
|
+
*
|
|
788
|
+
* @param page a document to run upon. Needs to be a full document, complete with body.
|
|
789
|
+
* @return Element
|
|
790
|
+
**/
|
|
791
|
+
/* eslint-disable-next-line complexity */
|
|
792
|
+
_grabArticle(page) {
|
|
793
|
+
this.log("**** grabArticle ****");
|
|
794
|
+
var doc = this._doc;
|
|
795
|
+
var isPaging = page !== null;
|
|
796
|
+
page = page ? page : this._doc.body;
|
|
797
|
+
if (!page) {
|
|
798
|
+
this.log("No body found in document. Abort.");
|
|
799
|
+
return null;
|
|
800
|
+
}
|
|
801
|
+
var pageCacheHtml = page.innerHTML;
|
|
802
|
+
while (true) {
|
|
803
|
+
this.log("Starting grabArticle loop");
|
|
804
|
+
var stripUnlikelyCandidates = this._flagIsActive(
|
|
805
|
+
this.FLAG_STRIP_UNLIKELYS
|
|
806
|
+
);
|
|
807
|
+
var elementsToScore = [];
|
|
808
|
+
var node = this._doc.documentElement;
|
|
809
|
+
let shouldRemoveTitleHeader = true;
|
|
810
|
+
while (node) {
|
|
811
|
+
if (node.tagName === "HTML") {
|
|
812
|
+
this._articleLang = node.getAttribute("lang");
|
|
813
|
+
}
|
|
814
|
+
var matchString = node.className + " " + node.id;
|
|
815
|
+
if (!this._isProbablyVisible(node)) {
|
|
816
|
+
this.log("Removing hidden node - " + matchString);
|
|
817
|
+
node = this._removeAndGetNext(node);
|
|
818
|
+
continue;
|
|
819
|
+
}
|
|
820
|
+
if (node.getAttribute("aria-modal") == "true" && node.getAttribute("role") == "dialog") {
|
|
821
|
+
node = this._removeAndGetNext(node);
|
|
822
|
+
continue;
|
|
823
|
+
}
|
|
824
|
+
if (!this._articleByline && !this._metadata.byline && this._isValidByline(node, matchString)) {
|
|
825
|
+
var endOfSearchMarkerNode = this._getNextNode(node, true);
|
|
826
|
+
var next = this._getNextNode(node);
|
|
827
|
+
var itemPropNameNode = null;
|
|
828
|
+
while (next && next != endOfSearchMarkerNode) {
|
|
829
|
+
var itemprop = next.getAttribute("itemprop");
|
|
830
|
+
if (itemprop && itemprop.includes("name")) {
|
|
831
|
+
itemPropNameNode = next;
|
|
832
|
+
break;
|
|
833
|
+
} else {
|
|
834
|
+
next = this._getNextNode(next);
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
this._articleByline = (itemPropNameNode ?? node).textContent.trim();
|
|
838
|
+
node = this._removeAndGetNext(node);
|
|
839
|
+
continue;
|
|
840
|
+
}
|
|
841
|
+
if (shouldRemoveTitleHeader && this._headerDuplicatesTitle(node)) {
|
|
842
|
+
this.log(
|
|
843
|
+
"Removing header: ",
|
|
844
|
+
node.textContent.trim(),
|
|
845
|
+
this._articleTitle.trim()
|
|
846
|
+
);
|
|
847
|
+
shouldRemoveTitleHeader = false;
|
|
848
|
+
node = this._removeAndGetNext(node);
|
|
849
|
+
continue;
|
|
850
|
+
}
|
|
851
|
+
if (stripUnlikelyCandidates) {
|
|
852
|
+
if (this.REGEXPS.unlikelyCandidates.test(matchString) && !this.REGEXPS.okMaybeItsACandidate.test(matchString) && !this._hasAncestorTag(node, "table") && !this._hasAncestorTag(node, "code") && node.tagName !== "BODY" && node.tagName !== "A") {
|
|
853
|
+
this.log("Removing unlikely candidate - " + matchString);
|
|
854
|
+
node = this._removeAndGetNext(node);
|
|
855
|
+
continue;
|
|
856
|
+
}
|
|
857
|
+
if (this.UNLIKELY_ROLES.includes(node.getAttribute("role"))) {
|
|
858
|
+
this.log(
|
|
859
|
+
"Removing content with role " + node.getAttribute("role") + " - " + matchString
|
|
860
|
+
);
|
|
861
|
+
node = this._removeAndGetNext(node);
|
|
862
|
+
continue;
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
if ((node.tagName === "DIV" || node.tagName === "SECTION" || node.tagName === "HEADER" || node.tagName === "H1" || node.tagName === "H2" || node.tagName === "H3" || node.tagName === "H4" || node.tagName === "H5" || node.tagName === "H6") && this._isElementWithoutContent(node)) {
|
|
866
|
+
node = this._removeAndGetNext(node);
|
|
867
|
+
continue;
|
|
868
|
+
}
|
|
869
|
+
if (this.DEFAULT_TAGS_TO_SCORE.includes(node.tagName)) {
|
|
870
|
+
elementsToScore.push(node);
|
|
871
|
+
}
|
|
872
|
+
if (node.tagName === "DIV") {
|
|
873
|
+
var p = null;
|
|
874
|
+
var childNode = node.firstChild;
|
|
875
|
+
while (childNode) {
|
|
876
|
+
var nextSibling = childNode.nextSibling;
|
|
877
|
+
if (this._isPhrasingContent(childNode)) {
|
|
878
|
+
if (p !== null) {
|
|
879
|
+
p.appendChild(childNode);
|
|
880
|
+
} else if (!this._isWhitespace(childNode)) {
|
|
881
|
+
p = doc.createElement("p");
|
|
882
|
+
node.replaceChild(p, childNode);
|
|
883
|
+
p.appendChild(childNode);
|
|
884
|
+
}
|
|
885
|
+
} else if (p !== null) {
|
|
886
|
+
while (p.lastChild && this._isWhitespace(p.lastChild)) {
|
|
887
|
+
p.lastChild.remove();
|
|
888
|
+
}
|
|
889
|
+
p = null;
|
|
890
|
+
}
|
|
891
|
+
childNode = nextSibling;
|
|
892
|
+
}
|
|
893
|
+
if (this._hasSingleTagInsideElement(node, "P") && this._getLinkDensity(node) < 0.25) {
|
|
894
|
+
var newNode = node.children[0];
|
|
895
|
+
node.parentNode.replaceChild(newNode, node);
|
|
896
|
+
node = newNode;
|
|
897
|
+
elementsToScore.push(node);
|
|
898
|
+
} else if (!this._hasChildBlockElement(node)) {
|
|
899
|
+
node = this._setNodeTag(node, "P");
|
|
900
|
+
elementsToScore.push(node);
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
node = this._getNextNode(node);
|
|
904
|
+
}
|
|
905
|
+
var candidates = [];
|
|
906
|
+
this._forEachNode(elementsToScore, function(elementToScore) {
|
|
907
|
+
if (!elementToScore.parentNode || typeof elementToScore.parentNode.tagName === "undefined") {
|
|
908
|
+
return;
|
|
909
|
+
}
|
|
910
|
+
var innerText = this._getInnerText(elementToScore);
|
|
911
|
+
if (innerText.length < 25) {
|
|
912
|
+
return;
|
|
913
|
+
}
|
|
914
|
+
var ancestors2 = this._getNodeAncestors(elementToScore, 5);
|
|
915
|
+
if (ancestors2.length === 0) {
|
|
916
|
+
return;
|
|
917
|
+
}
|
|
918
|
+
var contentScore = 0;
|
|
919
|
+
contentScore += 1;
|
|
920
|
+
contentScore += innerText.split(this.REGEXPS.commas).length;
|
|
921
|
+
contentScore += Math.min(Math.floor(innerText.length / 100), 3);
|
|
922
|
+
this._forEachNode(ancestors2, function(ancestor, level) {
|
|
923
|
+
if (!ancestor.tagName || !ancestor.parentNode || typeof ancestor.parentNode.tagName === "undefined") {
|
|
924
|
+
return;
|
|
925
|
+
}
|
|
926
|
+
if (typeof ancestor.readability === "undefined") {
|
|
927
|
+
this._initializeNode(ancestor);
|
|
928
|
+
candidates.push(ancestor);
|
|
929
|
+
}
|
|
930
|
+
if (level === 0) {
|
|
931
|
+
var scoreDivider = 1;
|
|
932
|
+
} else if (level === 1) {
|
|
933
|
+
scoreDivider = 2;
|
|
934
|
+
} else {
|
|
935
|
+
scoreDivider = level * 3;
|
|
936
|
+
}
|
|
937
|
+
ancestor.readability.contentScore += contentScore / scoreDivider;
|
|
938
|
+
});
|
|
939
|
+
});
|
|
940
|
+
var topCandidates = [];
|
|
941
|
+
for (var c = 0, cl = candidates.length; c < cl; c += 1) {
|
|
942
|
+
var candidate = candidates[c];
|
|
943
|
+
var candidateScore = candidate.readability.contentScore * (1 - this._getLinkDensity(candidate));
|
|
944
|
+
candidate.readability.contentScore = candidateScore;
|
|
945
|
+
this.log("Candidate:", candidate, "with score " + candidateScore);
|
|
946
|
+
for (var t = 0; t < this._nbTopCandidates; t++) {
|
|
947
|
+
var aTopCandidate = topCandidates[t];
|
|
948
|
+
if (!aTopCandidate || candidateScore > aTopCandidate.readability.contentScore) {
|
|
949
|
+
topCandidates.splice(t, 0, candidate);
|
|
950
|
+
if (topCandidates.length > this._nbTopCandidates) {
|
|
951
|
+
topCandidates.pop();
|
|
952
|
+
}
|
|
953
|
+
break;
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
var topCandidate = topCandidates[0] || null;
|
|
958
|
+
var neededToCreateTopCandidate = false;
|
|
959
|
+
var parentOfTopCandidate;
|
|
960
|
+
if (topCandidate === null || topCandidate.tagName === "BODY") {
|
|
961
|
+
topCandidate = doc.createElement("DIV");
|
|
962
|
+
neededToCreateTopCandidate = true;
|
|
963
|
+
while (page.firstChild) {
|
|
964
|
+
this.log("Moving child out:", page.firstChild);
|
|
965
|
+
topCandidate.appendChild(page.firstChild);
|
|
966
|
+
}
|
|
967
|
+
page.appendChild(topCandidate);
|
|
968
|
+
this._initializeNode(topCandidate);
|
|
969
|
+
} else if (topCandidate) {
|
|
970
|
+
var alternativeCandidateAncestors = [];
|
|
971
|
+
for (var i = 1; i < topCandidates.length; i++) {
|
|
972
|
+
if (topCandidates[i].readability.contentScore / topCandidate.readability.contentScore >= 0.75) {
|
|
973
|
+
alternativeCandidateAncestors.push(
|
|
974
|
+
this._getNodeAncestors(topCandidates[i])
|
|
975
|
+
);
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
var MINIMUM_TOPCANDIDATES = 3;
|
|
979
|
+
if (alternativeCandidateAncestors.length >= MINIMUM_TOPCANDIDATES) {
|
|
980
|
+
parentOfTopCandidate = topCandidate.parentNode;
|
|
981
|
+
while (parentOfTopCandidate.tagName !== "BODY") {
|
|
982
|
+
var listsContainingThisAncestor = 0;
|
|
983
|
+
for (var ancestorIndex = 0; ancestorIndex < alternativeCandidateAncestors.length && listsContainingThisAncestor < MINIMUM_TOPCANDIDATES; ancestorIndex++) {
|
|
984
|
+
listsContainingThisAncestor += Number(
|
|
985
|
+
alternativeCandidateAncestors[ancestorIndex].includes(
|
|
986
|
+
parentOfTopCandidate
|
|
987
|
+
)
|
|
988
|
+
);
|
|
989
|
+
}
|
|
990
|
+
if (listsContainingThisAncestor >= MINIMUM_TOPCANDIDATES) {
|
|
991
|
+
topCandidate = parentOfTopCandidate;
|
|
992
|
+
break;
|
|
993
|
+
}
|
|
994
|
+
parentOfTopCandidate = parentOfTopCandidate.parentNode;
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
if (!topCandidate.readability) {
|
|
998
|
+
this._initializeNode(topCandidate);
|
|
999
|
+
}
|
|
1000
|
+
parentOfTopCandidate = topCandidate.parentNode;
|
|
1001
|
+
var lastScore = topCandidate.readability.contentScore;
|
|
1002
|
+
var scoreThreshold = lastScore / 3;
|
|
1003
|
+
while (parentOfTopCandidate.tagName !== "BODY") {
|
|
1004
|
+
if (!parentOfTopCandidate.readability) {
|
|
1005
|
+
parentOfTopCandidate = parentOfTopCandidate.parentNode;
|
|
1006
|
+
continue;
|
|
1007
|
+
}
|
|
1008
|
+
var parentScore = parentOfTopCandidate.readability.contentScore;
|
|
1009
|
+
if (parentScore < scoreThreshold) {
|
|
1010
|
+
break;
|
|
1011
|
+
}
|
|
1012
|
+
if (parentScore > lastScore) {
|
|
1013
|
+
topCandidate = parentOfTopCandidate;
|
|
1014
|
+
break;
|
|
1015
|
+
}
|
|
1016
|
+
lastScore = parentOfTopCandidate.readability.contentScore;
|
|
1017
|
+
parentOfTopCandidate = parentOfTopCandidate.parentNode;
|
|
1018
|
+
}
|
|
1019
|
+
parentOfTopCandidate = topCandidate.parentNode;
|
|
1020
|
+
while (parentOfTopCandidate.tagName != "BODY" && parentOfTopCandidate.children.length == 1) {
|
|
1021
|
+
topCandidate = parentOfTopCandidate;
|
|
1022
|
+
parentOfTopCandidate = topCandidate.parentNode;
|
|
1023
|
+
}
|
|
1024
|
+
if (!topCandidate.readability) {
|
|
1025
|
+
this._initializeNode(topCandidate);
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
var articleContent = doc.createElement("DIV");
|
|
1029
|
+
if (isPaging) {
|
|
1030
|
+
articleContent.id = "readability-content";
|
|
1031
|
+
}
|
|
1032
|
+
var siblingScoreThreshold = Math.max(
|
|
1033
|
+
10,
|
|
1034
|
+
topCandidate.readability.contentScore * 0.2
|
|
1035
|
+
);
|
|
1036
|
+
parentOfTopCandidate = topCandidate.parentNode;
|
|
1037
|
+
var siblings = parentOfTopCandidate.children;
|
|
1038
|
+
for (var s = 0, sl = siblings.length; s < sl; s++) {
|
|
1039
|
+
var sibling = siblings[s];
|
|
1040
|
+
var append = false;
|
|
1041
|
+
this.log(
|
|
1042
|
+
"Looking at sibling node:",
|
|
1043
|
+
sibling,
|
|
1044
|
+
sibling.readability ? "with score " + sibling.readability.contentScore : ""
|
|
1045
|
+
);
|
|
1046
|
+
this.log(
|
|
1047
|
+
"Sibling has score",
|
|
1048
|
+
sibling.readability ? sibling.readability.contentScore : "Unknown"
|
|
1049
|
+
);
|
|
1050
|
+
if (sibling === topCandidate) {
|
|
1051
|
+
append = true;
|
|
1052
|
+
} else {
|
|
1053
|
+
var contentBonus = 0;
|
|
1054
|
+
if (sibling.className === topCandidate.className && topCandidate.className !== "") {
|
|
1055
|
+
contentBonus += topCandidate.readability.contentScore * 0.2;
|
|
1056
|
+
}
|
|
1057
|
+
if (sibling.readability && sibling.readability.contentScore + contentBonus >= siblingScoreThreshold) {
|
|
1058
|
+
append = true;
|
|
1059
|
+
} else if (sibling.nodeName === "P") {
|
|
1060
|
+
var linkDensity = this._getLinkDensity(sibling);
|
|
1061
|
+
var nodeContent = this._getInnerText(sibling);
|
|
1062
|
+
var nodeLength = nodeContent.length;
|
|
1063
|
+
if (nodeLength > 80 && linkDensity < 0.25) {
|
|
1064
|
+
append = true;
|
|
1065
|
+
} else if (nodeLength < 80 && nodeLength > 0 && linkDensity === 0 && nodeContent.search(/\.( |$)/) !== -1) {
|
|
1066
|
+
append = true;
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
if (append) {
|
|
1071
|
+
this.log("Appending node:", sibling);
|
|
1072
|
+
if (!this.ALTER_TO_DIV_EXCEPTIONS.includes(sibling.nodeName)) {
|
|
1073
|
+
this.log("Altering sibling:", sibling, "to div.");
|
|
1074
|
+
sibling = this._setNodeTag(sibling, "DIV");
|
|
1075
|
+
}
|
|
1076
|
+
articleContent.appendChild(sibling);
|
|
1077
|
+
siblings = parentOfTopCandidate.children;
|
|
1078
|
+
s -= 1;
|
|
1079
|
+
sl -= 1;
|
|
1080
|
+
}
|
|
1081
|
+
}
|
|
1082
|
+
if (this._debug) {
|
|
1083
|
+
this.log("Article content pre-prep: " + articleContent.innerHTML);
|
|
1084
|
+
}
|
|
1085
|
+
this._prepArticle(articleContent);
|
|
1086
|
+
if (this._debug) {
|
|
1087
|
+
this.log("Article content post-prep: " + articleContent.innerHTML);
|
|
1088
|
+
}
|
|
1089
|
+
if (neededToCreateTopCandidate) {
|
|
1090
|
+
topCandidate.id = "readability-page-1";
|
|
1091
|
+
topCandidate.className = "page";
|
|
1092
|
+
} else {
|
|
1093
|
+
var div = doc.createElement("DIV");
|
|
1094
|
+
div.id = "readability-page-1";
|
|
1095
|
+
div.className = "page";
|
|
1096
|
+
while (articleContent.firstChild) {
|
|
1097
|
+
div.appendChild(articleContent.firstChild);
|
|
1098
|
+
}
|
|
1099
|
+
articleContent.appendChild(div);
|
|
1100
|
+
}
|
|
1101
|
+
if (this._debug) {
|
|
1102
|
+
this.log("Article content after paging: " + articleContent.innerHTML);
|
|
1103
|
+
}
|
|
1104
|
+
var parseSuccessful = true;
|
|
1105
|
+
var textLength = this._getInnerText(articleContent, true).length;
|
|
1106
|
+
if (textLength < this._charThreshold) {
|
|
1107
|
+
parseSuccessful = false;
|
|
1108
|
+
page.innerHTML = pageCacheHtml;
|
|
1109
|
+
this._attempts.push({
|
|
1110
|
+
articleContent,
|
|
1111
|
+
textLength
|
|
1112
|
+
});
|
|
1113
|
+
if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) {
|
|
1114
|
+
this._removeFlag(this.FLAG_STRIP_UNLIKELYS);
|
|
1115
|
+
} else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) {
|
|
1116
|
+
this._removeFlag(this.FLAG_WEIGHT_CLASSES);
|
|
1117
|
+
} else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) {
|
|
1118
|
+
this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY);
|
|
1119
|
+
} else {
|
|
1120
|
+
this._attempts.sort(function(a, b) {
|
|
1121
|
+
return b.textLength - a.textLength;
|
|
1122
|
+
});
|
|
1123
|
+
if (!this._attempts[0].textLength) {
|
|
1124
|
+
return null;
|
|
1125
|
+
}
|
|
1126
|
+
articleContent = this._attempts[0].articleContent;
|
|
1127
|
+
parseSuccessful = true;
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
if (parseSuccessful) {
|
|
1131
|
+
var ancestors = [parentOfTopCandidate, topCandidate].concat(
|
|
1132
|
+
this._getNodeAncestors(parentOfTopCandidate)
|
|
1133
|
+
);
|
|
1134
|
+
this._someNode(ancestors, function(ancestor) {
|
|
1135
|
+
if (!ancestor.tagName) {
|
|
1136
|
+
return false;
|
|
1137
|
+
}
|
|
1138
|
+
var articleDir = ancestor.getAttribute("dir");
|
|
1139
|
+
if (articleDir) {
|
|
1140
|
+
this._articleDir = articleDir;
|
|
1141
|
+
return true;
|
|
1142
|
+
}
|
|
1143
|
+
return false;
|
|
1144
|
+
});
|
|
1145
|
+
return articleContent;
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
},
|
|
1149
|
+
/**
|
|
1150
|
+
* Converts some of the common HTML entities in string to their corresponding characters.
|
|
1151
|
+
*
|
|
1152
|
+
* @param str {string} - a string to unescape.
|
|
1153
|
+
* @return string without HTML entity.
|
|
1154
|
+
*/
|
|
1155
|
+
_unescapeHtmlEntities(str) {
|
|
1156
|
+
if (!str) {
|
|
1157
|
+
return str;
|
|
1158
|
+
}
|
|
1159
|
+
var htmlEscapeMap = this.HTML_ESCAPE_MAP;
|
|
1160
|
+
return str.replace(/&(quot|amp|apos|lt|gt);/g, function(_, tag) {
|
|
1161
|
+
return htmlEscapeMap[tag];
|
|
1162
|
+
}).replace(/&#(?:x([0-9a-f]+)|([0-9]+));/gi, function(_, hex, numStr) {
|
|
1163
|
+
var num = parseInt(hex || numStr, hex ? 16 : 10);
|
|
1164
|
+
if (num == 0 || num > 1114111 || num >= 55296 && num <= 57343) {
|
|
1165
|
+
num = 65533;
|
|
1166
|
+
}
|
|
1167
|
+
return String.fromCodePoint(num);
|
|
1168
|
+
});
|
|
1169
|
+
},
|
|
1170
|
+
/**
|
|
1171
|
+
* Try to extract metadata from JSON-LD object.
|
|
1172
|
+
* For now, only Schema.org objects of type Article or its subtypes are supported.
|
|
1173
|
+
* @return Object with any metadata that could be extracted (possibly none)
|
|
1174
|
+
*/
|
|
1175
|
+
_getJSONLD(doc) {
|
|
1176
|
+
var scripts = this._getAllNodesWithTag(doc, ["script"]);
|
|
1177
|
+
var metadata;
|
|
1178
|
+
this._forEachNode(scripts, function(jsonLdElement) {
|
|
1179
|
+
if (!metadata && jsonLdElement.getAttribute("type") === "application/ld+json") {
|
|
1180
|
+
try {
|
|
1181
|
+
var content = jsonLdElement.textContent.replace(
|
|
1182
|
+
/^\s*<!\[CDATA\[|\]\]>\s*$/g,
|
|
1183
|
+
""
|
|
1184
|
+
);
|
|
1185
|
+
var parsed = JSON.parse(content);
|
|
1186
|
+
if (Array.isArray(parsed)) {
|
|
1187
|
+
parsed = parsed.find((it) => {
|
|
1188
|
+
return it["@type"] && it["@type"].match(this.REGEXPS.jsonLdArticleTypes);
|
|
1189
|
+
});
|
|
1190
|
+
if (!parsed) {
|
|
1191
|
+
return;
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
var schemaDotOrgRegex = /^https?\:\/\/schema\.org\/?$/;
|
|
1195
|
+
var matches = typeof parsed["@context"] === "string" && parsed["@context"].match(schemaDotOrgRegex) || typeof parsed["@context"] === "object" && typeof parsed["@context"]["@vocab"] == "string" && parsed["@context"]["@vocab"].match(schemaDotOrgRegex);
|
|
1196
|
+
if (!matches) {
|
|
1197
|
+
return;
|
|
1198
|
+
}
|
|
1199
|
+
if (!parsed["@type"] && Array.isArray(parsed["@graph"])) {
|
|
1200
|
+
parsed = parsed["@graph"].find((it) => {
|
|
1201
|
+
return (it["@type"] || "").match(this.REGEXPS.jsonLdArticleTypes);
|
|
1202
|
+
});
|
|
1203
|
+
}
|
|
1204
|
+
if (!parsed || !parsed["@type"] || !parsed["@type"].match(this.REGEXPS.jsonLdArticleTypes)) {
|
|
1205
|
+
return;
|
|
1206
|
+
}
|
|
1207
|
+
metadata = {};
|
|
1208
|
+
if (typeof parsed.name === "string" && typeof parsed.headline === "string" && parsed.name !== parsed.headline) {
|
|
1209
|
+
var title = this._getArticleTitle();
|
|
1210
|
+
var nameMatches = this._textSimilarity(parsed.name, title) > 0.75;
|
|
1211
|
+
var headlineMatches = this._textSimilarity(parsed.headline, title) > 0.75;
|
|
1212
|
+
if (headlineMatches && !nameMatches) {
|
|
1213
|
+
metadata.title = parsed.headline;
|
|
1214
|
+
} else {
|
|
1215
|
+
metadata.title = parsed.name;
|
|
1216
|
+
}
|
|
1217
|
+
} else if (typeof parsed.name === "string") {
|
|
1218
|
+
metadata.title = parsed.name.trim();
|
|
1219
|
+
} else if (typeof parsed.headline === "string") {
|
|
1220
|
+
metadata.title = parsed.headline.trim();
|
|
1221
|
+
}
|
|
1222
|
+
if (parsed.author) {
|
|
1223
|
+
if (typeof parsed.author.name === "string") {
|
|
1224
|
+
metadata.byline = parsed.author.name.trim();
|
|
1225
|
+
} else if (Array.isArray(parsed.author) && parsed.author[0] && typeof parsed.author[0].name === "string") {
|
|
1226
|
+
metadata.byline = parsed.author.filter(function(author) {
|
|
1227
|
+
return author && typeof author.name === "string";
|
|
1228
|
+
}).map(function(author) {
|
|
1229
|
+
return author.name.trim();
|
|
1230
|
+
}).join(", ");
|
|
1231
|
+
}
|
|
1232
|
+
}
|
|
1233
|
+
if (typeof parsed.description === "string") {
|
|
1234
|
+
metadata.excerpt = parsed.description.trim();
|
|
1235
|
+
}
|
|
1236
|
+
if (parsed.publisher && typeof parsed.publisher.name === "string") {
|
|
1237
|
+
metadata.siteName = parsed.publisher.name.trim();
|
|
1238
|
+
}
|
|
1239
|
+
if (typeof parsed.datePublished === "string") {
|
|
1240
|
+
metadata.datePublished = parsed.datePublished.trim();
|
|
1241
|
+
}
|
|
1242
|
+
} catch (err) {
|
|
1243
|
+
this.log(err.message);
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
});
|
|
1247
|
+
return metadata ? metadata : {};
|
|
1248
|
+
},
|
|
1249
|
+
/**
|
|
1250
|
+
* Attempts to get excerpt and byline metadata for the article.
|
|
1251
|
+
*
|
|
1252
|
+
* @param {Object} jsonld — object containing any metadata that
|
|
1253
|
+
* could be extracted from JSON-LD object.
|
|
1254
|
+
*
|
|
1255
|
+
* @return Object with optional "excerpt" and "byline" properties
|
|
1256
|
+
*/
|
|
1257
|
+
_getArticleMetadata(jsonld) {
|
|
1258
|
+
var metadata = {};
|
|
1259
|
+
var values = {};
|
|
1260
|
+
var metaElements = this._doc.getElementsByTagName("meta");
|
|
1261
|
+
var propertyPattern = /\s*(article|dc|dcterm|og|twitter)\s*:\s*(author|creator|description|published_time|title|site_name)\s*/gi;
|
|
1262
|
+
var namePattern = /^\s*(?:(dc|dcterm|og|twitter|parsely|weibo:(article|webpage))\s*[-\.:]\s*)?(author|creator|pub-date|description|title|site_name)\s*$/i;
|
|
1263
|
+
this._forEachNode(metaElements, function(element) {
|
|
1264
|
+
var elementName = element.getAttribute("name");
|
|
1265
|
+
var elementProperty = element.getAttribute("property");
|
|
1266
|
+
var content = element.getAttribute("content");
|
|
1267
|
+
if (!content) {
|
|
1268
|
+
return;
|
|
1269
|
+
}
|
|
1270
|
+
var matches = null;
|
|
1271
|
+
var name = null;
|
|
1272
|
+
if (elementProperty) {
|
|
1273
|
+
matches = elementProperty.match(propertyPattern);
|
|
1274
|
+
if (matches) {
|
|
1275
|
+
name = matches[0].toLowerCase().replace(/\s/g, "");
|
|
1276
|
+
values[name] = content.trim();
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
if (!matches && elementName && namePattern.test(elementName)) {
|
|
1280
|
+
name = elementName;
|
|
1281
|
+
if (content) {
|
|
1282
|
+
name = name.toLowerCase().replace(/\s/g, "").replace(/\./g, ":");
|
|
1283
|
+
values[name] = content.trim();
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
});
|
|
1287
|
+
metadata.title = jsonld.title || values["dc:title"] || values["dcterm:title"] || values["og:title"] || values["weibo:article:title"] || values["weibo:webpage:title"] || values.title || values["twitter:title"] || values["parsely-title"];
|
|
1288
|
+
if (!metadata.title) {
|
|
1289
|
+
metadata.title = this._getArticleTitle();
|
|
1290
|
+
}
|
|
1291
|
+
const articleAuthor = typeof values["article:author"] === "string" && !this._isUrl(values["article:author"]) ? values["article:author"] : void 0;
|
|
1292
|
+
metadata.byline = jsonld.byline || values["dc:creator"] || values["dcterm:creator"] || values.author || values["parsely-author"] || articleAuthor;
|
|
1293
|
+
metadata.excerpt = jsonld.excerpt || values["dc:description"] || values["dcterm:description"] || values["og:description"] || values["weibo:article:description"] || values["weibo:webpage:description"] || values.description || values["twitter:description"];
|
|
1294
|
+
metadata.siteName = jsonld.siteName || values["og:site_name"];
|
|
1295
|
+
metadata.publishedTime = jsonld.datePublished || values["article:published_time"] || values["parsely-pub-date"] || null;
|
|
1296
|
+
metadata.title = this._unescapeHtmlEntities(metadata.title);
|
|
1297
|
+
metadata.byline = this._unescapeHtmlEntities(metadata.byline);
|
|
1298
|
+
metadata.excerpt = this._unescapeHtmlEntities(metadata.excerpt);
|
|
1299
|
+
metadata.siteName = this._unescapeHtmlEntities(metadata.siteName);
|
|
1300
|
+
metadata.publishedTime = this._unescapeHtmlEntities(metadata.publishedTime);
|
|
1301
|
+
return metadata;
|
|
1302
|
+
},
|
|
1303
|
+
/**
|
|
1304
|
+
* Check if node is image, or if node contains exactly only one image
|
|
1305
|
+
* whether as a direct child or as its descendants.
|
|
1306
|
+
*
|
|
1307
|
+
* @param Element
|
|
1308
|
+
**/
|
|
1309
|
+
_isSingleImage(node) {
|
|
1310
|
+
while (node) {
|
|
1311
|
+
if (node.tagName === "IMG") {
|
|
1312
|
+
return true;
|
|
1313
|
+
}
|
|
1314
|
+
if (node.children.length !== 1 || node.textContent.trim() !== "") {
|
|
1315
|
+
return false;
|
|
1316
|
+
}
|
|
1317
|
+
node = node.children[0];
|
|
1318
|
+
}
|
|
1319
|
+
return false;
|
|
1320
|
+
},
|
|
1321
|
+
/**
|
|
1322
|
+
* Find all <noscript> that are located after <img> nodes, and which contain only one
|
|
1323
|
+
* <img> element. Replace the first image with the image from inside the <noscript> tag,
|
|
1324
|
+
* and remove the <noscript> tag. This improves the quality of the images we use on
|
|
1325
|
+
* some sites (e.g. Medium).
|
|
1326
|
+
*
|
|
1327
|
+
* @param Element
|
|
1328
|
+
**/
|
|
1329
|
+
_unwrapNoscriptImages(doc) {
|
|
1330
|
+
var imgs = Array.from(doc.getElementsByTagName("img"));
|
|
1331
|
+
this._forEachNode(imgs, function(img) {
|
|
1332
|
+
for (var i = 0; i < img.attributes.length; i++) {
|
|
1333
|
+
var attr = img.attributes[i];
|
|
1334
|
+
switch (attr.name) {
|
|
1335
|
+
case "src":
|
|
1336
|
+
case "srcset":
|
|
1337
|
+
case "data-src":
|
|
1338
|
+
case "data-srcset":
|
|
1339
|
+
return;
|
|
1340
|
+
}
|
|
1341
|
+
if (/\.(jpg|jpeg|png|webp)/i.test(attr.value)) {
|
|
1342
|
+
return;
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
img.remove();
|
|
1346
|
+
});
|
|
1347
|
+
var noscripts = Array.from(doc.getElementsByTagName("noscript"));
|
|
1348
|
+
this._forEachNode(noscripts, function(noscript) {
|
|
1349
|
+
if (!this._isSingleImage(noscript)) {
|
|
1350
|
+
return;
|
|
1351
|
+
}
|
|
1352
|
+
var tmp = doc.createElement("div");
|
|
1353
|
+
tmp.innerHTML = noscript.innerHTML;
|
|
1354
|
+
var prevElement = noscript.previousElementSibling;
|
|
1355
|
+
if (prevElement && this._isSingleImage(prevElement)) {
|
|
1356
|
+
var prevImg = prevElement;
|
|
1357
|
+
if (prevImg.tagName !== "IMG") {
|
|
1358
|
+
prevImg = prevElement.getElementsByTagName("img")[0];
|
|
1359
|
+
}
|
|
1360
|
+
var newImg = tmp.getElementsByTagName("img")[0];
|
|
1361
|
+
for (var i = 0; i < prevImg.attributes.length; i++) {
|
|
1362
|
+
var attr = prevImg.attributes[i];
|
|
1363
|
+
if (attr.value === "") {
|
|
1364
|
+
continue;
|
|
1365
|
+
}
|
|
1366
|
+
if (attr.name === "src" || attr.name === "srcset" || /\.(jpg|jpeg|png|webp)/i.test(attr.value)) {
|
|
1367
|
+
if (newImg.getAttribute(attr.name) === attr.value) {
|
|
1368
|
+
continue;
|
|
1369
|
+
}
|
|
1370
|
+
var attrName = attr.name;
|
|
1371
|
+
if (newImg.hasAttribute(attrName)) {
|
|
1372
|
+
attrName = "data-old-" + attrName;
|
|
1373
|
+
}
|
|
1374
|
+
newImg.setAttribute(attrName, attr.value);
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
noscript.parentNode.replaceChild(tmp.firstElementChild, prevElement);
|
|
1378
|
+
}
|
|
1379
|
+
});
|
|
1380
|
+
},
|
|
1381
|
+
/**
|
|
1382
|
+
* Removes script tags from the document.
|
|
1383
|
+
*
|
|
1384
|
+
* @param Element
|
|
1385
|
+
**/
|
|
1386
|
+
_removeScripts(doc) {
|
|
1387
|
+
this._removeNodes(this._getAllNodesWithTag(doc, ["script", "noscript"]));
|
|
1388
|
+
},
|
|
1389
|
+
/**
|
|
1390
|
+
* Check if this node has only whitespace and a single element with given tag
|
|
1391
|
+
* Returns false if the DIV node contains non-empty text nodes
|
|
1392
|
+
* or if it contains no element with given tag or more than 1 element.
|
|
1393
|
+
*
|
|
1394
|
+
* @param Element
|
|
1395
|
+
* @param string tag of child element
|
|
1396
|
+
**/
|
|
1397
|
+
_hasSingleTagInsideElement(element, tag) {
|
|
1398
|
+
if (element.children.length != 1 || element.children[0].tagName !== tag) {
|
|
1399
|
+
return false;
|
|
1400
|
+
}
|
|
1401
|
+
return !this._someNode(element.childNodes, function(node) {
|
|
1402
|
+
return node.nodeType === this.TEXT_NODE && this.REGEXPS.hasContent.test(node.textContent);
|
|
1403
|
+
});
|
|
1404
|
+
},
|
|
1405
|
+
_isElementWithoutContent(node) {
|
|
1406
|
+
return node.nodeType === this.ELEMENT_NODE && !node.textContent.trim().length && (!node.children.length || node.children.length == node.getElementsByTagName("br").length + node.getElementsByTagName("hr").length);
|
|
1407
|
+
},
|
|
1408
|
+
/**
|
|
1409
|
+
* Determine whether element has any children block level elements.
|
|
1410
|
+
*
|
|
1411
|
+
* @param Element
|
|
1412
|
+
*/
|
|
1413
|
+
_hasChildBlockElement(element) {
|
|
1414
|
+
return this._someNode(element.childNodes, function(node) {
|
|
1415
|
+
return this.DIV_TO_P_ELEMS.has(node.tagName) || this._hasChildBlockElement(node);
|
|
1416
|
+
});
|
|
1417
|
+
},
|
|
1418
|
+
/***
|
|
1419
|
+
* Determine if a node qualifies as phrasing content.
|
|
1420
|
+
* https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/Content_categories#Phrasing_content
|
|
1421
|
+
**/
|
|
1422
|
+
_isPhrasingContent(node) {
|
|
1423
|
+
return node.nodeType === this.TEXT_NODE || this.PHRASING_ELEMS.includes(node.tagName) || (node.tagName === "A" || node.tagName === "DEL" || node.tagName === "INS") && this._everyNode(node.childNodes, this._isPhrasingContent);
|
|
1424
|
+
},
|
|
1425
|
+
_isWhitespace(node) {
|
|
1426
|
+
return node.nodeType === this.TEXT_NODE && node.textContent.trim().length === 0 || node.nodeType === this.ELEMENT_NODE && node.tagName === "BR";
|
|
1427
|
+
},
|
|
1428
|
+
/**
|
|
1429
|
+
* Get the inner text of a node - cross browser compatibly.
|
|
1430
|
+
* This also strips out any excess whitespace to be found.
|
|
1431
|
+
*
|
|
1432
|
+
* @param Element
|
|
1433
|
+
* @param Boolean normalizeSpaces (default: true)
|
|
1434
|
+
* @return string
|
|
1435
|
+
**/
|
|
1436
|
+
_getInnerText(e, normalizeSpaces) {
|
|
1437
|
+
normalizeSpaces = typeof normalizeSpaces === "undefined" ? true : normalizeSpaces;
|
|
1438
|
+
var textContent = e.textContent.trim();
|
|
1439
|
+
if (normalizeSpaces) {
|
|
1440
|
+
return textContent.replace(this.REGEXPS.normalize, " ");
|
|
1441
|
+
}
|
|
1442
|
+
return textContent;
|
|
1443
|
+
},
|
|
1444
|
+
/**
|
|
1445
|
+
* Get the number of times a string s appears in the node e.
|
|
1446
|
+
*
|
|
1447
|
+
* @param Element
|
|
1448
|
+
* @param string - what to split on. Default is ","
|
|
1449
|
+
* @return number (integer)
|
|
1450
|
+
**/
|
|
1451
|
+
_getCharCount(e, s) {
|
|
1452
|
+
s = s || ",";
|
|
1453
|
+
return this._getInnerText(e).split(s).length - 1;
|
|
1454
|
+
},
|
|
1455
|
+
/**
|
|
1456
|
+
* Remove the style attribute on every e and under.
|
|
1457
|
+
* TODO: Test if getElementsByTagName(*) is faster.
|
|
1458
|
+
*
|
|
1459
|
+
* @param Element
|
|
1460
|
+
* @return void
|
|
1461
|
+
**/
|
|
1462
|
+
_cleanStyles(e) {
|
|
1463
|
+
if (!e || e.tagName.toLowerCase() === "svg") {
|
|
1464
|
+
return;
|
|
1465
|
+
}
|
|
1466
|
+
for (var i = 0; i < this.PRESENTATIONAL_ATTRIBUTES.length; i++) {
|
|
1467
|
+
e.removeAttribute(this.PRESENTATIONAL_ATTRIBUTES[i]);
|
|
1468
|
+
}
|
|
1469
|
+
if (this.DEPRECATED_SIZE_ATTRIBUTE_ELEMS.includes(e.tagName)) {
|
|
1470
|
+
e.removeAttribute("width");
|
|
1471
|
+
e.removeAttribute("height");
|
|
1472
|
+
}
|
|
1473
|
+
var cur = e.firstElementChild;
|
|
1474
|
+
while (cur !== null) {
|
|
1475
|
+
this._cleanStyles(cur);
|
|
1476
|
+
cur = cur.nextElementSibling;
|
|
1477
|
+
}
|
|
1478
|
+
},
|
|
1479
|
+
/**
|
|
1480
|
+
* Get the density of links as a percentage of the content
|
|
1481
|
+
* This is the amount of text that is inside a link divided by the total text in the node.
|
|
1482
|
+
*
|
|
1483
|
+
* @param Element
|
|
1484
|
+
* @return number (float)
|
|
1485
|
+
**/
|
|
1486
|
+
_getLinkDensity(element) {
|
|
1487
|
+
var textLength = this._getInnerText(element).length;
|
|
1488
|
+
if (textLength === 0) {
|
|
1489
|
+
return 0;
|
|
1490
|
+
}
|
|
1491
|
+
var linkLength = 0;
|
|
1492
|
+
this._forEachNode(element.getElementsByTagName("a"), function(linkNode) {
|
|
1493
|
+
var href = linkNode.getAttribute("href");
|
|
1494
|
+
var coefficient = href && this.REGEXPS.hashUrl.test(href) ? 0.3 : 1;
|
|
1495
|
+
linkLength += this._getInnerText(linkNode).length * coefficient;
|
|
1496
|
+
});
|
|
1497
|
+
return linkLength / textLength;
|
|
1498
|
+
},
|
|
1499
|
+
/**
|
|
1500
|
+
* Get an elements class/id weight. Uses regular expressions to tell if this
|
|
1501
|
+
* element looks good or bad.
|
|
1502
|
+
*
|
|
1503
|
+
* @param Element
|
|
1504
|
+
* @return number (Integer)
|
|
1505
|
+
**/
|
|
1506
|
+
_getClassWeight(e) {
|
|
1507
|
+
if (!this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) {
|
|
1508
|
+
return 0;
|
|
1509
|
+
}
|
|
1510
|
+
var weight = 0;
|
|
1511
|
+
if (typeof e.className === "string" && e.className !== "") {
|
|
1512
|
+
if (this.REGEXPS.negative.test(e.className)) {
|
|
1513
|
+
weight -= 25;
|
|
1514
|
+
}
|
|
1515
|
+
if (this.REGEXPS.positive.test(e.className)) {
|
|
1516
|
+
weight += 25;
|
|
1517
|
+
}
|
|
1518
|
+
}
|
|
1519
|
+
if (typeof e.id === "string" && e.id !== "") {
|
|
1520
|
+
if (this.REGEXPS.negative.test(e.id)) {
|
|
1521
|
+
weight -= 25;
|
|
1522
|
+
}
|
|
1523
|
+
if (this.REGEXPS.positive.test(e.id)) {
|
|
1524
|
+
weight += 25;
|
|
1525
|
+
}
|
|
1526
|
+
}
|
|
1527
|
+
return weight;
|
|
1528
|
+
},
|
|
1529
|
+
/**
|
|
1530
|
+
* Clean a node of all elements of type "tag".
|
|
1531
|
+
* (Unless it's a youtube/vimeo video. People love movies.)
|
|
1532
|
+
*
|
|
1533
|
+
* @param Element
|
|
1534
|
+
* @param string tag to clean
|
|
1535
|
+
* @return void
|
|
1536
|
+
**/
|
|
1537
|
+
_clean(e, tag) {
|
|
1538
|
+
var isEmbed = ["object", "embed", "iframe"].includes(tag);
|
|
1539
|
+
this._removeNodes(this._getAllNodesWithTag(e, [tag]), function(element) {
|
|
1540
|
+
if (isEmbed) {
|
|
1541
|
+
for (var i = 0; i < element.attributes.length; i++) {
|
|
1542
|
+
if (this._allowedVideoRegex.test(element.attributes[i].value)) {
|
|
1543
|
+
return false;
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
if (element.tagName === "object" && this._allowedVideoRegex.test(element.innerHTML)) {
|
|
1547
|
+
return false;
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
return true;
|
|
1551
|
+
});
|
|
1552
|
+
},
|
|
1553
|
+
/**
|
|
1554
|
+
* Check if a given node has one of its ancestor tag name matching the
|
|
1555
|
+
* provided one.
|
|
1556
|
+
* @param HTMLElement node
|
|
1557
|
+
* @param String tagName
|
|
1558
|
+
* @param Number maxDepth
|
|
1559
|
+
* @param Function filterFn a filter to invoke to determine whether this node 'counts'
|
|
1560
|
+
* @return Boolean
|
|
1561
|
+
*/
|
|
1562
|
+
_hasAncestorTag(node, tagName, maxDepth, filterFn) {
|
|
1563
|
+
maxDepth = maxDepth || 3;
|
|
1564
|
+
tagName = tagName.toUpperCase();
|
|
1565
|
+
var depth = 0;
|
|
1566
|
+
while (node.parentNode) {
|
|
1567
|
+
if (maxDepth > 0 && depth > maxDepth) {
|
|
1568
|
+
return false;
|
|
1569
|
+
}
|
|
1570
|
+
if (node.parentNode.tagName === tagName && (!filterFn || filterFn(node.parentNode))) {
|
|
1571
|
+
return true;
|
|
1572
|
+
}
|
|
1573
|
+
node = node.parentNode;
|
|
1574
|
+
depth++;
|
|
1575
|
+
}
|
|
1576
|
+
return false;
|
|
1577
|
+
},
|
|
1578
|
+
/**
|
|
1579
|
+
* Return an object indicating how many rows and columns this table has.
|
|
1580
|
+
*/
|
|
1581
|
+
_getRowAndColumnCount(table) {
|
|
1582
|
+
var rows = 0;
|
|
1583
|
+
var columns = 0;
|
|
1584
|
+
var trs = table.getElementsByTagName("tr");
|
|
1585
|
+
for (var i = 0; i < trs.length; i++) {
|
|
1586
|
+
var rowspan = trs[i].getAttribute("rowspan") || 0;
|
|
1587
|
+
if (rowspan) {
|
|
1588
|
+
rowspan = parseInt(rowspan, 10);
|
|
1589
|
+
}
|
|
1590
|
+
rows += rowspan || 1;
|
|
1591
|
+
var columnsInThisRow = 0;
|
|
1592
|
+
var cells = trs[i].getElementsByTagName("td");
|
|
1593
|
+
for (var j = 0; j < cells.length; j++) {
|
|
1594
|
+
var colspan = cells[j].getAttribute("colspan") || 0;
|
|
1595
|
+
if (colspan) {
|
|
1596
|
+
colspan = parseInt(colspan, 10);
|
|
1597
|
+
}
|
|
1598
|
+
columnsInThisRow += colspan || 1;
|
|
1599
|
+
}
|
|
1600
|
+
columns = Math.max(columns, columnsInThisRow);
|
|
1601
|
+
}
|
|
1602
|
+
return { rows, columns };
|
|
1603
|
+
},
|
|
1604
|
+
/**
|
|
1605
|
+
* Look for 'data' (as opposed to 'layout') tables, for which we use
|
|
1606
|
+
* similar checks as
|
|
1607
|
+
* https://searchfox.org/mozilla-central/rev/f82d5c549f046cb64ce5602bfd894b7ae807c8f8/accessible/generic/TableAccessible.cpp#19
|
|
1608
|
+
*/
|
|
1609
|
+
_markDataTables(root) {
|
|
1610
|
+
var tables = root.getElementsByTagName("table");
|
|
1611
|
+
for (var i = 0; i < tables.length; i++) {
|
|
1612
|
+
var table = tables[i];
|
|
1613
|
+
var role = table.getAttribute("role");
|
|
1614
|
+
if (role == "presentation") {
|
|
1615
|
+
table._readabilityDataTable = false;
|
|
1616
|
+
continue;
|
|
1617
|
+
}
|
|
1618
|
+
var datatable = table.getAttribute("datatable");
|
|
1619
|
+
if (datatable == "0") {
|
|
1620
|
+
table._readabilityDataTable = false;
|
|
1621
|
+
continue;
|
|
1622
|
+
}
|
|
1623
|
+
var summary = table.getAttribute("summary");
|
|
1624
|
+
if (summary) {
|
|
1625
|
+
table._readabilityDataTable = true;
|
|
1626
|
+
continue;
|
|
1627
|
+
}
|
|
1628
|
+
var caption = table.getElementsByTagName("caption")[0];
|
|
1629
|
+
if (caption && caption.childNodes.length) {
|
|
1630
|
+
table._readabilityDataTable = true;
|
|
1631
|
+
continue;
|
|
1632
|
+
}
|
|
1633
|
+
var dataTableDescendants = ["col", "colgroup", "tfoot", "thead", "th"];
|
|
1634
|
+
var descendantExists = function(tag) {
|
|
1635
|
+
return !!table.getElementsByTagName(tag)[0];
|
|
1636
|
+
};
|
|
1637
|
+
if (dataTableDescendants.some(descendantExists)) {
|
|
1638
|
+
this.log("Data table because found data-y descendant");
|
|
1639
|
+
table._readabilityDataTable = true;
|
|
1640
|
+
continue;
|
|
1641
|
+
}
|
|
1642
|
+
if (table.getElementsByTagName("table")[0]) {
|
|
1643
|
+
table._readabilityDataTable = false;
|
|
1644
|
+
continue;
|
|
1645
|
+
}
|
|
1646
|
+
var sizeInfo = this._getRowAndColumnCount(table);
|
|
1647
|
+
if (sizeInfo.columns == 1 || sizeInfo.rows == 1) {
|
|
1648
|
+
table._readabilityDataTable = false;
|
|
1649
|
+
continue;
|
|
1650
|
+
}
|
|
1651
|
+
if (sizeInfo.rows >= 10 || sizeInfo.columns > 4) {
|
|
1652
|
+
table._readabilityDataTable = true;
|
|
1653
|
+
continue;
|
|
1654
|
+
}
|
|
1655
|
+
table._readabilityDataTable = sizeInfo.rows * sizeInfo.columns > 10;
|
|
1656
|
+
}
|
|
1657
|
+
},
|
|
1658
|
+
/* convert images and figures that have properties like data-src into images that can be loaded without JS */
|
|
1659
|
+
_fixLazyImages(root) {
|
|
1660
|
+
this._forEachNode(
|
|
1661
|
+
this._getAllNodesWithTag(root, ["img", "picture", "figure"]),
|
|
1662
|
+
function(elem) {
|
|
1663
|
+
if (elem.src && this.REGEXPS.b64DataUrl.test(elem.src)) {
|
|
1664
|
+
var parts = this.REGEXPS.b64DataUrl.exec(elem.src);
|
|
1665
|
+
if (parts[1] === "image/svg+xml") {
|
|
1666
|
+
return;
|
|
1667
|
+
}
|
|
1668
|
+
var srcCouldBeRemoved = false;
|
|
1669
|
+
for (var i = 0; i < elem.attributes.length; i++) {
|
|
1670
|
+
var attr = elem.attributes[i];
|
|
1671
|
+
if (attr.name === "src") {
|
|
1672
|
+
continue;
|
|
1673
|
+
}
|
|
1674
|
+
if (/\.(jpg|jpeg|png|webp)/i.test(attr.value)) {
|
|
1675
|
+
srcCouldBeRemoved = true;
|
|
1676
|
+
break;
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
if (srcCouldBeRemoved) {
|
|
1680
|
+
var b64starts = parts[0].length;
|
|
1681
|
+
var b64length = elem.src.length - b64starts;
|
|
1682
|
+
if (b64length < 133) {
|
|
1683
|
+
elem.removeAttribute("src");
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
if ((elem.src || elem.srcset && elem.srcset != "null") && !elem.className.toLowerCase().includes("lazy")) {
|
|
1688
|
+
return;
|
|
1689
|
+
}
|
|
1690
|
+
for (var j = 0; j < elem.attributes.length; j++) {
|
|
1691
|
+
attr = elem.attributes[j];
|
|
1692
|
+
if (attr.name === "src" || attr.name === "srcset" || attr.name === "alt") {
|
|
1693
|
+
continue;
|
|
1694
|
+
}
|
|
1695
|
+
var copyTo = null;
|
|
1696
|
+
if (/\.(jpg|jpeg|png|webp)\s+\d/.test(attr.value)) {
|
|
1697
|
+
copyTo = "srcset";
|
|
1698
|
+
} else if (/^\s*\S+\.(jpg|jpeg|png|webp)\S*\s*$/.test(attr.value)) {
|
|
1699
|
+
copyTo = "src";
|
|
1700
|
+
}
|
|
1701
|
+
if (copyTo) {
|
|
1702
|
+
if (elem.tagName === "IMG" || elem.tagName === "PICTURE") {
|
|
1703
|
+
elem.setAttribute(copyTo, attr.value);
|
|
1704
|
+
} else if (elem.tagName === "FIGURE" && !this._getAllNodesWithTag(elem, ["img", "picture"]).length) {
|
|
1705
|
+
var img = this._doc.createElement("img");
|
|
1706
|
+
img.setAttribute(copyTo, attr.value);
|
|
1707
|
+
elem.appendChild(img);
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
);
|
|
1713
|
+
},
|
|
1714
|
+
_getTextDensity(e, tags) {
|
|
1715
|
+
var textLength = this._getInnerText(e, true).length;
|
|
1716
|
+
if (textLength === 0) {
|
|
1717
|
+
return 0;
|
|
1718
|
+
}
|
|
1719
|
+
var childrenLength = 0;
|
|
1720
|
+
var children = this._getAllNodesWithTag(e, tags);
|
|
1721
|
+
this._forEachNode(
|
|
1722
|
+
children,
|
|
1723
|
+
(child) => childrenLength += this._getInnerText(child, true).length
|
|
1724
|
+
);
|
|
1725
|
+
return childrenLength / textLength;
|
|
1726
|
+
},
|
|
1727
|
+
/**
|
|
1728
|
+
* Clean an element of all tags of type "tag" if they look fishy.
|
|
1729
|
+
* "Fishy" is an algorithm based on content length, classnames, link density, number of images & embeds, etc.
|
|
1730
|
+
*
|
|
1731
|
+
* @return void
|
|
1732
|
+
**/
|
|
1733
|
+
_cleanConditionally(e, tag) {
|
|
1734
|
+
if (!this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) {
|
|
1735
|
+
return;
|
|
1736
|
+
}
|
|
1737
|
+
this._removeNodes(this._getAllNodesWithTag(e, [tag]), function(node) {
|
|
1738
|
+
var isDataTable = function(t) {
|
|
1739
|
+
return t._readabilityDataTable;
|
|
1740
|
+
};
|
|
1741
|
+
var isList = tag === "ul" || tag === "ol";
|
|
1742
|
+
if (!isList) {
|
|
1743
|
+
var listLength = 0;
|
|
1744
|
+
var listNodes = this._getAllNodesWithTag(node, ["ul", "ol"]);
|
|
1745
|
+
this._forEachNode(
|
|
1746
|
+
listNodes,
|
|
1747
|
+
(list) => listLength += this._getInnerText(list).length
|
|
1748
|
+
);
|
|
1749
|
+
isList = listLength / this._getInnerText(node).length > 0.9;
|
|
1750
|
+
}
|
|
1751
|
+
if (tag === "table" && isDataTable(node)) {
|
|
1752
|
+
return false;
|
|
1753
|
+
}
|
|
1754
|
+
if (this._hasAncestorTag(node, "table", -1, isDataTable)) {
|
|
1755
|
+
return false;
|
|
1756
|
+
}
|
|
1757
|
+
if (this._hasAncestorTag(node, "code")) {
|
|
1758
|
+
return false;
|
|
1759
|
+
}
|
|
1760
|
+
if ([...node.getElementsByTagName("table")].some(
|
|
1761
|
+
(tbl) => tbl._readabilityDataTable
|
|
1762
|
+
)) {
|
|
1763
|
+
return false;
|
|
1764
|
+
}
|
|
1765
|
+
var weight = this._getClassWeight(node);
|
|
1766
|
+
this.log("Cleaning Conditionally", node);
|
|
1767
|
+
var contentScore = 0;
|
|
1768
|
+
if (weight + contentScore < 0) {
|
|
1769
|
+
return true;
|
|
1770
|
+
}
|
|
1771
|
+
if (this._getCharCount(node, ",") < 10) {
|
|
1772
|
+
var p = node.getElementsByTagName("p").length;
|
|
1773
|
+
var img = node.getElementsByTagName("img").length;
|
|
1774
|
+
var li = node.getElementsByTagName("li").length - 100;
|
|
1775
|
+
var input = node.getElementsByTagName("input").length;
|
|
1776
|
+
var headingDensity = this._getTextDensity(node, [
|
|
1777
|
+
"h1",
|
|
1778
|
+
"h2",
|
|
1779
|
+
"h3",
|
|
1780
|
+
"h4",
|
|
1781
|
+
"h5",
|
|
1782
|
+
"h6"
|
|
1783
|
+
]);
|
|
1784
|
+
var embedCount = 0;
|
|
1785
|
+
var embeds = this._getAllNodesWithTag(node, [
|
|
1786
|
+
"object",
|
|
1787
|
+
"embed",
|
|
1788
|
+
"iframe"
|
|
1789
|
+
]);
|
|
1790
|
+
for (var i = 0; i < embeds.length; i++) {
|
|
1791
|
+
for (var j = 0; j < embeds[i].attributes.length; j++) {
|
|
1792
|
+
if (this._allowedVideoRegex.test(embeds[i].attributes[j].value)) {
|
|
1793
|
+
return false;
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1796
|
+
if (embeds[i].tagName === "object" && this._allowedVideoRegex.test(embeds[i].innerHTML)) {
|
|
1797
|
+
return false;
|
|
1798
|
+
}
|
|
1799
|
+
embedCount++;
|
|
1800
|
+
}
|
|
1801
|
+
var innerText = this._getInnerText(node);
|
|
1802
|
+
if (this.REGEXPS.adWords.test(innerText) || this.REGEXPS.loadingWords.test(innerText)) {
|
|
1803
|
+
return true;
|
|
1804
|
+
}
|
|
1805
|
+
var contentLength = innerText.length;
|
|
1806
|
+
var linkDensity = this._getLinkDensity(node);
|
|
1807
|
+
var textishTags = ["SPAN", "LI", "TD"].concat(
|
|
1808
|
+
Array.from(this.DIV_TO_P_ELEMS)
|
|
1809
|
+
);
|
|
1810
|
+
var textDensity = this._getTextDensity(node, textishTags);
|
|
1811
|
+
var isFigureChild = this._hasAncestorTag(node, "figure");
|
|
1812
|
+
const shouldRemoveNode = () => {
|
|
1813
|
+
const errs = [];
|
|
1814
|
+
if (!isFigureChild && img > 1 && p / img < 0.5) {
|
|
1815
|
+
errs.push(`Bad p to img ratio (img=${img}, p=${p})`);
|
|
1816
|
+
}
|
|
1817
|
+
if (!isList && li > p) {
|
|
1818
|
+
errs.push(`Too many li's outside of a list. (li=${li} > p=${p})`);
|
|
1819
|
+
}
|
|
1820
|
+
if (input > Math.floor(p / 3)) {
|
|
1821
|
+
errs.push(`Too many inputs per p. (input=${input}, p=${p})`);
|
|
1822
|
+
}
|
|
1823
|
+
if (!isList && !isFigureChild && headingDensity < 0.9 && contentLength < 25 && (img === 0 || img > 2) && linkDensity > 0) {
|
|
1824
|
+
errs.push(
|
|
1825
|
+
`Suspiciously short. (headingDensity=${headingDensity}, img=${img}, linkDensity=${linkDensity})`
|
|
1826
|
+
);
|
|
1827
|
+
}
|
|
1828
|
+
if (!isList && weight < 25 && linkDensity > 0.2 + this._linkDensityModifier) {
|
|
1829
|
+
errs.push(
|
|
1830
|
+
`Low weight and a little linky. (linkDensity=${linkDensity})`
|
|
1831
|
+
);
|
|
1832
|
+
}
|
|
1833
|
+
if (weight >= 25 && linkDensity > 0.5 + this._linkDensityModifier) {
|
|
1834
|
+
errs.push(
|
|
1835
|
+
`High weight and mostly links. (linkDensity=${linkDensity})`
|
|
1836
|
+
);
|
|
1837
|
+
}
|
|
1838
|
+
if (embedCount === 1 && contentLength < 75 || embedCount > 1) {
|
|
1839
|
+
errs.push(
|
|
1840
|
+
`Suspicious embed. (embedCount=${embedCount}, contentLength=${contentLength})`
|
|
1841
|
+
);
|
|
1842
|
+
}
|
|
1843
|
+
if (img === 0 && textDensity === 0) {
|
|
1844
|
+
errs.push(
|
|
1845
|
+
`No useful content. (img=${img}, textDensity=${textDensity})`
|
|
1846
|
+
);
|
|
1847
|
+
}
|
|
1848
|
+
if (errs.length) {
|
|
1849
|
+
this.log("Checks failed", errs);
|
|
1850
|
+
return true;
|
|
1851
|
+
}
|
|
1852
|
+
return false;
|
|
1853
|
+
};
|
|
1854
|
+
var haveToRemove = shouldRemoveNode();
|
|
1855
|
+
if (isList && haveToRemove) {
|
|
1856
|
+
for (var x = 0; x < node.children.length; x++) {
|
|
1857
|
+
let child = node.children[x];
|
|
1858
|
+
if (child.children.length > 1) {
|
|
1859
|
+
return haveToRemove;
|
|
1860
|
+
}
|
|
1861
|
+
}
|
|
1862
|
+
let li_count = node.getElementsByTagName("li").length;
|
|
1863
|
+
if (img == li_count) {
|
|
1864
|
+
return false;
|
|
1865
|
+
}
|
|
1866
|
+
}
|
|
1867
|
+
return haveToRemove;
|
|
1868
|
+
}
|
|
1869
|
+
return false;
|
|
1870
|
+
});
|
|
1871
|
+
},
|
|
1872
|
+
/**
|
|
1873
|
+
* Clean out elements that match the specified conditions
|
|
1874
|
+
*
|
|
1875
|
+
* @param Element
|
|
1876
|
+
* @param Function determines whether a node should be removed
|
|
1877
|
+
* @return void
|
|
1878
|
+
**/
|
|
1879
|
+
_cleanMatchedNodes(e, filter) {
|
|
1880
|
+
var endOfSearchMarkerNode = this._getNextNode(e, true);
|
|
1881
|
+
var next = this._getNextNode(e);
|
|
1882
|
+
while (next && next != endOfSearchMarkerNode) {
|
|
1883
|
+
if (filter.call(this, next, next.className + " " + next.id)) {
|
|
1884
|
+
next = this._removeAndGetNext(next);
|
|
1885
|
+
} else {
|
|
1886
|
+
next = this._getNextNode(next);
|
|
1887
|
+
}
|
|
1888
|
+
}
|
|
1889
|
+
},
|
|
1890
|
+
/**
|
|
1891
|
+
* Clean out spurious headers from an Element.
|
|
1892
|
+
*
|
|
1893
|
+
* @param Element
|
|
1894
|
+
* @return void
|
|
1895
|
+
**/
|
|
1896
|
+
_cleanHeaders(e) {
|
|
1897
|
+
let headingNodes = this._getAllNodesWithTag(e, ["h1", "h2"]);
|
|
1898
|
+
this._removeNodes(headingNodes, function(node) {
|
|
1899
|
+
let shouldRemove = this._getClassWeight(node) < 0;
|
|
1900
|
+
if (shouldRemove) {
|
|
1901
|
+
this.log("Removing header with low class weight:", node);
|
|
1902
|
+
}
|
|
1903
|
+
return shouldRemove;
|
|
1904
|
+
});
|
|
1905
|
+
},
|
|
1906
|
+
/**
|
|
1907
|
+
* Check if this node is an H1 or H2 element whose content is mostly
|
|
1908
|
+
* the same as the article title.
|
|
1909
|
+
*
|
|
1910
|
+
* @param Element the node to check.
|
|
1911
|
+
* @return boolean indicating whether this is a title-like header.
|
|
1912
|
+
*/
|
|
1913
|
+
_headerDuplicatesTitle(node) {
|
|
1914
|
+
if (node.tagName != "H1" && node.tagName != "H2") {
|
|
1915
|
+
return false;
|
|
1916
|
+
}
|
|
1917
|
+
var heading = this._getInnerText(node, false);
|
|
1918
|
+
this.log("Evaluating similarity of header:", heading, this._articleTitle);
|
|
1919
|
+
return this._textSimilarity(this._articleTitle, heading) > 0.75;
|
|
1920
|
+
},
|
|
1921
|
+
_flagIsActive(flag) {
|
|
1922
|
+
return (this._flags & flag) > 0;
|
|
1923
|
+
},
|
|
1924
|
+
_removeFlag(flag) {
|
|
1925
|
+
this._flags = this._flags & ~flag;
|
|
1926
|
+
},
|
|
1927
|
+
_isProbablyVisible(node) {
|
|
1928
|
+
return (!node.style || node.style.display != "none") && (!node.style || node.style.visibility != "hidden") && !node.hasAttribute("hidden") && //check for "fallback-image" so that wikimedia math images are displayed
|
|
1929
|
+
(!node.hasAttribute("aria-hidden") || node.getAttribute("aria-hidden") != "true" || node.className && node.className.includes && node.className.includes("fallback-image"));
|
|
1930
|
+
},
|
|
1931
|
+
/**
|
|
1932
|
+
* Runs readability.
|
|
1933
|
+
*
|
|
1934
|
+
* Workflow:
|
|
1935
|
+
* 1. Prep the document by removing script tags, css, etc.
|
|
1936
|
+
* 2. Build readability's DOM tree.
|
|
1937
|
+
* 3. Grab the article content from the current dom tree.
|
|
1938
|
+
* 4. Replace the current DOM tree with the new one.
|
|
1939
|
+
* 5. Read peacefully.
|
|
1940
|
+
*
|
|
1941
|
+
* @return void
|
|
1942
|
+
**/
|
|
1943
|
+
parse() {
|
|
1944
|
+
if (this._maxElemsToParse > 0) {
|
|
1945
|
+
var numTags = this._doc.getElementsByTagName("*").length;
|
|
1946
|
+
if (numTags > this._maxElemsToParse) {
|
|
1947
|
+
throw new Error(
|
|
1948
|
+
"Aborting parsing document; " + numTags + " elements found"
|
|
1949
|
+
);
|
|
1950
|
+
}
|
|
1951
|
+
}
|
|
1952
|
+
this._unwrapNoscriptImages(this._doc);
|
|
1953
|
+
var jsonLd = this._disableJSONLD ? {} : this._getJSONLD(this._doc);
|
|
1954
|
+
this._removeScripts(this._doc);
|
|
1955
|
+
this._prepDocument();
|
|
1956
|
+
var metadata = this._getArticleMetadata(jsonLd);
|
|
1957
|
+
this._metadata = metadata;
|
|
1958
|
+
this._articleTitle = metadata.title;
|
|
1959
|
+
var articleContent = this._grabArticle();
|
|
1960
|
+
if (!articleContent) {
|
|
1961
|
+
return null;
|
|
1962
|
+
}
|
|
1963
|
+
this.log("Grabbed: " + articleContent.innerHTML);
|
|
1964
|
+
this._postProcessContent(articleContent);
|
|
1965
|
+
if (!metadata.excerpt) {
|
|
1966
|
+
var paragraphs = articleContent.getElementsByTagName("p");
|
|
1967
|
+
if (paragraphs.length) {
|
|
1968
|
+
metadata.excerpt = paragraphs[0].textContent.trim();
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
var textContent = articleContent.textContent;
|
|
1972
|
+
return {
|
|
1973
|
+
title: this._articleTitle,
|
|
1974
|
+
byline: metadata.byline || this._articleByline,
|
|
1975
|
+
dir: this._articleDir,
|
|
1976
|
+
lang: this._articleLang,
|
|
1977
|
+
content: this._serializer(articleContent),
|
|
1978
|
+
textContent,
|
|
1979
|
+
length: textContent.length,
|
|
1980
|
+
excerpt: metadata.excerpt,
|
|
1981
|
+
siteName: metadata.siteName || this._articleSiteName,
|
|
1982
|
+
publishedTime: metadata.publishedTime
|
|
1983
|
+
};
|
|
1984
|
+
}
|
|
1985
|
+
};
|
|
1986
|
+
{
|
|
1987
|
+
module.exports = Readability2;
|
|
1988
|
+
}
|
|
1989
|
+
})(Readability);
|
|
1990
|
+
return Readability.exports;
|
|
1991
|
+
}
|
|
1992
|
+
var ReadabilityReaderable = { exports: {} };
|
|
1993
|
+
var hasRequiredReadabilityReaderable;
|
|
1994
|
+
function requireReadabilityReaderable() {
|
|
1995
|
+
if (hasRequiredReadabilityReaderable) return ReadabilityReaderable.exports;
|
|
1996
|
+
hasRequiredReadabilityReaderable = 1;
|
|
1997
|
+
(function(module) {
|
|
1998
|
+
var REGEXPS = {
|
|
1999
|
+
// NOTE: These two regular expressions are duplicated in
|
|
2000
|
+
// Readability.js. Please keep both copies in sync.
|
|
2001
|
+
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
|
|
2002
|
+
okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i
|
|
2003
|
+
};
|
|
2004
|
+
function isNodeVisible(node) {
|
|
2005
|
+
return (!node.style || node.style.display != "none") && !node.hasAttribute("hidden") && //check for "fallback-image" so that wikimedia math images are displayed
|
|
2006
|
+
(!node.hasAttribute("aria-hidden") || node.getAttribute("aria-hidden") != "true" || node.className && node.className.includes && node.className.includes("fallback-image"));
|
|
2007
|
+
}
|
|
2008
|
+
function isProbablyReaderable(doc, options = {}) {
|
|
2009
|
+
if (typeof options == "function") {
|
|
2010
|
+
options = { visibilityChecker: options };
|
|
2011
|
+
}
|
|
2012
|
+
var defaultOptions = {
|
|
2013
|
+
minScore: 20,
|
|
2014
|
+
minContentLength: 140,
|
|
2015
|
+
visibilityChecker: isNodeVisible
|
|
2016
|
+
};
|
|
2017
|
+
options = Object.assign(defaultOptions, options);
|
|
2018
|
+
var nodes = doc.querySelectorAll("p, pre, article");
|
|
2019
|
+
var brNodes = doc.querySelectorAll("div > br");
|
|
2020
|
+
if (brNodes.length) {
|
|
2021
|
+
var set = new Set(nodes);
|
|
2022
|
+
[].forEach.call(brNodes, function(node) {
|
|
2023
|
+
set.add(node.parentNode);
|
|
2024
|
+
});
|
|
2025
|
+
nodes = Array.from(set);
|
|
2026
|
+
}
|
|
2027
|
+
var score = 0;
|
|
2028
|
+
return [].some.call(nodes, function(node) {
|
|
2029
|
+
if (!options.visibilityChecker(node)) {
|
|
2030
|
+
return false;
|
|
2031
|
+
}
|
|
2032
|
+
var matchString = node.className + " " + node.id;
|
|
2033
|
+
if (REGEXPS.unlikelyCandidates.test(matchString) && !REGEXPS.okMaybeItsACandidate.test(matchString)) {
|
|
2034
|
+
return false;
|
|
2035
|
+
}
|
|
2036
|
+
if (node.matches("li p")) {
|
|
2037
|
+
return false;
|
|
2038
|
+
}
|
|
2039
|
+
var textContentLength = node.textContent.trim().length;
|
|
2040
|
+
if (textContentLength < options.minContentLength) {
|
|
2041
|
+
return false;
|
|
2042
|
+
}
|
|
2043
|
+
score += Math.sqrt(textContentLength - options.minContentLength);
|
|
2044
|
+
if (score > options.minScore) {
|
|
2045
|
+
return true;
|
|
2046
|
+
}
|
|
2047
|
+
return false;
|
|
2048
|
+
});
|
|
2049
|
+
}
|
|
2050
|
+
{
|
|
2051
|
+
module.exports = isProbablyReaderable;
|
|
2052
|
+
}
|
|
2053
|
+
})(ReadabilityReaderable);
|
|
2054
|
+
return ReadabilityReaderable.exports;
|
|
2055
|
+
}
|
|
2056
|
+
var readability;
|
|
2057
|
+
var hasRequiredReadability;
|
|
2058
|
+
function requireReadability() {
|
|
2059
|
+
if (hasRequiredReadability) return readability;
|
|
2060
|
+
hasRequiredReadability = 1;
|
|
2061
|
+
var Readability2 = requireReadability$1();
|
|
2062
|
+
var isProbablyReaderable = requireReadabilityReaderable();
|
|
2063
|
+
readability = {
|
|
2064
|
+
Readability: Readability2,
|
|
2065
|
+
isProbablyReaderable
|
|
2066
|
+
};
|
|
2067
|
+
return readability;
|
|
2068
|
+
}
|
|
2069
|
+
var readabilityExports = requireReadability();
|
|
2070
|
+
function escapeSelectorValue(value) {
|
|
2071
|
+
if (typeof CSS !== "undefined" && typeof CSS.escape === "function") {
|
|
2072
|
+
return CSS.escape(value);
|
|
2073
|
+
}
|
|
2074
|
+
return value.replace(/["\\]/g, "\\$&");
|
|
2075
|
+
}
|
|
2076
|
+
function uniqueSelector(document2, candidate) {
|
|
2077
|
+
if (!candidate) return null;
|
|
2078
|
+
try {
|
|
2079
|
+
return document2.querySelectorAll(candidate).length === 1 ? candidate : null;
|
|
2080
|
+
} catch {
|
|
2081
|
+
return null;
|
|
2082
|
+
}
|
|
2083
|
+
}
|
|
2084
|
+
function uniqueAttributeSelector(el, attribute) {
|
|
2085
|
+
const value = el.getAttribute(attribute)?.trim();
|
|
2086
|
+
if (!value) return null;
|
|
2087
|
+
const candidate = `${el.tagName.toLowerCase()}[${attribute}="${escapeSelectorValue(value)}"]`;
|
|
2088
|
+
return uniqueSelector(el.ownerDocument, candidate);
|
|
2089
|
+
}
|
|
2090
|
+
function generateStableSelector(el) {
|
|
2091
|
+
const document2 = el.ownerDocument;
|
|
2092
|
+
if (el.id) {
|
|
2093
|
+
return `#${escapeSelectorValue(el.id)}`;
|
|
2094
|
+
}
|
|
2095
|
+
for (const attribute of ["data-testid", "name", "form", "aria-label"]) {
|
|
2096
|
+
const candidate = uniqueAttributeSelector(el, attribute);
|
|
2097
|
+
if (candidate) return candidate;
|
|
2098
|
+
}
|
|
2099
|
+
const parts = [];
|
|
2100
|
+
let current = el;
|
|
2101
|
+
while (current) {
|
|
2102
|
+
if (current.id) {
|
|
2103
|
+
parts.unshift(`#${escapeSelectorValue(current.id)}`);
|
|
2104
|
+
break;
|
|
2105
|
+
}
|
|
2106
|
+
const tag = current.tagName.toLowerCase();
|
|
2107
|
+
const parent = current.parentElement;
|
|
2108
|
+
if (!parent) {
|
|
2109
|
+
parts.unshift(tag);
|
|
2110
|
+
break;
|
|
2111
|
+
}
|
|
2112
|
+
const siblings = Array.from(parent.children).filter(
|
|
2113
|
+
(child) => child.tagName === current.tagName
|
|
2114
|
+
);
|
|
2115
|
+
const index = siblings.indexOf(current) + 1;
|
|
2116
|
+
parts.unshift(
|
|
2117
|
+
siblings.length > 1 ? `${tag}:nth-of-type(${index})` : tag
|
|
2118
|
+
);
|
|
2119
|
+
current = parent;
|
|
2120
|
+
if (uniqueSelector(document2, parts.join(" > "))) {
|
|
2121
|
+
break;
|
|
2122
|
+
}
|
|
2123
|
+
}
|
|
2124
|
+
return uniqueSelector(document2, parts.join(" > ")) || parts.join(" > ");
|
|
2125
|
+
}
|
|
2126
|
+
let elementIndex = 0;
|
|
2127
|
+
const elementSelectors = {};
|
|
2128
|
+
let indexedElements = /* @__PURE__ */ new WeakMap();
|
|
2129
|
+
let activeOverlays = [];
|
|
2130
|
+
function generateSelector(el) {
|
|
2131
|
+
return generateStableSelector(el);
|
|
2132
|
+
}
|
|
2133
|
+
function assignIndex(el) {
|
|
2134
|
+
const existing = indexedElements.get(el);
|
|
2135
|
+
if (existing != null) return existing;
|
|
2136
|
+
elementIndex += 1;
|
|
2137
|
+
elementSelectors[elementIndex] = generateSelector(el);
|
|
2138
|
+
indexedElements.set(el, elementIndex);
|
|
2139
|
+
return elementIndex;
|
|
2140
|
+
}
|
|
2141
|
+
function getNodeTextByIds(ids) {
|
|
2142
|
+
if (!ids) return void 0;
|
|
2143
|
+
const text = ids.split(/\s+/).map((id) => document.getElementById(id)?.textContent?.trim() || "").filter(Boolean).join(" ").trim();
|
|
2144
|
+
return text || void 0;
|
|
2145
|
+
}
|
|
2146
|
+
function getTrimmedText(value) {
|
|
2147
|
+
const text = value?.trim();
|
|
2148
|
+
return text || void 0;
|
|
2149
|
+
}
|
|
2150
|
+
function pushPropertyValue(target, key, value) {
|
|
2151
|
+
if (!key || value == null) return;
|
|
2152
|
+
const existing = target[key];
|
|
2153
|
+
if (existing === void 0) {
|
|
2154
|
+
target[key] = value;
|
|
2155
|
+
return;
|
|
2156
|
+
}
|
|
2157
|
+
if (Array.isArray(existing)) {
|
|
2158
|
+
existing.push(value);
|
|
2159
|
+
return;
|
|
2160
|
+
}
|
|
2161
|
+
target[key] = [existing, value];
|
|
2162
|
+
}
|
|
2163
|
+
function getStructuredElementValue(el) {
|
|
2164
|
+
if (el instanceof HTMLMetaElement) {
|
|
2165
|
+
return getTrimmedText(el.content);
|
|
2166
|
+
}
|
|
2167
|
+
if (el instanceof HTMLAnchorElement || el instanceof HTMLAreaElement || el instanceof HTMLLinkElement) {
|
|
2168
|
+
return getTrimmedText(el.href);
|
|
2169
|
+
}
|
|
2170
|
+
if (el instanceof HTMLImageElement || el instanceof HTMLAudioElement || el instanceof HTMLVideoElement || el instanceof HTMLSourceElement || el instanceof HTMLTrackElement || el instanceof HTMLIFrameElement || el instanceof HTMLEmbedElement) {
|
|
2171
|
+
return getTrimmedText(el.src);
|
|
2172
|
+
}
|
|
2173
|
+
if (el instanceof HTMLObjectElement) {
|
|
2174
|
+
return getTrimmedText(el.data);
|
|
2175
|
+
}
|
|
2176
|
+
if (el instanceof HTMLDataElement || el instanceof HTMLMeterElement) {
|
|
2177
|
+
return getTrimmedText(el.value);
|
|
2178
|
+
}
|
|
2179
|
+
if (el instanceof HTMLTimeElement) {
|
|
2180
|
+
return getTrimmedText(el.dateTime) || getTrimmedText(el.textContent);
|
|
2181
|
+
}
|
|
2182
|
+
if (el instanceof HTMLInputElement || el instanceof HTMLSelectElement || el instanceof HTMLTextAreaElement) {
|
|
2183
|
+
return getTrimmedText(el.value);
|
|
2184
|
+
}
|
|
2185
|
+
const contentAttr = getTrimmedText(el.getAttribute("content"));
|
|
2186
|
+
if (contentAttr) return contentAttr;
|
|
2187
|
+
const resourceAttr = getTrimmedText(el.getAttribute("resource")) || getTrimmedText(el.getAttribute("href")) || getTrimmedText(el.getAttribute("src")) || getTrimmedText(el.getAttribute("datetime")) || getTrimmedText(el.getAttribute("data"));
|
|
2188
|
+
if (resourceAttr) return resourceAttr;
|
|
2189
|
+
return getTrimmedText(el.textContent);
|
|
2190
|
+
}
|
|
2191
|
+
function isElementVisible(el) {
|
|
2192
|
+
if (!(el instanceof HTMLElement)) return true;
|
|
2193
|
+
const style = window.getComputedStyle(el);
|
|
2194
|
+
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") {
|
|
2195
|
+
return false;
|
|
2196
|
+
}
|
|
2197
|
+
if (el.hasAttribute("hidden") || el.getAttribute("aria-hidden") === "true") {
|
|
2198
|
+
return false;
|
|
2199
|
+
}
|
|
2200
|
+
const rect = el.getBoundingClientRect();
|
|
2201
|
+
return rect.width > 0 && rect.height > 0;
|
|
2202
|
+
}
|
|
2203
|
+
function isInViewportRect(rect) {
|
|
2204
|
+
const viewportWidth = window.innerWidth || document.documentElement?.clientWidth || 0;
|
|
2205
|
+
const viewportHeight = window.innerHeight || document.documentElement?.clientHeight || 0;
|
|
2206
|
+
return rect.width > 0 && rect.height > 0 && rect.bottom > 0 && rect.right > 0 && rect.top < viewportHeight && rect.left < viewportWidth;
|
|
2207
|
+
}
|
|
2208
|
+
function isFullyInViewportRect(rect) {
|
|
2209
|
+
const viewportWidth = window.innerWidth || document.documentElement?.clientWidth || 0;
|
|
2210
|
+
const viewportHeight = window.innerHeight || document.documentElement?.clientHeight || 0;
|
|
2211
|
+
return rect.width > 0 && rect.height > 0 && rect.top >= 0 && rect.left >= 0 && rect.bottom <= viewportHeight && rect.right <= viewportWidth;
|
|
2212
|
+
}
|
|
2213
|
+
function parseZIndex(style) {
|
|
2214
|
+
const value = Number.parseInt(style.zIndex, 10);
|
|
2215
|
+
return Number.isFinite(value) ? value : 0;
|
|
2216
|
+
}
|
|
2217
|
+
function getViewportCenterCoverage(rect) {
|
|
2218
|
+
const viewportWidth = window.innerWidth || document.documentElement?.clientWidth || 0;
|
|
2219
|
+
const viewportHeight = window.innerHeight || document.documentElement?.clientHeight || 0;
|
|
2220
|
+
const centerX = viewportWidth / 2;
|
|
2221
|
+
const centerY = viewportHeight / 2;
|
|
2222
|
+
return rect.left <= centerX && rect.right >= centerX && rect.top <= centerY && rect.bottom >= centerY;
|
|
2223
|
+
}
|
|
2224
|
+
function getOverlayLabel(el) {
|
|
2225
|
+
return getTrimmedText(el.getAttribute("aria-label")) || getNodeTextByIds(el.getAttribute("aria-labelledby")) || getTrimmedText(el.id) || void 0;
|
|
2226
|
+
}
|
|
2227
|
+
function getOverlayType(el) {
|
|
2228
|
+
const tag = el.tagName.toLowerCase();
|
|
2229
|
+
const role = el.getAttribute("role");
|
|
2230
|
+
if (tag === "dialog" || role === "dialog" || role === "alertdialog") {
|
|
2231
|
+
return "dialog";
|
|
2232
|
+
}
|
|
2233
|
+
if (el.getAttribute("aria-modal") === "true") {
|
|
2234
|
+
return "modal";
|
|
2235
|
+
}
|
|
2236
|
+
return "overlay";
|
|
2237
|
+
}
|
|
2238
|
+
function detectOverlays() {
|
|
2239
|
+
if (!document.body) return [];
|
|
2240
|
+
const viewportWidth = window.innerWidth || document.documentElement?.clientWidth || 0;
|
|
2241
|
+
const viewportHeight = window.innerHeight || document.documentElement?.clientHeight || 0;
|
|
2242
|
+
const viewportArea = Math.max(1, viewportWidth * viewportHeight);
|
|
2243
|
+
const overlays = [];
|
|
2244
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2245
|
+
Array.from(document.body.querySelectorAll("*")).forEach((node) => {
|
|
2246
|
+
if (!(node instanceof HTMLElement) || seen.has(node)) return;
|
|
2247
|
+
if (!isElementVisible(node)) return;
|
|
2248
|
+
const style = window.getComputedStyle(node);
|
|
2249
|
+
if (style.pointerEvents === "none") return;
|
|
2250
|
+
const rect = node.getBoundingClientRect();
|
|
2251
|
+
if (!isInViewportRect(rect)) return;
|
|
2252
|
+
const position = style.position;
|
|
2253
|
+
const zIndex = parseZIndex(style);
|
|
2254
|
+
const areaRatio = rect.width * rect.height / viewportArea;
|
|
2255
|
+
const overlayType = getOverlayType(node);
|
|
2256
|
+
const dialogLike = overlayType === "dialog" || overlayType === "modal";
|
|
2257
|
+
const blockingSurface = (position === "fixed" || position === "sticky") && zIndex >= 10 && areaRatio >= 0.3 && getViewportCenterCoverage(rect);
|
|
2258
|
+
if (!dialogLike && !blockingSurface) return;
|
|
2259
|
+
seen.add(node);
|
|
2260
|
+
overlays.push({
|
|
2261
|
+
element: node,
|
|
2262
|
+
type: overlayType ?? "overlay",
|
|
2263
|
+
role: getTrimmedText(node.getAttribute("role")) || void 0,
|
|
2264
|
+
label: getOverlayLabel(node),
|
|
2265
|
+
selector: generateSelector(node),
|
|
2266
|
+
text: getTrimmedText(node.textContent)?.slice(0, 160),
|
|
2267
|
+
blocksInteraction: dialogLike || blockingSurface,
|
|
2268
|
+
zIndex
|
|
2269
|
+
});
|
|
2270
|
+
});
|
|
2271
|
+
return overlays.sort((a, b) => {
|
|
2272
|
+
if ((a.blocksInteraction ? 1 : 0) !== (b.blocksInteraction ? 1 : 0)) {
|
|
2273
|
+
return (b.blocksInteraction ? 1 : 0) - (a.blocksInteraction ? 1 : 0);
|
|
2274
|
+
}
|
|
2275
|
+
return b.zIndex - a.zIndex;
|
|
2276
|
+
});
|
|
2277
|
+
}
|
|
2278
|
+
function isLikelyDormantOverlay(el) {
|
|
2279
|
+
const tag = el.tagName.toLowerCase();
|
|
2280
|
+
const role = getTrimmedText(el.getAttribute("role")) || "";
|
|
2281
|
+
const attrs = [
|
|
2282
|
+
el.id,
|
|
2283
|
+
el.className,
|
|
2284
|
+
el.getAttribute("data-testid"),
|
|
2285
|
+
el.getAttribute("data-test"),
|
|
2286
|
+
el.getAttribute("aria-label"),
|
|
2287
|
+
el.getAttribute("title"),
|
|
2288
|
+
el.getAttribute("data-module-name")
|
|
2289
|
+
].filter(Boolean).join(" ").toLowerCase();
|
|
2290
|
+
const text = getTrimmedText(el.textContent)?.toLowerCase() || "";
|
|
2291
|
+
if (tag === "dialog" || role === "dialog" || role === "alertdialog" || el.getAttribute("aria-modal") === "true") {
|
|
2292
|
+
return true;
|
|
2293
|
+
}
|
|
2294
|
+
return /cookie|consent|privacy|gdpr|ccpa|onetrust|ot-sdk|trustarc|didomi|sp_message|qc-cmp|cmp|newsletter|subscribe/.test(
|
|
2295
|
+
`${attrs} ${text.slice(0, 200)}`
|
|
2296
|
+
);
|
|
2297
|
+
}
|
|
2298
|
+
function detectDormantOverlays() {
|
|
2299
|
+
if (!document.body) return [];
|
|
2300
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2301
|
+
const matches = [];
|
|
2302
|
+
Array.from(document.body.querySelectorAll("*")).forEach((node) => {
|
|
2303
|
+
if (!(node instanceof HTMLElement)) return;
|
|
2304
|
+
if (isElementVisible(node)) return;
|
|
2305
|
+
if (!isLikelyDormantOverlay(node)) return;
|
|
2306
|
+
const selector = generateSelector(node);
|
|
2307
|
+
if (!selector || seen.has(selector)) return;
|
|
2308
|
+
seen.add(selector);
|
|
2309
|
+
matches.push({
|
|
2310
|
+
type: getOverlayType(node) ?? "overlay",
|
|
2311
|
+
role: getTrimmedText(node.getAttribute("role")) || void 0,
|
|
2312
|
+
label: getOverlayLabel(node),
|
|
2313
|
+
selector,
|
|
2314
|
+
text: getTrimmedText(node.textContent)?.slice(0, 160)
|
|
2315
|
+
});
|
|
2316
|
+
});
|
|
2317
|
+
return matches.slice(0, 10);
|
|
2318
|
+
}
|
|
2319
|
+
function samplePointForRect(rect) {
|
|
2320
|
+
if (!isInViewportRect(rect)) return null;
|
|
2321
|
+
const viewportWidth = window.innerWidth || document.documentElement?.clientWidth || 0;
|
|
2322
|
+
const viewportHeight = window.innerHeight || document.documentElement?.clientHeight || 0;
|
|
2323
|
+
const maxX = Math.max(0, viewportWidth - 1);
|
|
2324
|
+
const maxY = Math.max(0, viewportHeight - 1);
|
|
2325
|
+
return {
|
|
2326
|
+
x: Math.min(maxX, Math.max(0, rect.left + rect.width / 2)),
|
|
2327
|
+
y: Math.min(maxY, Math.max(0, rect.top + rect.height / 2))
|
|
2328
|
+
};
|
|
2329
|
+
}
|
|
2330
|
+
function getVisibilityState(el) {
|
|
2331
|
+
if (!(el instanceof HTMLElement)) {
|
|
2332
|
+
return {
|
|
2333
|
+
visible: true,
|
|
2334
|
+
inViewport: true,
|
|
2335
|
+
fullyInViewport: true,
|
|
2336
|
+
obscured: false,
|
|
2337
|
+
blockedByOverlay: false
|
|
2338
|
+
};
|
|
2339
|
+
}
|
|
2340
|
+
const rect = el.getBoundingClientRect();
|
|
2341
|
+
const visible = isElementVisible(el);
|
|
2342
|
+
const inViewport = visible && isInViewportRect(rect);
|
|
2343
|
+
const fullyInViewport = visible && isFullyInViewportRect(rect);
|
|
2344
|
+
let obscured = false;
|
|
2345
|
+
let blockedByOverlay = false;
|
|
2346
|
+
if (inViewport) {
|
|
2347
|
+
const point = samplePointForRect(rect);
|
|
2348
|
+
if (point) {
|
|
2349
|
+
const topElement = document.elementFromPoint(point.x, point.y);
|
|
2350
|
+
if (topElement && topElement !== el && !el.contains(topElement) && !(topElement instanceof HTMLElement && topElement.contains(el))) {
|
|
2351
|
+
obscured = true;
|
|
2352
|
+
blockedByOverlay = activeOverlays.some(
|
|
2353
|
+
(overlay) => overlay.blocksInteraction && overlay.element.contains(topElement) && !overlay.element.contains(el)
|
|
2354
|
+
);
|
|
2355
|
+
}
|
|
2356
|
+
}
|
|
2357
|
+
}
|
|
2358
|
+
return {
|
|
2359
|
+
visible,
|
|
2360
|
+
inViewport,
|
|
2361
|
+
fullyInViewport,
|
|
2362
|
+
obscured,
|
|
2363
|
+
blockedByOverlay
|
|
2364
|
+
};
|
|
2365
|
+
}
|
|
2366
|
+
function getViewportSnapshot() {
|
|
2367
|
+
const scrollingElement = document.scrollingElement || document.documentElement || document.body;
|
|
2368
|
+
const scrollXCandidates = [
|
|
2369
|
+
window.scrollX,
|
|
2370
|
+
window.pageXOffset,
|
|
2371
|
+
window.visualViewport?.pageLeft,
|
|
2372
|
+
scrollingElement?.scrollLeft,
|
|
2373
|
+
document.documentElement?.scrollLeft,
|
|
2374
|
+
document.body?.scrollLeft
|
|
2375
|
+
].filter((value) => typeof value === "number");
|
|
2376
|
+
const scrollYCandidates = [
|
|
2377
|
+
window.scrollY,
|
|
2378
|
+
window.pageYOffset,
|
|
2379
|
+
window.visualViewport?.pageTop,
|
|
2380
|
+
scrollingElement?.scrollTop,
|
|
2381
|
+
document.documentElement?.scrollTop,
|
|
2382
|
+
document.body?.scrollTop
|
|
2383
|
+
].filter((value) => typeof value === "number");
|
|
2384
|
+
return {
|
|
2385
|
+
width: window.innerWidth || document.documentElement?.clientWidth || 0,
|
|
2386
|
+
height: window.innerHeight || document.documentElement?.clientHeight || 0,
|
|
2387
|
+
scrollX: Math.max(0, ...scrollXCandidates),
|
|
2388
|
+
scrollY: Math.max(0, ...scrollYCandidates)
|
|
2389
|
+
};
|
|
2390
|
+
}
|
|
2391
|
+
function isElementDisabled(el) {
|
|
2392
|
+
return el.hasAttribute("disabled") || el.getAttribute("aria-disabled") === "true";
|
|
2393
|
+
}
|
|
2394
|
+
function getElementContext(el) {
|
|
2395
|
+
let parent = el.parentElement;
|
|
2396
|
+
while (parent) {
|
|
2397
|
+
const tag = parent.tagName.toLowerCase();
|
|
2398
|
+
const role = parent.getAttribute("role");
|
|
2399
|
+
if (tag === "nav" || role === "navigation") return "nav";
|
|
2400
|
+
if (tag === "header" || role === "banner") return "header";
|
|
2401
|
+
if (tag === "main" || role === "main") return "main";
|
|
2402
|
+
if (tag === "footer" || role === "contentinfo") return "footer";
|
|
2403
|
+
if (tag === "aside" || role === "complementary") return "sidebar";
|
|
2404
|
+
if (tag === "article" || role === "article") return "article";
|
|
2405
|
+
if (tag === "dialog" || role === "dialog" || role === "alertdialog") {
|
|
2406
|
+
return "dialog";
|
|
2407
|
+
}
|
|
2408
|
+
if (tag === "form") return `form${parent.id ? `#${parent.id}` : ""}`;
|
|
2409
|
+
parent = parent.parentElement;
|
|
2410
|
+
}
|
|
2411
|
+
return "content";
|
|
2412
|
+
}
|
|
2413
|
+
function getInputLabel(el) {
|
|
2414
|
+
if (el.id) {
|
|
2415
|
+
const label = document.querySelector(
|
|
2416
|
+
`label[for="${escapeSelectorValue(el.id)}"]`
|
|
2417
|
+
);
|
|
2418
|
+
if (label) return getTrimmedText(label.textContent);
|
|
2419
|
+
}
|
|
2420
|
+
const parentLabel = el.closest("label");
|
|
2421
|
+
if (parentLabel) {
|
|
2422
|
+
const clone = parentLabel.cloneNode(true);
|
|
2423
|
+
clone.querySelectorAll("input, select, textarea").forEach((input) => {
|
|
2424
|
+
input.remove();
|
|
2425
|
+
});
|
|
2426
|
+
const text = getTrimmedText(clone.textContent);
|
|
2427
|
+
if (text) return text;
|
|
2428
|
+
}
|
|
2429
|
+
return getTrimmedText(el.getAttribute("aria-label")) || getNodeTextByIds(el.getAttribute("aria-labelledby")) || getTrimmedText(el.getAttribute("placeholder")) || void 0;
|
|
2430
|
+
}
|
|
2431
|
+
function getElementRole(el) {
|
|
2432
|
+
return getTrimmedText(el.getAttribute("role")) || (el.tagName.toLowerCase() === "a" ? "link" : el.tagName.toLowerCase() === "button" ? "button" : void 0);
|
|
2433
|
+
}
|
|
2434
|
+
function getElementDescription(el) {
|
|
2435
|
+
return getTrimmedText(el.getAttribute("aria-description")) || getNodeTextByIds(el.getAttribute("aria-describedby")) || getTrimmedText(el.getAttribute("title")) || void 0;
|
|
2436
|
+
}
|
|
2437
|
+
function getElementValue(el) {
|
|
2438
|
+
if (el instanceof HTMLSelectElement) {
|
|
2439
|
+
return getTrimmedText(el.value);
|
|
2440
|
+
}
|
|
2441
|
+
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
|
2442
|
+
if (el.type === "password") return void 0;
|
|
2443
|
+
if (el.type === "checkbox" || el.type === "radio") {
|
|
2444
|
+
return void 0;
|
|
2445
|
+
}
|
|
2446
|
+
return getTrimmedText(el.value);
|
|
2447
|
+
}
|
|
2448
|
+
return void 0;
|
|
2449
|
+
}
|
|
2450
|
+
function getSelectOptions(el) {
|
|
2451
|
+
const options = Array.from(el.options).map((option) => ({
|
|
2452
|
+
label: option.textContent?.trim() || option.value.trim(),
|
|
2453
|
+
value: option.value
|
|
2454
|
+
})).filter((o) => o.label || o.value).slice(0, 25);
|
|
2455
|
+
return options.length > 0 ? options : void 0;
|
|
2456
|
+
}
|
|
2457
|
+
function getAriaBoolean(el, attr) {
|
|
2458
|
+
const val = el.getAttribute(attr);
|
|
2459
|
+
if (val === "true") return true;
|
|
2460
|
+
if (val === "false") return false;
|
|
2461
|
+
return void 0;
|
|
2462
|
+
}
|
|
2463
|
+
function buildBaseMetadata(el) {
|
|
2464
|
+
return {
|
|
2465
|
+
context: getElementContext(el),
|
|
2466
|
+
selector: generateSelector(el),
|
|
2467
|
+
index: assignIndex(el),
|
|
2468
|
+
role: getElementRole(el),
|
|
2469
|
+
description: getElementDescription(el),
|
|
2470
|
+
...getVisibilityState(el),
|
|
2471
|
+
disabled: isElementDisabled(el),
|
|
2472
|
+
ariaExpanded: getAriaBoolean(el, "aria-expanded"),
|
|
2473
|
+
ariaPressed: getAriaBoolean(el, "aria-pressed"),
|
|
2474
|
+
ariaSelected: getAriaBoolean(el, "aria-selected")
|
|
2475
|
+
};
|
|
2476
|
+
}
|
|
2477
|
+
function extractHeadings() {
|
|
2478
|
+
return Array.from(document.querySelectorAll("h1, h2, h3, h4, h5, h6")).map((el) => {
|
|
2479
|
+
const text = el.textContent?.trim() || "";
|
|
2480
|
+
if (!text) return null;
|
|
2481
|
+
return {
|
|
2482
|
+
level: Number.parseInt(el.tagName[1], 10),
|
|
2483
|
+
text
|
|
2484
|
+
};
|
|
2485
|
+
}).filter((value) => Boolean(value));
|
|
2486
|
+
}
|
|
2487
|
+
function extractNavigation() {
|
|
2488
|
+
const navigation = [];
|
|
2489
|
+
document.querySelectorAll(
|
|
2490
|
+
'nav, [role="navigation"], header nav, [role="banner"] nav'
|
|
2491
|
+
).forEach((nav) => {
|
|
2492
|
+
nav.querySelectorAll("a[href]").forEach((link) => {
|
|
2493
|
+
const anchor = link;
|
|
2494
|
+
const text = anchor.textContent?.trim();
|
|
2495
|
+
if (!text || anchor.getAttribute("href")?.startsWith("#")) return;
|
|
2496
|
+
navigation.push({
|
|
2497
|
+
type: "link",
|
|
2498
|
+
text: text.slice(0, 100),
|
|
2499
|
+
href: anchor.href.slice(0, 500),
|
|
2500
|
+
...buildBaseMetadata(anchor),
|
|
2501
|
+
context: "nav"
|
|
2502
|
+
});
|
|
2503
|
+
});
|
|
2504
|
+
});
|
|
2505
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2506
|
+
return navigation.filter((item) => {
|
|
2507
|
+
if (!item.href || seen.has(item.href)) return false;
|
|
2508
|
+
seen.add(item.href);
|
|
2509
|
+
return true;
|
|
2510
|
+
});
|
|
2511
|
+
}
|
|
2512
|
+
function getFieldMetadata(el) {
|
|
2513
|
+
const meta = {};
|
|
2514
|
+
const name = el.name;
|
|
2515
|
+
if (name) meta.name = name;
|
|
2516
|
+
const autocomplete = el.getAttribute("autocomplete");
|
|
2517
|
+
if (autocomplete) meta.autocomplete = autocomplete;
|
|
2518
|
+
if (el instanceof HTMLInputElement && (el.type === "checkbox" || el.type === "radio")) {
|
|
2519
|
+
meta.checked = el.checked;
|
|
2520
|
+
}
|
|
2521
|
+
if (el instanceof HTMLInputElement) {
|
|
2522
|
+
if (el.maxLength >= 0) meta.maxLength = el.maxLength;
|
|
2523
|
+
const min = el.getAttribute("min");
|
|
2524
|
+
if (min) meta.min = min;
|
|
2525
|
+
const max = el.getAttribute("max");
|
|
2526
|
+
if (max) meta.max = max;
|
|
2527
|
+
const pattern = el.getAttribute("pattern");
|
|
2528
|
+
if (pattern) meta.pattern = pattern;
|
|
2529
|
+
}
|
|
2530
|
+
if (el instanceof HTMLTextAreaElement) {
|
|
2531
|
+
if (el.maxLength >= 0) meta.maxLength = el.maxLength;
|
|
2532
|
+
}
|
|
2533
|
+
return meta;
|
|
2534
|
+
}
|
|
2535
|
+
function extractInteractiveElements() {
|
|
2536
|
+
const elements = [];
|
|
2537
|
+
document.querySelectorAll(
|
|
2538
|
+
'button, [role="button"], input[type="submit"], input[type="button"]'
|
|
2539
|
+
).forEach((btn) => {
|
|
2540
|
+
const input = btn;
|
|
2541
|
+
const text = btn.textContent?.trim() || input.value || btn.getAttribute("aria-label") || "Button";
|
|
2542
|
+
elements.push({
|
|
2543
|
+
type: "button",
|
|
2544
|
+
text: text.slice(0, 100),
|
|
2545
|
+
...buildBaseMetadata(btn)
|
|
2546
|
+
});
|
|
2547
|
+
});
|
|
2548
|
+
document.querySelectorAll("a[href]").forEach((link) => {
|
|
2549
|
+
const anchor = link;
|
|
2550
|
+
const text = anchor.textContent?.trim();
|
|
2551
|
+
if (!text || anchor.getAttribute("href")?.startsWith("#")) return;
|
|
2552
|
+
const context = getElementContext(anchor);
|
|
2553
|
+
if (context === "nav") return;
|
|
2554
|
+
elements.push({
|
|
2555
|
+
type: "link",
|
|
2556
|
+
text: text.slice(0, 100),
|
|
2557
|
+
href: anchor.href.slice(0, 500),
|
|
2558
|
+
...buildBaseMetadata(anchor),
|
|
2559
|
+
context
|
|
2560
|
+
});
|
|
2561
|
+
});
|
|
2562
|
+
document.querySelectorAll(
|
|
2563
|
+
'input:not([type="hidden"]):not([type="submit"]):not([type="button"]), select, textarea'
|
|
2564
|
+
).forEach((input) => {
|
|
2565
|
+
const element = input;
|
|
2566
|
+
const tag = input.tagName.toLowerCase();
|
|
2567
|
+
elements.push({
|
|
2568
|
+
type: tag === "select" ? "select" : tag === "textarea" ? "textarea" : "input",
|
|
2569
|
+
label: getInputLabel(element)?.slice(0, 100),
|
|
2570
|
+
inputType: element.getAttribute("type") || void 0,
|
|
2571
|
+
placeholder: element.getAttribute("placeholder") || void 0,
|
|
2572
|
+
required: element.hasAttribute("required") || void 0,
|
|
2573
|
+
value: getElementValue(element),
|
|
2574
|
+
options: element instanceof HTMLSelectElement ? getSelectOptions(element) : void 0,
|
|
2575
|
+
...buildBaseMetadata(input),
|
|
2576
|
+
...getFieldMetadata(element)
|
|
2577
|
+
});
|
|
2578
|
+
});
|
|
2579
|
+
return elements;
|
|
2580
|
+
}
|
|
2581
|
+
function extractForms() {
|
|
2582
|
+
const forms = [];
|
|
2583
|
+
function isSubmitControlForForm(el, form) {
|
|
2584
|
+
if (el instanceof HTMLButtonElement) {
|
|
2585
|
+
const type = getTrimmedText(el.getAttribute("type"))?.toLowerCase();
|
|
2586
|
+
return (!type || type === "submit") && el.form === form;
|
|
2587
|
+
}
|
|
2588
|
+
return el instanceof HTMLInputElement && (el.type === "submit" || el.type === "image") && el.form === form;
|
|
2589
|
+
}
|
|
2590
|
+
document.querySelectorAll("form").forEach((form) => {
|
|
2591
|
+
const fields = [];
|
|
2592
|
+
form.querySelectorAll(
|
|
2593
|
+
"input:not([type='hidden']):not([type='submit']):not([type='button']):not([type='image']), select, textarea"
|
|
2594
|
+
).forEach((input) => {
|
|
2595
|
+
const element = input;
|
|
2596
|
+
const tag = input.tagName.toLowerCase();
|
|
2597
|
+
fields.push({
|
|
2598
|
+
type: tag === "select" ? "select" : tag === "textarea" ? "textarea" : "input",
|
|
2599
|
+
label: getInputLabel(element)?.slice(0, 100),
|
|
2600
|
+
inputType: element.getAttribute("type") || void 0,
|
|
2601
|
+
placeholder: element.getAttribute("placeholder") || void 0,
|
|
2602
|
+
required: element.hasAttribute("required") || void 0,
|
|
2603
|
+
value: getElementValue(element),
|
|
2604
|
+
options: element instanceof HTMLSelectElement ? getSelectOptions(element) : void 0,
|
|
2605
|
+
...buildBaseMetadata(input),
|
|
2606
|
+
...getFieldMetadata(element)
|
|
2607
|
+
});
|
|
2608
|
+
});
|
|
2609
|
+
Array.from(
|
|
2610
|
+
document.querySelectorAll(
|
|
2611
|
+
"button, input[type='submit'], input[type='image']"
|
|
2612
|
+
)
|
|
2613
|
+
).filter((control) => isSubmitControlForForm(control, form)).forEach((btn) => {
|
|
2614
|
+
const input = btn;
|
|
2615
|
+
const text = btn.textContent?.trim() || input.value || btn.getAttribute("aria-label") || "Submit";
|
|
2616
|
+
fields.push({
|
|
2617
|
+
type: "button",
|
|
2618
|
+
text: text.slice(0, 100),
|
|
2619
|
+
...buildBaseMetadata(btn)
|
|
2620
|
+
});
|
|
2621
|
+
});
|
|
2622
|
+
forms.push({
|
|
2623
|
+
id: form.id || void 0,
|
|
2624
|
+
action: form.getAttribute("action") || void 0,
|
|
2625
|
+
method: form.getAttribute("method") || void 0,
|
|
2626
|
+
fields
|
|
2627
|
+
});
|
|
2628
|
+
});
|
|
2629
|
+
return forms;
|
|
2630
|
+
}
|
|
2631
|
+
function extractLandmarks() {
|
|
2632
|
+
const landmarks = [];
|
|
2633
|
+
const selectors = [
|
|
2634
|
+
"header, [role='banner']",
|
|
2635
|
+
"nav, [role='navigation']",
|
|
2636
|
+
"main, [role='main']",
|
|
2637
|
+
"aside, [role='complementary']",
|
|
2638
|
+
"footer, [role='contentinfo']",
|
|
2639
|
+
"article, [role='article']",
|
|
2640
|
+
"section, [role='region']",
|
|
2641
|
+
"[role='search']",
|
|
2642
|
+
"[role='form']",
|
|
2643
|
+
"dialog, [role='dialog'], [role='alertdialog']"
|
|
2644
|
+
];
|
|
2645
|
+
selectors.forEach((selector) => {
|
|
2646
|
+
document.querySelectorAll(selector).forEach((el) => {
|
|
2647
|
+
const tag = el.tagName.toLowerCase();
|
|
2648
|
+
const role = el.getAttribute("role") || (tag === "header" ? "banner" : tag === "nav" ? "navigation" : tag === "main" ? "main" : tag === "aside" ? "complementary" : tag === "footer" ? "contentinfo" : tag === "article" ? "article" : tag === "section" ? "region" : tag === "dialog" ? "dialog" : "generic");
|
|
2649
|
+
landmarks.push({
|
|
2650
|
+
role,
|
|
2651
|
+
label: getTrimmedText(el.getAttribute("aria-label")) || getNodeTextByIds(el.getAttribute("aria-labelledby")) || getTrimmedText(el.id),
|
|
2652
|
+
text: getTrimmedText(el.textContent)?.slice(0, 200)
|
|
2653
|
+
});
|
|
2654
|
+
});
|
|
2655
|
+
});
|
|
2656
|
+
return landmarks;
|
|
2657
|
+
}
|
|
2658
|
+
function extractJsonLd() {
|
|
2659
|
+
const results = [];
|
|
2660
|
+
const scripts = document.querySelectorAll('script[type="application/ld+json"]');
|
|
2661
|
+
for (const script of scripts) {
|
|
2662
|
+
try {
|
|
2663
|
+
const parsed = JSON.parse(script.textContent || "");
|
|
2664
|
+
if (Array.isArray(parsed)) {
|
|
2665
|
+
for (const item of parsed) {
|
|
2666
|
+
if (item && typeof item === "object") results.push(item);
|
|
2667
|
+
}
|
|
2668
|
+
} else if (parsed && typeof parsed === "object") {
|
|
2669
|
+
results.push(parsed);
|
|
2670
|
+
}
|
|
2671
|
+
} catch {
|
|
2672
|
+
}
|
|
2673
|
+
}
|
|
2674
|
+
return results;
|
|
2675
|
+
}
|
|
2676
|
+
function extractMetaTags() {
|
|
2677
|
+
const tags = {};
|
|
2678
|
+
document.querySelectorAll("meta[name], meta[property], meta[itemprop]").forEach((el) => {
|
|
2679
|
+
if (!(el instanceof HTMLMetaElement)) return;
|
|
2680
|
+
const key = getTrimmedText(el.getAttribute("property")) || getTrimmedText(el.getAttribute("name")) || getTrimmedText(el.getAttribute("itemprop"));
|
|
2681
|
+
const value = getTrimmedText(el.content);
|
|
2682
|
+
if (!key || !value || tags[key]) return;
|
|
2683
|
+
if (key === "description" || key === "author" || key.startsWith("og:") || key.startsWith("article:") || key.startsWith("product:") || key.startsWith("recipe:") || key.startsWith("twitter:")) {
|
|
2684
|
+
tags[key] = value;
|
|
2685
|
+
}
|
|
2686
|
+
});
|
|
2687
|
+
const canonical = document.querySelector('link[rel="canonical"]');
|
|
2688
|
+
if (canonical instanceof HTMLLinkElement && canonical.href) {
|
|
2689
|
+
tags.canonical = canonical.href;
|
|
2690
|
+
}
|
|
2691
|
+
return tags;
|
|
2692
|
+
}
|
|
2693
|
+
function extractMicrodata() {
|
|
2694
|
+
const serializeItem = (scope, depth = 0) => {
|
|
2695
|
+
if (depth > 3) return null;
|
|
2696
|
+
const item = {};
|
|
2697
|
+
const itemType = getTrimmedText(scope.getAttribute("itemtype"));
|
|
2698
|
+
const itemId = getTrimmedText(scope.getAttribute("itemid"));
|
|
2699
|
+
if (itemType) {
|
|
2700
|
+
const types = itemType.split(/\s+/).filter(Boolean);
|
|
2701
|
+
item["@type"] = types.length === 1 ? types[0] : types;
|
|
2702
|
+
}
|
|
2703
|
+
if (itemId) item["@id"] = itemId;
|
|
2704
|
+
scope.querySelectorAll("[itemprop]").forEach((node) => {
|
|
2705
|
+
if (!(node instanceof HTMLElement)) return;
|
|
2706
|
+
const nearestScope = node.closest("[itemscope]");
|
|
2707
|
+
const isNestedItemRoot = nearestScope === node && node.hasAttribute("itemscope");
|
|
2708
|
+
if (nearestScope !== scope && !isNestedItemRoot) {
|
|
2709
|
+
return;
|
|
2710
|
+
}
|
|
2711
|
+
if (isNestedItemRoot && node.parentElement?.closest("[itemscope]") !== scope) {
|
|
2712
|
+
return;
|
|
2713
|
+
}
|
|
2714
|
+
const propNames = (node.getAttribute("itemprop") || "").split(/\s+/).map((name) => name.trim()).filter(Boolean);
|
|
2715
|
+
if (propNames.length === 0) return;
|
|
2716
|
+
const value = node.hasAttribute("itemscope") && isNestedItemRoot ? serializeItem(node, depth + 1) : getStructuredElementValue(node);
|
|
2717
|
+
if (value == null) return;
|
|
2718
|
+
propNames.forEach((name) => pushPropertyValue(item, name, value));
|
|
2719
|
+
});
|
|
2720
|
+
return Object.keys(item).length > 0 ? item : null;
|
|
2721
|
+
};
|
|
2722
|
+
return Array.from(document.querySelectorAll("[itemscope]")).filter(
|
|
2723
|
+
(node) => node instanceof HTMLElement && !node.hasAttribute("itemprop")
|
|
2724
|
+
).map((scope) => serializeItem(scope)).filter((item) => item !== null);
|
|
2725
|
+
}
|
|
2726
|
+
function extractRdfa() {
|
|
2727
|
+
const serializeEntity = (scope, depth = 0) => {
|
|
2728
|
+
if (depth > 3) return null;
|
|
2729
|
+
const entity = {};
|
|
2730
|
+
const typeAttr = getTrimmedText(scope.getAttribute("typeof"));
|
|
2731
|
+
const about = getTrimmedText(scope.getAttribute("about")) || getTrimmedText(scope.getAttribute("resource")) || getTrimmedText(scope.getAttribute("href")) || getTrimmedText(scope.getAttribute("src"));
|
|
2732
|
+
if (typeAttr) {
|
|
2733
|
+
const types = typeAttr.split(/\s+/).filter(Boolean);
|
|
2734
|
+
entity["@type"] = types.length === 1 ? types[0] : types;
|
|
2735
|
+
}
|
|
2736
|
+
if (about) entity["@id"] = about;
|
|
2737
|
+
scope.querySelectorAll("[property]").forEach((node) => {
|
|
2738
|
+
if (!(node instanceof HTMLElement)) return;
|
|
2739
|
+
const nearestTypedAncestor = node.closest("[typeof]");
|
|
2740
|
+
const isNestedEntityRoot = nearestTypedAncestor === node && node.hasAttribute("typeof");
|
|
2741
|
+
if (nearestTypedAncestor !== scope && !isNestedEntityRoot) {
|
|
2742
|
+
return;
|
|
2743
|
+
}
|
|
2744
|
+
if (isNestedEntityRoot && node.parentElement?.closest("[typeof]") !== scope && node !== scope) {
|
|
2745
|
+
return;
|
|
2746
|
+
}
|
|
2747
|
+
const propNames = (node.getAttribute("property") || "").split(/\s+/).map((name) => name.trim()).filter(Boolean);
|
|
2748
|
+
if (propNames.length === 0) return;
|
|
2749
|
+
const value = node.hasAttribute("typeof") && isNestedEntityRoot && node !== scope ? serializeEntity(node, depth + 1) : getStructuredElementValue(node);
|
|
2750
|
+
if (value == null) return;
|
|
2751
|
+
propNames.forEach((name) => pushPropertyValue(entity, name, value));
|
|
2752
|
+
});
|
|
2753
|
+
return Object.keys(entity).length > 0 ? entity : null;
|
|
2754
|
+
};
|
|
2755
|
+
return Array.from(document.querySelectorAll("[typeof]")).filter((node) => node instanceof HTMLElement).map((scope) => serializeEntity(scope)).filter((entity) => entity !== null);
|
|
2756
|
+
}
|
|
2757
|
+
function withHighlightLabelsRemoved(read) {
|
|
2758
|
+
const labels = Array.from(
|
|
2759
|
+
document.querySelectorAll(".__vessel-highlight-label[data-vessel-highlight]")
|
|
2760
|
+
).filter((node) => node instanceof HTMLElement);
|
|
2761
|
+
const removed = labels.map((label) => {
|
|
2762
|
+
const parent = label.parentNode;
|
|
2763
|
+
if (!parent) return null;
|
|
2764
|
+
const nextSibling = label.nextSibling;
|
|
2765
|
+
parent.removeChild(label);
|
|
2766
|
+
return { label, parent, nextSibling };
|
|
2767
|
+
}).filter(
|
|
2768
|
+
(entry) => entry !== null
|
|
2769
|
+
);
|
|
2770
|
+
try {
|
|
2771
|
+
return read();
|
|
2772
|
+
} finally {
|
|
2773
|
+
for (let i = removed.length - 1; i >= 0; i -= 1) {
|
|
2774
|
+
const { label, parent, nextSibling } = removed[i];
|
|
2775
|
+
parent.insertBefore(label, nextSibling);
|
|
2776
|
+
}
|
|
2777
|
+
}
|
|
2778
|
+
}
|
|
2779
|
+
function getVisiblePageText() {
|
|
2780
|
+
return withHighlightLabelsRemoved(
|
|
2781
|
+
() => document.body?.innerText || document.documentElement?.innerText || ""
|
|
2782
|
+
);
|
|
2783
|
+
}
|
|
2784
|
+
function vesselExtractContent() {
|
|
2785
|
+
const extractStructuredContent = (article) => {
|
|
2786
|
+
activeOverlays = detectOverlays();
|
|
2787
|
+
return {
|
|
2788
|
+
title: article?.title || document.title,
|
|
2789
|
+
content: article?.textContent || getVisiblePageText(),
|
|
2790
|
+
htmlContent: article?.content || "",
|
|
2791
|
+
byline: article?.byline || "",
|
|
2792
|
+
excerpt: article?.excerpt || "",
|
|
2793
|
+
url: window.location.href,
|
|
2794
|
+
headings: extractHeadings(),
|
|
2795
|
+
navigation: extractNavigation(),
|
|
2796
|
+
interactiveElements: extractInteractiveElements(),
|
|
2797
|
+
forms: extractForms(),
|
|
2798
|
+
viewport: getViewportSnapshot(),
|
|
2799
|
+
overlays: activeOverlays.map(
|
|
2800
|
+
({ element: _element, zIndex: _zIndex, ...overlay }) => overlay
|
|
2801
|
+
),
|
|
2802
|
+
dormantOverlays: detectDormantOverlays(),
|
|
2803
|
+
landmarks: extractLandmarks(),
|
|
2804
|
+
jsonLd: extractJsonLd(),
|
|
2805
|
+
microdata: extractMicrodata(),
|
|
2806
|
+
rdfa: extractRdfa(),
|
|
2807
|
+
metaTags: extractMetaTags()
|
|
2808
|
+
};
|
|
2809
|
+
};
|
|
2810
|
+
try {
|
|
2811
|
+
elementIndex = 0;
|
|
2812
|
+
activeOverlays = [];
|
|
2813
|
+
Object.keys(elementSelectors).forEach(
|
|
2814
|
+
(key) => delete elementSelectors[key]
|
|
2815
|
+
);
|
|
2816
|
+
const documentClone = document.cloneNode(true);
|
|
2817
|
+
const reader = new readabilityExports.Readability(documentClone);
|
|
2818
|
+
const article = reader.parse();
|
|
2819
|
+
return extractStructuredContent(article || void 0);
|
|
2820
|
+
} catch (error) {
|
|
2821
|
+
console.error("Vessel content extraction error:", error);
|
|
2822
|
+
return extractStructuredContent();
|
|
2823
|
+
}
|
|
2824
|
+
}
|
|
2825
|
+
function resolveElementSelector(index) {
|
|
2826
|
+
return elementSelectors[index] || null;
|
|
2827
|
+
}
|
|
2828
|
+
electron.contextBridge.exposeInMainWorld("__vessel", {
|
|
2829
|
+
extractContent: vesselExtractContent,
|
|
2830
|
+
getElementSelector: resolveElementSelector,
|
|
2831
|
+
notifyHighlightSelection: (text) => {
|
|
2832
|
+
if (typeof text === "string" && text.trim()) {
|
|
2833
|
+
electron.ipcRenderer.send("vessel:highlight-selection", text.trim());
|
|
2834
|
+
}
|
|
2835
|
+
}
|
|
2836
|
+
});
|