@pablopunk/tau 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-storage-Dk3CUY3q.js +1814 -0
- package/dist/bin.js +23 -0
- package/dist/cli-entry-DcDpQ30L.js +13 -0
- package/dist/client-C91-b2CL.js +1 -0
- package/dist/client-y2WkNuZj.js +7977 -0
- package/dist/dist-BFP19NOO.js +432 -0
- package/dist/dist-Bh1bQHZA.js +38890 -0
- package/dist/dist-Cq73_r65.js +3 -0
- package/dist/esm-CMGh_WhC.js +9446 -0
- package/dist/execute-BeGXhU1O.js +1757 -0
- package/dist/fake-claude-connectors-DSP9UC8d.js +1 -0
- package/dist/fake-claude-connectors-sDLeP10T.js +107 -0
- package/dist/fake-claude-sdk-CvqW_1iQ.js +571 -0
- package/dist/fake-claude-thread-script-2iHHwUZM.js +48 -0
- package/dist/last-exit-write-ksHeHGrC.js +9 -0
- package/dist/latest-version-di9hD7cY.js +106 -0
- package/dist/lib-DIX7q_HJ.js +1460 -0
- package/dist/machines-behind-BjGgmSs_.js +257 -0
- package/dist/main-D-Tg9b5Y.js +1610 -0
- package/dist/manager-B_WHbb1U.js +589 -0
- package/dist/names-cTrEXB6w.js +35 -0
- package/dist/node-BpwblPFP.js +56269 -0
- package/dist/pdfjs-nTcHjDPT.js +69856 -0
- package/dist/photon_rs-DBr7W_rK.js +4252 -0
- package/dist/promise.try-CAAOYvMP.js +1426 -0
- package/dist/readability-DsDNO63a.js +1670 -0
- package/dist/real-deps-f96UmLBz.js +89 -0
- package/dist/rolldown-runtime-1p2KPIdQ.js +44 -0
- package/dist/runtime-i-oEtgAl.js +2721 -0
- package/dist/sdk-missing-CHbVbY1G.js +10 -0
- package/dist/sdk-port-BGw9ORUR.js +215 -0
- package/dist/serve-Cmy0rUWr.js +2932 -0
- package/dist/serve-Dmzn5V0l.js +1 -0
- package/dist/service-command-C74K0saj.js +34 -0
- package/dist/service-command-DqcstJ8e.js +1 -0
- package/dist/shell-CQa4e7rx.js +643 -0
- package/dist/skills/builtin/tau/SKILL.md +51 -0
- package/dist/src-DO9LUlhi.js +44088 -0
- package/dist/streamableHttp-BBaRYV9h.js +1 -0
- package/dist/streamableHttp-DPAoMlJd.js +1931 -0
- package/dist/tool-DBZK2lci.js +14335 -0
- package/dist/tool-server-BjP37EJg.js +40 -0
- package/dist/turndown.cjs-Bo1xLg2S.js +15620 -0
- package/dist/turndown.cjs-jpUF50AR.js +3 -0
- package/dist/types-C4M3CRp2.js +7592 -0
- package/dist/undici-8cI97lnP.js +27110 -0
- package/dist/wait-for-lock-B1Qx8GWg.js +1 -0
- package/dist/wait-for-lock-CrSl2mi1.js +25 -0
- package/dist/watchdog.js +47 -0
- package/dist/web/dist/assets/addon-fit-DIOBYJe3.js +1 -0
- package/dist/web/dist/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
- package/dist/web/dist/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
- package/dist/web/dist/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
- package/dist/web/dist/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
- package/dist/web/dist/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
- package/dist/web/dist/assets/index-BktLKZHR.js +108 -0
- package/dist/web/dist/assets/index-D_Aa3Oou.css +2 -0
- package/dist/web/dist/assets/jetbrains-mono-cyrillic-wght-normal-D73BlboJ.woff2 +0 -0
- package/dist/web/dist/assets/jetbrains-mono-greek-wght-normal-Bw9x6K1M.woff2 +0 -0
- package/dist/web/dist/assets/jetbrains-mono-latin-ext-wght-normal-DBQx-q_a.woff2 +0 -0
- package/dist/web/dist/assets/jetbrains-mono-latin-wght-normal-B9CIFXIH.woff2 +0 -0
- package/dist/web/dist/assets/jetbrains-mono-vietnamese-wght-normal-Bt-aOZkq.woff2 +0 -0
- package/dist/web/dist/assets/wght-DMLdVLkb.css +1 -0
- package/dist/web/dist/assets/xterm-BrP-ENHg.css +1 -0
- package/dist/web/dist/assets/xterm-CerOhKYV.js +16 -0
- package/dist/web/dist/favicon.svg +14 -0
- package/dist/web/dist/index.html +25 -0
- package/package.json +53 -4
- package/README.md +0 -3
|
@@ -0,0 +1,1670 @@
|
|
|
1
|
+
import { t as __commonJSMin } from "./rolldown-runtime-1p2KPIdQ.js";
|
|
2
|
+
//#region ../../node_modules/.pnpm/@mozilla+readability@0.6.0/node_modules/@mozilla/readability/Readability.js
|
|
3
|
+
var require_Readability = /* @__PURE__ */ __commonJSMin(((exports, module) => {
|
|
4
|
+
/**
|
|
5
|
+
* Public constructor.
|
|
6
|
+
* @param {HTMLDocument} doc The document to parse.
|
|
7
|
+
* @param {Object} options The options object.
|
|
8
|
+
*/
|
|
9
|
+
function Readability(doc, options) {
|
|
10
|
+
if (options && options.documentElement) {
|
|
11
|
+
doc = options;
|
|
12
|
+
options = arguments[2];
|
|
13
|
+
} else if (!doc || !doc.documentElement) throw new Error("First argument to Readability constructor should be a document object.");
|
|
14
|
+
options = options || {};
|
|
15
|
+
this._doc = doc;
|
|
16
|
+
this._docJSDOMParser = this._doc.firstChild.__JSDOMParser__;
|
|
17
|
+
this._articleTitle = null;
|
|
18
|
+
this._articleByline = null;
|
|
19
|
+
this._articleDir = null;
|
|
20
|
+
this._articleSiteName = null;
|
|
21
|
+
this._attempts = [];
|
|
22
|
+
this._metadata = {};
|
|
23
|
+
this._debug = !!options.debug;
|
|
24
|
+
this._maxElemsToParse = options.maxElemsToParse || this.DEFAULT_MAX_ELEMS_TO_PARSE;
|
|
25
|
+
this._nbTopCandidates = options.nbTopCandidates || this.DEFAULT_N_TOP_CANDIDATES;
|
|
26
|
+
this._charThreshold = options.charThreshold || this.DEFAULT_CHAR_THRESHOLD;
|
|
27
|
+
this._classesToPreserve = this.CLASSES_TO_PRESERVE.concat(options.classesToPreserve || []);
|
|
28
|
+
this._keepClasses = !!options.keepClasses;
|
|
29
|
+
this._serializer = options.serializer || function(el) {
|
|
30
|
+
return el.innerHTML;
|
|
31
|
+
};
|
|
32
|
+
this._disableJSONLD = !!options.disableJSONLD;
|
|
33
|
+
this._allowedVideoRegex = options.allowedVideoRegex || this.REGEXPS.videos;
|
|
34
|
+
this._linkDensityModifier = options.linkDensityModifier || 0;
|
|
35
|
+
this._flags = this.FLAG_STRIP_UNLIKELYS | this.FLAG_WEIGHT_CLASSES | this.FLAG_CLEAN_CONDITIONALLY;
|
|
36
|
+
if (this._debug) {
|
|
37
|
+
let logNode = function(node) {
|
|
38
|
+
if (node.nodeType == node.TEXT_NODE) return `${node.nodeName} ("${node.textContent}")`;
|
|
39
|
+
let attrPairs = Array.from(node.attributes || [], function(attr) {
|
|
40
|
+
return `${attr.name}="${attr.value}"`;
|
|
41
|
+
}).join(" ");
|
|
42
|
+
return `<${node.localName} ${attrPairs}>`;
|
|
43
|
+
};
|
|
44
|
+
this.log = function() {
|
|
45
|
+
if (typeof console !== "undefined") {
|
|
46
|
+
let args = Array.from(arguments, (arg) => {
|
|
47
|
+
if (arg && arg.nodeType == this.ELEMENT_NODE) return logNode(arg);
|
|
48
|
+
return arg;
|
|
49
|
+
});
|
|
50
|
+
args.unshift("Reader: (Readability)");
|
|
51
|
+
console.log(...args);
|
|
52
|
+
} else if (typeof dump !== "undefined") {
|
|
53
|
+
var msg = Array.prototype.map.call(arguments, function(x) {
|
|
54
|
+
return x && x.nodeName ? logNode(x) : x;
|
|
55
|
+
}).join(" ");
|
|
56
|
+
dump("Reader: (Readability) " + msg + "\n");
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
} else this.log = function() {};
|
|
60
|
+
}
|
|
61
|
+
Readability.prototype = {
|
|
62
|
+
FLAG_STRIP_UNLIKELYS: 1,
|
|
63
|
+
FLAG_WEIGHT_CLASSES: 2,
|
|
64
|
+
FLAG_CLEAN_CONDITIONALLY: 4,
|
|
65
|
+
ELEMENT_NODE: 1,
|
|
66
|
+
TEXT_NODE: 3,
|
|
67
|
+
DEFAULT_MAX_ELEMS_TO_PARSE: 0,
|
|
68
|
+
DEFAULT_N_TOP_CANDIDATES: 5,
|
|
69
|
+
DEFAULT_TAGS_TO_SCORE: "section,h2,h3,h4,h5,h6,p,td,pre".toUpperCase().split(","),
|
|
70
|
+
DEFAULT_CHAR_THRESHOLD: 500,
|
|
71
|
+
REGEXPS: {
|
|
72
|
+
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
|
|
73
|
+
okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i,
|
|
74
|
+
positive: /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,
|
|
75
|
+
negative: /-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget/i,
|
|
76
|
+
extraneous: /print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,
|
|
77
|
+
byline: /byline|author|dateline|writtenby|p-author/i,
|
|
78
|
+
replaceFonts: /<(\/?)font[^>]*>/gi,
|
|
79
|
+
normalize: /\s{2,}/g,
|
|
80
|
+
videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,
|
|
81
|
+
shareElements: /(\b|_)(share|sharedaddy)(\b|_)/i,
|
|
82
|
+
nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,
|
|
83
|
+
prevLink: /(prev|earl|old|new|<|«)/i,
|
|
84
|
+
tokenize: /\W+/g,
|
|
85
|
+
whitespace: /^\s*$/,
|
|
86
|
+
hasContent: /\S$/,
|
|
87
|
+
hashUrl: /^#.+/,
|
|
88
|
+
srcsetUrl: /(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))/g,
|
|
89
|
+
b64DataUrl: /^data:\s*([^\s;,]+)\s*;\s*base64\s*,/i,
|
|
90
|
+
commas: /\u002C|\u060C|\uFE50|\uFE10|\uFE11|\u2E41|\u2E34|\u2E32|\uFF0C/g,
|
|
91
|
+
jsonLdArticleTypes: /^Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|AskPublicNewsArticle|BackgroundNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|SatiricalArticle|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference$/,
|
|
92
|
+
adWords: /^(ad(vertising|vertisement)?|pub(licité)?|werb(ung)?|广告|Реклама|Anuncio)$/iu,
|
|
93
|
+
loadingWords: /^((loading|正在加载|Загрузка|chargement|cargando)(…|\.\.\.)?)$/iu
|
|
94
|
+
},
|
|
95
|
+
UNLIKELY_ROLES: [
|
|
96
|
+
"menu",
|
|
97
|
+
"menubar",
|
|
98
|
+
"complementary",
|
|
99
|
+
"navigation",
|
|
100
|
+
"alert",
|
|
101
|
+
"alertdialog",
|
|
102
|
+
"dialog"
|
|
103
|
+
],
|
|
104
|
+
DIV_TO_P_ELEMS: /* @__PURE__ */ new Set([
|
|
105
|
+
"BLOCKQUOTE",
|
|
106
|
+
"DL",
|
|
107
|
+
"DIV",
|
|
108
|
+
"IMG",
|
|
109
|
+
"OL",
|
|
110
|
+
"P",
|
|
111
|
+
"PRE",
|
|
112
|
+
"TABLE",
|
|
113
|
+
"UL"
|
|
114
|
+
]),
|
|
115
|
+
ALTER_TO_DIV_EXCEPTIONS: [
|
|
116
|
+
"DIV",
|
|
117
|
+
"ARTICLE",
|
|
118
|
+
"SECTION",
|
|
119
|
+
"P",
|
|
120
|
+
"OL",
|
|
121
|
+
"UL"
|
|
122
|
+
],
|
|
123
|
+
PRESENTATIONAL_ATTRIBUTES: [
|
|
124
|
+
"align",
|
|
125
|
+
"background",
|
|
126
|
+
"bgcolor",
|
|
127
|
+
"border",
|
|
128
|
+
"cellpadding",
|
|
129
|
+
"cellspacing",
|
|
130
|
+
"frame",
|
|
131
|
+
"hspace",
|
|
132
|
+
"rules",
|
|
133
|
+
"style",
|
|
134
|
+
"valign",
|
|
135
|
+
"vspace"
|
|
136
|
+
],
|
|
137
|
+
DEPRECATED_SIZE_ATTRIBUTE_ELEMS: [
|
|
138
|
+
"TABLE",
|
|
139
|
+
"TH",
|
|
140
|
+
"TD",
|
|
141
|
+
"HR",
|
|
142
|
+
"PRE"
|
|
143
|
+
],
|
|
144
|
+
PHRASING_ELEMS: [
|
|
145
|
+
"ABBR",
|
|
146
|
+
"AUDIO",
|
|
147
|
+
"B",
|
|
148
|
+
"BDO",
|
|
149
|
+
"BR",
|
|
150
|
+
"BUTTON",
|
|
151
|
+
"CITE",
|
|
152
|
+
"CODE",
|
|
153
|
+
"DATA",
|
|
154
|
+
"DATALIST",
|
|
155
|
+
"DFN",
|
|
156
|
+
"EM",
|
|
157
|
+
"EMBED",
|
|
158
|
+
"I",
|
|
159
|
+
"IMG",
|
|
160
|
+
"INPUT",
|
|
161
|
+
"KBD",
|
|
162
|
+
"LABEL",
|
|
163
|
+
"MARK",
|
|
164
|
+
"MATH",
|
|
165
|
+
"METER",
|
|
166
|
+
"NOSCRIPT",
|
|
167
|
+
"OBJECT",
|
|
168
|
+
"OUTPUT",
|
|
169
|
+
"PROGRESS",
|
|
170
|
+
"Q",
|
|
171
|
+
"RUBY",
|
|
172
|
+
"SAMP",
|
|
173
|
+
"SCRIPT",
|
|
174
|
+
"SELECT",
|
|
175
|
+
"SMALL",
|
|
176
|
+
"SPAN",
|
|
177
|
+
"STRONG",
|
|
178
|
+
"SUB",
|
|
179
|
+
"SUP",
|
|
180
|
+
"TEXTAREA",
|
|
181
|
+
"TIME",
|
|
182
|
+
"VAR",
|
|
183
|
+
"WBR"
|
|
184
|
+
],
|
|
185
|
+
CLASSES_TO_PRESERVE: ["page"],
|
|
186
|
+
HTML_ESCAPE_MAP: {
|
|
187
|
+
lt: "<",
|
|
188
|
+
gt: ">",
|
|
189
|
+
amp: "&",
|
|
190
|
+
quot: "\"",
|
|
191
|
+
apos: "'"
|
|
192
|
+
},
|
|
193
|
+
/**
|
|
194
|
+
* Run any post-process modifications to article content as necessary.
|
|
195
|
+
*
|
|
196
|
+
* @param Element
|
|
197
|
+
* @return void
|
|
198
|
+
**/
|
|
199
|
+
_postProcessContent(articleContent) {
|
|
200
|
+
this._fixRelativeUris(articleContent);
|
|
201
|
+
this._simplifyNestedElements(articleContent);
|
|
202
|
+
if (!this._keepClasses) this._cleanClasses(articleContent);
|
|
203
|
+
},
|
|
204
|
+
/**
|
|
205
|
+
* Iterates over a NodeList, calls `filterFn` for each node and removes node
|
|
206
|
+
* if function returned `true`.
|
|
207
|
+
*
|
|
208
|
+
* If function is not passed, removes all the nodes in node list.
|
|
209
|
+
*
|
|
210
|
+
* @param NodeList nodeList The nodes to operate on
|
|
211
|
+
* @param Function filterFn the function to use as a filter
|
|
212
|
+
* @return void
|
|
213
|
+
*/
|
|
214
|
+
_removeNodes(nodeList, filterFn) {
|
|
215
|
+
if (this._docJSDOMParser && nodeList._isLiveNodeList) throw new Error("Do not pass live node lists to _removeNodes");
|
|
216
|
+
for (var i = nodeList.length - 1; i >= 0; i--) {
|
|
217
|
+
var node = nodeList[i];
|
|
218
|
+
var parentNode = node.parentNode;
|
|
219
|
+
if (parentNode) {
|
|
220
|
+
if (!filterFn || filterFn.call(this, node, i, nodeList)) parentNode.removeChild(node);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
},
|
|
224
|
+
/**
|
|
225
|
+
* Iterates over a NodeList, and calls _setNodeTag for each node.
|
|
226
|
+
*
|
|
227
|
+
* @param NodeList nodeList The nodes to operate on
|
|
228
|
+
* @param String newTagName the new tag name to use
|
|
229
|
+
* @return void
|
|
230
|
+
*/
|
|
231
|
+
_replaceNodeTags(nodeList, newTagName) {
|
|
232
|
+
if (this._docJSDOMParser && nodeList._isLiveNodeList) throw new Error("Do not pass live node lists to _replaceNodeTags");
|
|
233
|
+
for (const node of nodeList) this._setNodeTag(node, newTagName);
|
|
234
|
+
},
|
|
235
|
+
/**
|
|
236
|
+
* Iterate over a NodeList, which doesn't natively fully implement the Array
|
|
237
|
+
* interface.
|
|
238
|
+
*
|
|
239
|
+
* For convenience, the current object context is applied to the provided
|
|
240
|
+
* iterate function.
|
|
241
|
+
*
|
|
242
|
+
* @param NodeList nodeList The NodeList.
|
|
243
|
+
* @param Function fn The iterate function.
|
|
244
|
+
* @return void
|
|
245
|
+
*/
|
|
246
|
+
_forEachNode(nodeList, fn) {
|
|
247
|
+
Array.prototype.forEach.call(nodeList, fn, this);
|
|
248
|
+
},
|
|
249
|
+
/**
|
|
250
|
+
* Iterate over a NodeList, and return the first node that passes
|
|
251
|
+
* the supplied test function
|
|
252
|
+
*
|
|
253
|
+
* For convenience, the current object context is applied to the provided
|
|
254
|
+
* test function.
|
|
255
|
+
*
|
|
256
|
+
* @param NodeList nodeList The NodeList.
|
|
257
|
+
* @param Function fn The test function.
|
|
258
|
+
* @return void
|
|
259
|
+
*/
|
|
260
|
+
_findNode(nodeList, fn) {
|
|
261
|
+
return Array.prototype.find.call(nodeList, fn, this);
|
|
262
|
+
},
|
|
263
|
+
/**
|
|
264
|
+
* Iterate over a NodeList, return true if any of the provided iterate
|
|
265
|
+
* function calls returns true, false otherwise.
|
|
266
|
+
*
|
|
267
|
+
* For convenience, the current object context is applied to the
|
|
268
|
+
* provided iterate function.
|
|
269
|
+
*
|
|
270
|
+
* @param NodeList nodeList The NodeList.
|
|
271
|
+
* @param Function fn The iterate function.
|
|
272
|
+
* @return Boolean
|
|
273
|
+
*/
|
|
274
|
+
_someNode(nodeList, fn) {
|
|
275
|
+
return Array.prototype.some.call(nodeList, fn, this);
|
|
276
|
+
},
|
|
277
|
+
/**
|
|
278
|
+
* Iterate over a NodeList, return true if all of the provided iterate
|
|
279
|
+
* function calls return true, false otherwise.
|
|
280
|
+
*
|
|
281
|
+
* For convenience, the current object context is applied to the
|
|
282
|
+
* provided iterate function.
|
|
283
|
+
*
|
|
284
|
+
* @param NodeList nodeList The NodeList.
|
|
285
|
+
* @param Function fn The iterate function.
|
|
286
|
+
* @return Boolean
|
|
287
|
+
*/
|
|
288
|
+
_everyNode(nodeList, fn) {
|
|
289
|
+
return Array.prototype.every.call(nodeList, fn, this);
|
|
290
|
+
},
|
|
291
|
+
_getAllNodesWithTag(node, tagNames) {
|
|
292
|
+
if (node.querySelectorAll) return node.querySelectorAll(tagNames.join(","));
|
|
293
|
+
return [].concat.apply([], tagNames.map(function(tag) {
|
|
294
|
+
var collection = node.getElementsByTagName(tag);
|
|
295
|
+
return Array.isArray(collection) ? collection : Array.from(collection);
|
|
296
|
+
}));
|
|
297
|
+
},
|
|
298
|
+
/**
|
|
299
|
+
* Removes the class="" attribute from every element in the given
|
|
300
|
+
* subtree, except those that match CLASSES_TO_PRESERVE and
|
|
301
|
+
* the classesToPreserve array from the options object.
|
|
302
|
+
*
|
|
303
|
+
* @param Element
|
|
304
|
+
* @return void
|
|
305
|
+
*/
|
|
306
|
+
_cleanClasses(node) {
|
|
307
|
+
var classesToPreserve = this._classesToPreserve;
|
|
308
|
+
var className = (node.getAttribute("class") || "").split(/\s+/).filter((cls) => classesToPreserve.includes(cls)).join(" ");
|
|
309
|
+
if (className) node.setAttribute("class", className);
|
|
310
|
+
else node.removeAttribute("class");
|
|
311
|
+
for (node = node.firstElementChild; node; node = node.nextElementSibling) this._cleanClasses(node);
|
|
312
|
+
},
|
|
313
|
+
/**
|
|
314
|
+
* Tests whether a string is a URL or not.
|
|
315
|
+
*
|
|
316
|
+
* @param {string} str The string to test
|
|
317
|
+
* @return {boolean} true if str is a URL, false if not
|
|
318
|
+
*/
|
|
319
|
+
_isUrl(str) {
|
|
320
|
+
try {
|
|
321
|
+
new URL(str);
|
|
322
|
+
return true;
|
|
323
|
+
} catch {
|
|
324
|
+
return false;
|
|
325
|
+
}
|
|
326
|
+
},
|
|
327
|
+
/**
|
|
328
|
+
* Converts each <a> and <img> uri in the given element to an absolute URI,
|
|
329
|
+
* ignoring #ref URIs.
|
|
330
|
+
*
|
|
331
|
+
* @param Element
|
|
332
|
+
* @return void
|
|
333
|
+
*/
|
|
334
|
+
_fixRelativeUris(articleContent) {
|
|
335
|
+
var baseURI = this._doc.baseURI;
|
|
336
|
+
var documentURI = this._doc.documentURI;
|
|
337
|
+
function toAbsoluteURI(uri) {
|
|
338
|
+
if (baseURI == documentURI && uri.charAt(0) == "#") return uri;
|
|
339
|
+
try {
|
|
340
|
+
return new URL(uri, baseURI).href;
|
|
341
|
+
} catch (ex) {}
|
|
342
|
+
return uri;
|
|
343
|
+
}
|
|
344
|
+
var links = this._getAllNodesWithTag(articleContent, ["a"]);
|
|
345
|
+
this._forEachNode(links, function(link) {
|
|
346
|
+
var href = link.getAttribute("href");
|
|
347
|
+
if (href) {
|
|
348
|
+
if (href.indexOf("javascript:") === 0) {
|
|
349
|
+
if (link.childNodes.length === 1 && link.childNodes[0].nodeType === this.TEXT_NODE) {
|
|
350
|
+
var text = this._doc.createTextNode(link.textContent);
|
|
351
|
+
link.parentNode.replaceChild(text, link);
|
|
352
|
+
} else {
|
|
353
|
+
var container = this._doc.createElement("span");
|
|
354
|
+
while (link.firstChild) container.appendChild(link.firstChild);
|
|
355
|
+
link.parentNode.replaceChild(container, link);
|
|
356
|
+
}
|
|
357
|
+
} else link.setAttribute("href", toAbsoluteURI(href));
|
|
358
|
+
}
|
|
359
|
+
});
|
|
360
|
+
var medias = this._getAllNodesWithTag(articleContent, [
|
|
361
|
+
"img",
|
|
362
|
+
"picture",
|
|
363
|
+
"figure",
|
|
364
|
+
"video",
|
|
365
|
+
"audio",
|
|
366
|
+
"source"
|
|
367
|
+
]);
|
|
368
|
+
this._forEachNode(medias, function(media) {
|
|
369
|
+
var src = media.getAttribute("src");
|
|
370
|
+
var poster = media.getAttribute("poster");
|
|
371
|
+
var srcset = media.getAttribute("srcset");
|
|
372
|
+
if (src) media.setAttribute("src", toAbsoluteURI(src));
|
|
373
|
+
if (poster) media.setAttribute("poster", toAbsoluteURI(poster));
|
|
374
|
+
if (srcset) {
|
|
375
|
+
var newSrcset = srcset.replace(this.REGEXPS.srcsetUrl, function(_, p1, p2, p3) {
|
|
376
|
+
return toAbsoluteURI(p1) + (p2 || "") + p3;
|
|
377
|
+
});
|
|
378
|
+
media.setAttribute("srcset", newSrcset);
|
|
379
|
+
}
|
|
380
|
+
});
|
|
381
|
+
},
|
|
382
|
+
_simplifyNestedElements(articleContent) {
|
|
383
|
+
var node = articleContent;
|
|
384
|
+
while (node) {
|
|
385
|
+
if (node.parentNode && ["DIV", "SECTION"].includes(node.tagName) && !(node.id && node.id.startsWith("readability"))) {
|
|
386
|
+
if (this._isElementWithoutContent(node)) {
|
|
387
|
+
node = this._removeAndGetNext(node);
|
|
388
|
+
continue;
|
|
389
|
+
} else if (this._hasSingleTagInsideElement(node, "DIV") || this._hasSingleTagInsideElement(node, "SECTION")) {
|
|
390
|
+
var child = node.children[0];
|
|
391
|
+
for (var i = 0; i < node.attributes.length; i++) child.setAttributeNode(node.attributes[i].cloneNode());
|
|
392
|
+
node.parentNode.replaceChild(child, node);
|
|
393
|
+
node = child;
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
node = this._getNextNode(node);
|
|
398
|
+
}
|
|
399
|
+
},
|
|
400
|
+
/**
|
|
401
|
+
* Get the article title as an H1.
|
|
402
|
+
*
|
|
403
|
+
* @return string
|
|
404
|
+
**/
|
|
405
|
+
_getArticleTitle() {
|
|
406
|
+
var doc = this._doc;
|
|
407
|
+
var curTitle = "";
|
|
408
|
+
var origTitle = "";
|
|
409
|
+
try {
|
|
410
|
+
curTitle = origTitle = doc.title.trim();
|
|
411
|
+
if (typeof curTitle !== "string") curTitle = origTitle = this._getInnerText(doc.getElementsByTagName("title")[0]);
|
|
412
|
+
} catch (e) {}
|
|
413
|
+
var titleHadHierarchicalSeparators = false;
|
|
414
|
+
function wordCount(str) {
|
|
415
|
+
return str.split(/\s+/).length;
|
|
416
|
+
}
|
|
417
|
+
if (/ [\|\-\\\/>»] /.test(curTitle)) {
|
|
418
|
+
titleHadHierarchicalSeparators = / [\\\/>»] /.test(curTitle);
|
|
419
|
+
let allSeparators = Array.from(origTitle.matchAll(/ [\|\-\\\/>»] /gi));
|
|
420
|
+
curTitle = origTitle.substring(0, allSeparators.pop().index);
|
|
421
|
+
if (wordCount(curTitle) < 3) curTitle = origTitle.replace(/^[^\|\-\\\/>»]*[\|\-\\\/>»]/gi, "");
|
|
422
|
+
} else if (curTitle.includes(": ")) {
|
|
423
|
+
var headings = this._getAllNodesWithTag(doc, ["h1", "h2"]);
|
|
424
|
+
var trimmedTitle = curTitle.trim();
|
|
425
|
+
if (!this._someNode(headings, function(heading) {
|
|
426
|
+
return heading.textContent.trim() === trimmedTitle;
|
|
427
|
+
})) {
|
|
428
|
+
curTitle = origTitle.substring(origTitle.lastIndexOf(":") + 1);
|
|
429
|
+
if (wordCount(curTitle) < 3) curTitle = origTitle.substring(origTitle.indexOf(":") + 1);
|
|
430
|
+
else if (wordCount(origTitle.substr(0, origTitle.indexOf(":"))) > 5) curTitle = origTitle;
|
|
431
|
+
}
|
|
432
|
+
} else if (curTitle.length > 150 || curTitle.length < 15) {
|
|
433
|
+
var hOnes = doc.getElementsByTagName("h1");
|
|
434
|
+
if (hOnes.length === 1) curTitle = this._getInnerText(hOnes[0]);
|
|
435
|
+
}
|
|
436
|
+
curTitle = curTitle.trim().replace(this.REGEXPS.normalize, " ");
|
|
437
|
+
var curTitleWordCount = wordCount(curTitle);
|
|
438
|
+
if (curTitleWordCount <= 4 && (!titleHadHierarchicalSeparators || curTitleWordCount != wordCount(origTitle.replace(/[\|\-\\\/>»]+/g, "")) - 1)) curTitle = origTitle;
|
|
439
|
+
return curTitle;
|
|
440
|
+
},
|
|
441
|
+
/**
|
|
442
|
+
* Prepare the HTML document for readability to scrape it.
|
|
443
|
+
* This includes things like stripping javascript, CSS, and handling terrible markup.
|
|
444
|
+
*
|
|
445
|
+
* @return void
|
|
446
|
+
**/
|
|
447
|
+
_prepDocument() {
|
|
448
|
+
var doc = this._doc;
|
|
449
|
+
this._removeNodes(this._getAllNodesWithTag(doc, ["style"]));
|
|
450
|
+
if (doc.body) this._replaceBrs(doc.body);
|
|
451
|
+
this._replaceNodeTags(this._getAllNodesWithTag(doc, ["font"]), "SPAN");
|
|
452
|
+
},
|
|
453
|
+
/**
|
|
454
|
+
* Finds the next node, starting from the given node, and ignoring
|
|
455
|
+
* whitespace in between. If the given node is an element, the same node is
|
|
456
|
+
* returned.
|
|
457
|
+
*/
|
|
458
|
+
_nextNode(node) {
|
|
459
|
+
var next = node;
|
|
460
|
+
while (next && next.nodeType != this.ELEMENT_NODE && this.REGEXPS.whitespace.test(next.textContent)) next = next.nextSibling;
|
|
461
|
+
return next;
|
|
462
|
+
},
|
|
463
|
+
/**
|
|
464
|
+
* Replaces 2 or more successive <br> elements with a single <p>.
|
|
465
|
+
* Whitespace between <br> elements are ignored. For example:
|
|
466
|
+
* <div>foo<br>bar<br> <br><br>abc</div>
|
|
467
|
+
* will become:
|
|
468
|
+
* <div>foo<br>bar<p>abc</p></div>
|
|
469
|
+
*/
|
|
470
|
+
_replaceBrs(elem) {
|
|
471
|
+
this._forEachNode(this._getAllNodesWithTag(elem, ["br"]), function(br) {
|
|
472
|
+
var next = br.nextSibling;
|
|
473
|
+
var replaced = false;
|
|
474
|
+
while ((next = this._nextNode(next)) && next.tagName == "BR") {
|
|
475
|
+
replaced = true;
|
|
476
|
+
var brSibling = next.nextSibling;
|
|
477
|
+
next.remove();
|
|
478
|
+
next = brSibling;
|
|
479
|
+
}
|
|
480
|
+
if (replaced) {
|
|
481
|
+
var p = this._doc.createElement("p");
|
|
482
|
+
br.parentNode.replaceChild(p, br);
|
|
483
|
+
next = p.nextSibling;
|
|
484
|
+
while (next) {
|
|
485
|
+
if (next.tagName == "BR") {
|
|
486
|
+
var nextElem = this._nextNode(next.nextSibling);
|
|
487
|
+
if (nextElem && nextElem.tagName == "BR") break;
|
|
488
|
+
}
|
|
489
|
+
if (!this._isPhrasingContent(next)) break;
|
|
490
|
+
var sibling = next.nextSibling;
|
|
491
|
+
p.appendChild(next);
|
|
492
|
+
next = sibling;
|
|
493
|
+
}
|
|
494
|
+
while (p.lastChild && this._isWhitespace(p.lastChild)) p.lastChild.remove();
|
|
495
|
+
if (p.parentNode.tagName === "P") this._setNodeTag(p.parentNode, "DIV");
|
|
496
|
+
}
|
|
497
|
+
});
|
|
498
|
+
},
|
|
499
|
+
_setNodeTag(node, tag) {
|
|
500
|
+
this.log("_setNodeTag", node, tag);
|
|
501
|
+
if (this._docJSDOMParser) {
|
|
502
|
+
node.localName = tag.toLowerCase();
|
|
503
|
+
node.tagName = tag.toUpperCase();
|
|
504
|
+
return node;
|
|
505
|
+
}
|
|
506
|
+
var replacement = node.ownerDocument.createElement(tag);
|
|
507
|
+
while (node.firstChild) replacement.appendChild(node.firstChild);
|
|
508
|
+
node.parentNode.replaceChild(replacement, node);
|
|
509
|
+
if (node.readability) replacement.readability = node.readability;
|
|
510
|
+
for (var i = 0; i < node.attributes.length; i++) replacement.setAttributeNode(node.attributes[i].cloneNode());
|
|
511
|
+
return replacement;
|
|
512
|
+
},
|
|
513
|
+
/**
|
|
514
|
+
* Prepare the article node for display. Clean out any inline styles,
|
|
515
|
+
* iframes, forms, strip extraneous <p> tags, etc.
|
|
516
|
+
*
|
|
517
|
+
* @param Element
|
|
518
|
+
* @return void
|
|
519
|
+
**/
|
|
520
|
+
_prepArticle(articleContent) {
|
|
521
|
+
this._cleanStyles(articleContent);
|
|
522
|
+
this._markDataTables(articleContent);
|
|
523
|
+
this._fixLazyImages(articleContent);
|
|
524
|
+
this._cleanConditionally(articleContent, "form");
|
|
525
|
+
this._cleanConditionally(articleContent, "fieldset");
|
|
526
|
+
this._clean(articleContent, "object");
|
|
527
|
+
this._clean(articleContent, "embed");
|
|
528
|
+
this._clean(articleContent, "footer");
|
|
529
|
+
this._clean(articleContent, "link");
|
|
530
|
+
this._clean(articleContent, "aside");
|
|
531
|
+
var shareElementThreshold = this.DEFAULT_CHAR_THRESHOLD;
|
|
532
|
+
this._forEachNode(articleContent.children, function(topCandidate) {
|
|
533
|
+
this._cleanMatchedNodes(topCandidate, function(node, matchString) {
|
|
534
|
+
return this.REGEXPS.shareElements.test(matchString) && node.textContent.length < shareElementThreshold;
|
|
535
|
+
});
|
|
536
|
+
});
|
|
537
|
+
this._clean(articleContent, "iframe");
|
|
538
|
+
this._clean(articleContent, "input");
|
|
539
|
+
this._clean(articleContent, "textarea");
|
|
540
|
+
this._clean(articleContent, "select");
|
|
541
|
+
this._clean(articleContent, "button");
|
|
542
|
+
this._cleanHeaders(articleContent);
|
|
543
|
+
this._cleanConditionally(articleContent, "table");
|
|
544
|
+
this._cleanConditionally(articleContent, "ul");
|
|
545
|
+
this._cleanConditionally(articleContent, "div");
|
|
546
|
+
this._replaceNodeTags(this._getAllNodesWithTag(articleContent, ["h1"]), "h2");
|
|
547
|
+
this._removeNodes(this._getAllNodesWithTag(articleContent, ["p"]), function(paragraph) {
|
|
548
|
+
return this._getAllNodesWithTag(paragraph, [
|
|
549
|
+
"img",
|
|
550
|
+
"embed",
|
|
551
|
+
"object",
|
|
552
|
+
"iframe"
|
|
553
|
+
]).length === 0 && !this._getInnerText(paragraph, false);
|
|
554
|
+
});
|
|
555
|
+
this._forEachNode(this._getAllNodesWithTag(articleContent, ["br"]), function(br) {
|
|
556
|
+
var next = this._nextNode(br.nextSibling);
|
|
557
|
+
if (next && next.tagName == "P") br.remove();
|
|
558
|
+
});
|
|
559
|
+
this._forEachNode(this._getAllNodesWithTag(articleContent, ["table"]), function(table) {
|
|
560
|
+
var tbody = this._hasSingleTagInsideElement(table, "TBODY") ? table.firstElementChild : table;
|
|
561
|
+
if (this._hasSingleTagInsideElement(tbody, "TR")) {
|
|
562
|
+
var row = tbody.firstElementChild;
|
|
563
|
+
if (this._hasSingleTagInsideElement(row, "TD")) {
|
|
564
|
+
var cell = row.firstElementChild;
|
|
565
|
+
cell = this._setNodeTag(cell, this._everyNode(cell.childNodes, this._isPhrasingContent) ? "P" : "DIV");
|
|
566
|
+
table.parentNode.replaceChild(cell, table);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
});
|
|
570
|
+
},
|
|
571
|
+
/**
|
|
572
|
+
* Initialize a node with the readability object. Also checks the
|
|
573
|
+
* className/id for special names to add to its score.
|
|
574
|
+
*
|
|
575
|
+
* @param Element
|
|
576
|
+
* @return void
|
|
577
|
+
**/
|
|
578
|
+
_initializeNode(node) {
|
|
579
|
+
node.readability = { contentScore: 0 };
|
|
580
|
+
switch (node.tagName) {
|
|
581
|
+
case "DIV":
|
|
582
|
+
node.readability.contentScore += 5;
|
|
583
|
+
break;
|
|
584
|
+
case "PRE":
|
|
585
|
+
case "TD":
|
|
586
|
+
case "BLOCKQUOTE":
|
|
587
|
+
node.readability.contentScore += 3;
|
|
588
|
+
break;
|
|
589
|
+
case "ADDRESS":
|
|
590
|
+
case "OL":
|
|
591
|
+
case "UL":
|
|
592
|
+
case "DL":
|
|
593
|
+
case "DD":
|
|
594
|
+
case "DT":
|
|
595
|
+
case "LI":
|
|
596
|
+
case "FORM":
|
|
597
|
+
node.readability.contentScore -= 3;
|
|
598
|
+
break;
|
|
599
|
+
case "H1":
|
|
600
|
+
case "H2":
|
|
601
|
+
case "H3":
|
|
602
|
+
case "H4":
|
|
603
|
+
case "H5":
|
|
604
|
+
case "H6":
|
|
605
|
+
case "TH": node.readability.contentScore -= 5;
|
|
606
|
+
}
|
|
607
|
+
node.readability.contentScore += this._getClassWeight(node);
|
|
608
|
+
},
|
|
609
|
+
_removeAndGetNext(node) {
|
|
610
|
+
var nextNode = this._getNextNode(node, true);
|
|
611
|
+
node.remove();
|
|
612
|
+
return nextNode;
|
|
613
|
+
},
|
|
614
|
+
/**
|
|
615
|
+
* Traverse the DOM from node to node, starting at the node passed in.
|
|
616
|
+
* Pass true for the second parameter to indicate this node itself
|
|
617
|
+
* (and its kids) are going away, and we want the next node over.
|
|
618
|
+
*
|
|
619
|
+
* Calling this in a loop will traverse the DOM depth-first.
|
|
620
|
+
*
|
|
621
|
+
* @param {Element} node
|
|
622
|
+
* @param {boolean} ignoreSelfAndKids
|
|
623
|
+
* @return {Element}
|
|
624
|
+
*/
|
|
625
|
+
_getNextNode(node, ignoreSelfAndKids) {
|
|
626
|
+
if (!ignoreSelfAndKids && node.firstElementChild) return node.firstElementChild;
|
|
627
|
+
if (node.nextElementSibling) return node.nextElementSibling;
|
|
628
|
+
do
|
|
629
|
+
node = node.parentNode;
|
|
630
|
+
while (node && !node.nextElementSibling);
|
|
631
|
+
return node && node.nextElementSibling;
|
|
632
|
+
},
|
|
633
|
+
_textSimilarity(textA, textB) {
|
|
634
|
+
var tokensA = textA.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean);
|
|
635
|
+
var tokensB = textB.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean);
|
|
636
|
+
if (!tokensA.length || !tokensB.length) return 0;
|
|
637
|
+
return 1 - tokensB.filter((token) => !tokensA.includes(token)).join(" ").length / tokensB.join(" ").length;
|
|
638
|
+
},
|
|
639
|
+
/**
|
|
640
|
+
* Checks whether an element node contains a valid byline
|
|
641
|
+
*
|
|
642
|
+
* @param node {Element}
|
|
643
|
+
* @param matchString {string}
|
|
644
|
+
* @return boolean
|
|
645
|
+
*/
|
|
646
|
+
_isValidByline(node, matchString) {
|
|
647
|
+
var rel = node.getAttribute("rel");
|
|
648
|
+
var itemprop = node.getAttribute("itemprop");
|
|
649
|
+
var bylineLength = node.textContent.trim().length;
|
|
650
|
+
return (rel === "author" || itemprop && itemprop.includes("author") || this.REGEXPS.byline.test(matchString)) && !!bylineLength && bylineLength < 100;
|
|
651
|
+
},
|
|
652
|
+
_getNodeAncestors(node, maxDepth) {
|
|
653
|
+
maxDepth = maxDepth || 0;
|
|
654
|
+
var i = 0, ancestors = [];
|
|
655
|
+
while (node.parentNode) {
|
|
656
|
+
ancestors.push(node.parentNode);
|
|
657
|
+
if (maxDepth && ++i === maxDepth) break;
|
|
658
|
+
node = node.parentNode;
|
|
659
|
+
}
|
|
660
|
+
return ancestors;
|
|
661
|
+
},
|
|
662
|
+
/***
|
|
663
|
+
* grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
|
|
664
|
+
* most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
|
|
665
|
+
*
|
|
666
|
+
* @param page a document to run upon. Needs to be a full document, complete with body.
|
|
667
|
+
* @return Element
|
|
668
|
+
**/
|
|
669
|
+
_grabArticle(page) {
|
|
670
|
+
this.log("**** grabArticle ****");
|
|
671
|
+
var doc = this._doc;
|
|
672
|
+
var isPaging = page !== null;
|
|
673
|
+
page = page ? page : this._doc.body;
|
|
674
|
+
if (!page) {
|
|
675
|
+
this.log("No body found in document. Abort.");
|
|
676
|
+
return null;
|
|
677
|
+
}
|
|
678
|
+
var pageCacheHtml = page.innerHTML;
|
|
679
|
+
while (true) {
|
|
680
|
+
this.log("Starting grabArticle loop");
|
|
681
|
+
var stripUnlikelyCandidates = this._flagIsActive(this.FLAG_STRIP_UNLIKELYS);
|
|
682
|
+
var elementsToScore = [];
|
|
683
|
+
var node = this._doc.documentElement;
|
|
684
|
+
let shouldRemoveTitleHeader = true;
|
|
685
|
+
while (node) {
|
|
686
|
+
if (node.tagName === "HTML") this._articleLang = node.getAttribute("lang");
|
|
687
|
+
var matchString = node.className + " " + node.id;
|
|
688
|
+
if (!this._isProbablyVisible(node)) {
|
|
689
|
+
this.log("Removing hidden node - " + matchString);
|
|
690
|
+
node = this._removeAndGetNext(node);
|
|
691
|
+
continue;
|
|
692
|
+
}
|
|
693
|
+
if (node.getAttribute("aria-modal") == "true" && node.getAttribute("role") == "dialog") {
|
|
694
|
+
node = this._removeAndGetNext(node);
|
|
695
|
+
continue;
|
|
696
|
+
}
|
|
697
|
+
if (!this._articleByline && !this._metadata.byline && this._isValidByline(node, matchString)) {
|
|
698
|
+
var endOfSearchMarkerNode = this._getNextNode(node, true);
|
|
699
|
+
var next = this._getNextNode(node);
|
|
700
|
+
var itemPropNameNode = null;
|
|
701
|
+
while (next && next != endOfSearchMarkerNode) {
|
|
702
|
+
var itemprop = next.getAttribute("itemprop");
|
|
703
|
+
if (itemprop && itemprop.includes("name")) {
|
|
704
|
+
itemPropNameNode = next;
|
|
705
|
+
break;
|
|
706
|
+
} else next = this._getNextNode(next);
|
|
707
|
+
}
|
|
708
|
+
this._articleByline = (itemPropNameNode ?? node).textContent.trim();
|
|
709
|
+
node = this._removeAndGetNext(node);
|
|
710
|
+
continue;
|
|
711
|
+
}
|
|
712
|
+
if (shouldRemoveTitleHeader && this._headerDuplicatesTitle(node)) {
|
|
713
|
+
this.log("Removing header: ", node.textContent.trim(), this._articleTitle.trim());
|
|
714
|
+
shouldRemoveTitleHeader = false;
|
|
715
|
+
node = this._removeAndGetNext(node);
|
|
716
|
+
continue;
|
|
717
|
+
}
|
|
718
|
+
if (stripUnlikelyCandidates) {
|
|
719
|
+
if (this.REGEXPS.unlikelyCandidates.test(matchString) && !this.REGEXPS.okMaybeItsACandidate.test(matchString) && !this._hasAncestorTag(node, "table") && !this._hasAncestorTag(node, "code") && node.tagName !== "BODY" && node.tagName !== "A") {
|
|
720
|
+
this.log("Removing unlikely candidate - " + matchString);
|
|
721
|
+
node = this._removeAndGetNext(node);
|
|
722
|
+
continue;
|
|
723
|
+
}
|
|
724
|
+
if (this.UNLIKELY_ROLES.includes(node.getAttribute("role"))) {
|
|
725
|
+
this.log("Removing content with role " + node.getAttribute("role") + " - " + matchString);
|
|
726
|
+
node = this._removeAndGetNext(node);
|
|
727
|
+
continue;
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
if ((node.tagName === "DIV" || node.tagName === "SECTION" || node.tagName === "HEADER" || node.tagName === "H1" || node.tagName === "H2" || node.tagName === "H3" || node.tagName === "H4" || node.tagName === "H5" || node.tagName === "H6") && this._isElementWithoutContent(node)) {
|
|
731
|
+
node = this._removeAndGetNext(node);
|
|
732
|
+
continue;
|
|
733
|
+
}
|
|
734
|
+
if (this.DEFAULT_TAGS_TO_SCORE.includes(node.tagName)) elementsToScore.push(node);
|
|
735
|
+
if (node.tagName === "DIV") {
|
|
736
|
+
var p = null;
|
|
737
|
+
var childNode = node.firstChild;
|
|
738
|
+
while (childNode) {
|
|
739
|
+
var nextSibling = childNode.nextSibling;
|
|
740
|
+
if (this._isPhrasingContent(childNode)) {
|
|
741
|
+
if (p !== null) p.appendChild(childNode);
|
|
742
|
+
else if (!this._isWhitespace(childNode)) {
|
|
743
|
+
p = doc.createElement("p");
|
|
744
|
+
node.replaceChild(p, childNode);
|
|
745
|
+
p.appendChild(childNode);
|
|
746
|
+
}
|
|
747
|
+
} else if (p !== null) {
|
|
748
|
+
while (p.lastChild && this._isWhitespace(p.lastChild)) p.lastChild.remove();
|
|
749
|
+
p = null;
|
|
750
|
+
}
|
|
751
|
+
childNode = nextSibling;
|
|
752
|
+
}
|
|
753
|
+
if (this._hasSingleTagInsideElement(node, "P") && this._getLinkDensity(node) < .25) {
|
|
754
|
+
var newNode = node.children[0];
|
|
755
|
+
node.parentNode.replaceChild(newNode, node);
|
|
756
|
+
node = newNode;
|
|
757
|
+
elementsToScore.push(node);
|
|
758
|
+
} else if (!this._hasChildBlockElement(node)) {
|
|
759
|
+
node = this._setNodeTag(node, "P");
|
|
760
|
+
elementsToScore.push(node);
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
node = this._getNextNode(node);
|
|
764
|
+
}
|
|
765
|
+
/**
|
|
766
|
+
* Loop through all paragraphs, and assign a score to them based on how content-y they look.
|
|
767
|
+
* Then add their score to their parent node.
|
|
768
|
+
*
|
|
769
|
+
* A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
|
|
770
|
+
**/
|
|
771
|
+
var candidates = [];
|
|
772
|
+
this._forEachNode(elementsToScore, function(elementToScore) {
|
|
773
|
+
if (!elementToScore.parentNode || typeof elementToScore.parentNode.tagName === "undefined") return;
|
|
774
|
+
var innerText = this._getInnerText(elementToScore);
|
|
775
|
+
if (innerText.length < 25) return;
|
|
776
|
+
var ancestors = this._getNodeAncestors(elementToScore, 5);
|
|
777
|
+
if (ancestors.length === 0) return;
|
|
778
|
+
var contentScore = 0;
|
|
779
|
+
contentScore += 1;
|
|
780
|
+
contentScore += innerText.split(this.REGEXPS.commas).length;
|
|
781
|
+
contentScore += Math.min(Math.floor(innerText.length / 100), 3);
|
|
782
|
+
this._forEachNode(ancestors, function(ancestor, level) {
|
|
783
|
+
if (!ancestor.tagName || !ancestor.parentNode || typeof ancestor.parentNode.tagName === "undefined") return;
|
|
784
|
+
if (typeof ancestor.readability === "undefined") {
|
|
785
|
+
this._initializeNode(ancestor);
|
|
786
|
+
candidates.push(ancestor);
|
|
787
|
+
}
|
|
788
|
+
if (level === 0) var scoreDivider = 1;
|
|
789
|
+
else if (level === 1) scoreDivider = 2;
|
|
790
|
+
else scoreDivider = level * 3;
|
|
791
|
+
ancestor.readability.contentScore += contentScore / scoreDivider;
|
|
792
|
+
});
|
|
793
|
+
});
|
|
794
|
+
var topCandidates = [];
|
|
795
|
+
for (var c = 0, cl = candidates.length; c < cl; c += 1) {
|
|
796
|
+
var candidate = candidates[c];
|
|
797
|
+
var candidateScore = candidate.readability.contentScore * (1 - this._getLinkDensity(candidate));
|
|
798
|
+
candidate.readability.contentScore = candidateScore;
|
|
799
|
+
this.log("Candidate:", candidate, "with score " + candidateScore);
|
|
800
|
+
for (var t = 0; t < this._nbTopCandidates; t++) {
|
|
801
|
+
var aTopCandidate = topCandidates[t];
|
|
802
|
+
if (!aTopCandidate || candidateScore > aTopCandidate.readability.contentScore) {
|
|
803
|
+
topCandidates.splice(t, 0, candidate);
|
|
804
|
+
if (topCandidates.length > this._nbTopCandidates) topCandidates.pop();
|
|
805
|
+
break;
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
var topCandidate = topCandidates[0] || null;
|
|
810
|
+
var neededToCreateTopCandidate = false;
|
|
811
|
+
var parentOfTopCandidate;
|
|
812
|
+
if (topCandidate === null || topCandidate.tagName === "BODY") {
|
|
813
|
+
topCandidate = doc.createElement("DIV");
|
|
814
|
+
neededToCreateTopCandidate = true;
|
|
815
|
+
while (page.firstChild) {
|
|
816
|
+
this.log("Moving child out:", page.firstChild);
|
|
817
|
+
topCandidate.appendChild(page.firstChild);
|
|
818
|
+
}
|
|
819
|
+
page.appendChild(topCandidate);
|
|
820
|
+
this._initializeNode(topCandidate);
|
|
821
|
+
} else if (topCandidate) {
|
|
822
|
+
var alternativeCandidateAncestors = [];
|
|
823
|
+
for (var i = 1; i < topCandidates.length; i++) if (topCandidates[i].readability.contentScore / topCandidate.readability.contentScore >= .75) alternativeCandidateAncestors.push(this._getNodeAncestors(topCandidates[i]));
|
|
824
|
+
var MINIMUM_TOPCANDIDATES = 3;
|
|
825
|
+
if (alternativeCandidateAncestors.length >= MINIMUM_TOPCANDIDATES) {
|
|
826
|
+
parentOfTopCandidate = topCandidate.parentNode;
|
|
827
|
+
while (parentOfTopCandidate.tagName !== "BODY") {
|
|
828
|
+
var listsContainingThisAncestor = 0;
|
|
829
|
+
for (var ancestorIndex = 0; ancestorIndex < alternativeCandidateAncestors.length && listsContainingThisAncestor < MINIMUM_TOPCANDIDATES; ancestorIndex++) listsContainingThisAncestor += Number(alternativeCandidateAncestors[ancestorIndex].includes(parentOfTopCandidate));
|
|
830
|
+
if (listsContainingThisAncestor >= MINIMUM_TOPCANDIDATES) {
|
|
831
|
+
topCandidate = parentOfTopCandidate;
|
|
832
|
+
break;
|
|
833
|
+
}
|
|
834
|
+
parentOfTopCandidate = parentOfTopCandidate.parentNode;
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
if (!topCandidate.readability) this._initializeNode(topCandidate);
|
|
838
|
+
parentOfTopCandidate = topCandidate.parentNode;
|
|
839
|
+
var lastScore = topCandidate.readability.contentScore;
|
|
840
|
+
var scoreThreshold = lastScore / 3;
|
|
841
|
+
while (parentOfTopCandidate.tagName !== "BODY") {
|
|
842
|
+
if (!parentOfTopCandidate.readability) {
|
|
843
|
+
parentOfTopCandidate = parentOfTopCandidate.parentNode;
|
|
844
|
+
continue;
|
|
845
|
+
}
|
|
846
|
+
var parentScore = parentOfTopCandidate.readability.contentScore;
|
|
847
|
+
if (parentScore < scoreThreshold) break;
|
|
848
|
+
if (parentScore > lastScore) {
|
|
849
|
+
topCandidate = parentOfTopCandidate;
|
|
850
|
+
break;
|
|
851
|
+
}
|
|
852
|
+
lastScore = parentOfTopCandidate.readability.contentScore;
|
|
853
|
+
parentOfTopCandidate = parentOfTopCandidate.parentNode;
|
|
854
|
+
}
|
|
855
|
+
parentOfTopCandidate = topCandidate.parentNode;
|
|
856
|
+
while (parentOfTopCandidate.tagName != "BODY" && parentOfTopCandidate.children.length == 1) {
|
|
857
|
+
topCandidate = parentOfTopCandidate;
|
|
858
|
+
parentOfTopCandidate = topCandidate.parentNode;
|
|
859
|
+
}
|
|
860
|
+
if (!topCandidate.readability) this._initializeNode(topCandidate);
|
|
861
|
+
}
|
|
862
|
+
var articleContent = doc.createElement("DIV");
|
|
863
|
+
if (isPaging) articleContent.id = "readability-content";
|
|
864
|
+
var siblingScoreThreshold = Math.max(10, topCandidate.readability.contentScore * .2);
|
|
865
|
+
parentOfTopCandidate = topCandidate.parentNode;
|
|
866
|
+
var siblings = parentOfTopCandidate.children;
|
|
867
|
+
for (var s = 0, sl = siblings.length; s < sl; s++) {
|
|
868
|
+
var sibling = siblings[s];
|
|
869
|
+
var append = false;
|
|
870
|
+
this.log("Looking at sibling node:", sibling, sibling.readability ? "with score " + sibling.readability.contentScore : "");
|
|
871
|
+
this.log("Sibling has score", sibling.readability ? sibling.readability.contentScore : "Unknown");
|
|
872
|
+
if (sibling === topCandidate) append = true;
|
|
873
|
+
else {
|
|
874
|
+
var contentBonus = 0;
|
|
875
|
+
if (sibling.className === topCandidate.className && topCandidate.className !== "") contentBonus += topCandidate.readability.contentScore * .2;
|
|
876
|
+
if (sibling.readability && sibling.readability.contentScore + contentBonus >= siblingScoreThreshold) append = true;
|
|
877
|
+
else if (sibling.nodeName === "P") {
|
|
878
|
+
var linkDensity = this._getLinkDensity(sibling);
|
|
879
|
+
var nodeContent = this._getInnerText(sibling);
|
|
880
|
+
var nodeLength = nodeContent.length;
|
|
881
|
+
if (nodeLength > 80 && linkDensity < .25) append = true;
|
|
882
|
+
else if (nodeLength < 80 && nodeLength > 0 && linkDensity === 0 && nodeContent.search(/\.( |$)/) !== -1) append = true;
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
if (append) {
|
|
886
|
+
this.log("Appending node:", sibling);
|
|
887
|
+
if (!this.ALTER_TO_DIV_EXCEPTIONS.includes(sibling.nodeName)) {
|
|
888
|
+
this.log("Altering sibling:", sibling, "to div.");
|
|
889
|
+
sibling = this._setNodeTag(sibling, "DIV");
|
|
890
|
+
}
|
|
891
|
+
articleContent.appendChild(sibling);
|
|
892
|
+
siblings = parentOfTopCandidate.children;
|
|
893
|
+
s -= 1;
|
|
894
|
+
sl -= 1;
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
if (this._debug) this.log("Article content pre-prep: " + articleContent.innerHTML);
|
|
898
|
+
this._prepArticle(articleContent);
|
|
899
|
+
if (this._debug) this.log("Article content post-prep: " + articleContent.innerHTML);
|
|
900
|
+
if (neededToCreateTopCandidate) {
|
|
901
|
+
topCandidate.id = "readability-page-1";
|
|
902
|
+
topCandidate.className = "page";
|
|
903
|
+
} else {
|
|
904
|
+
var div = doc.createElement("DIV");
|
|
905
|
+
div.id = "readability-page-1";
|
|
906
|
+
div.className = "page";
|
|
907
|
+
while (articleContent.firstChild) div.appendChild(articleContent.firstChild);
|
|
908
|
+
articleContent.appendChild(div);
|
|
909
|
+
}
|
|
910
|
+
if (this._debug) this.log("Article content after paging: " + articleContent.innerHTML);
|
|
911
|
+
var parseSuccessful = true;
|
|
912
|
+
var textLength = this._getInnerText(articleContent, true).length;
|
|
913
|
+
if (textLength < this._charThreshold) {
|
|
914
|
+
parseSuccessful = false;
|
|
915
|
+
page.innerHTML = pageCacheHtml;
|
|
916
|
+
this._attempts.push({
|
|
917
|
+
articleContent,
|
|
918
|
+
textLength
|
|
919
|
+
});
|
|
920
|
+
if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) this._removeFlag(this.FLAG_STRIP_UNLIKELYS);
|
|
921
|
+
else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) this._removeFlag(this.FLAG_WEIGHT_CLASSES);
|
|
922
|
+
else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY);
|
|
923
|
+
else {
|
|
924
|
+
this._attempts.sort(function(a, b) {
|
|
925
|
+
return b.textLength - a.textLength;
|
|
926
|
+
});
|
|
927
|
+
if (!this._attempts[0].textLength) return null;
|
|
928
|
+
articleContent = this._attempts[0].articleContent;
|
|
929
|
+
parseSuccessful = true;
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
if (parseSuccessful) {
|
|
933
|
+
var ancestors = [parentOfTopCandidate, topCandidate].concat(this._getNodeAncestors(parentOfTopCandidate));
|
|
934
|
+
this._someNode(ancestors, function(ancestor) {
|
|
935
|
+
if (!ancestor.tagName) return false;
|
|
936
|
+
var articleDir = ancestor.getAttribute("dir");
|
|
937
|
+
if (articleDir) {
|
|
938
|
+
this._articleDir = articleDir;
|
|
939
|
+
return true;
|
|
940
|
+
}
|
|
941
|
+
return false;
|
|
942
|
+
});
|
|
943
|
+
return articleContent;
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
},
|
|
947
|
+
/**
|
|
948
|
+
* Converts some of the common HTML entities in string to their corresponding characters.
|
|
949
|
+
*
|
|
950
|
+
* @param str {string} - a string to unescape.
|
|
951
|
+
* @return string without HTML entity.
|
|
952
|
+
*/
|
|
953
|
+
_unescapeHtmlEntities(str) {
|
|
954
|
+
if (!str) return str;
|
|
955
|
+
var htmlEscapeMap = this.HTML_ESCAPE_MAP;
|
|
956
|
+
return str.replace(/&(quot|amp|apos|lt|gt);/g, function(_, tag) {
|
|
957
|
+
return htmlEscapeMap[tag];
|
|
958
|
+
}).replace(/&#(?:x([0-9a-f]+)|([0-9]+));/gi, function(_, hex, numStr) {
|
|
959
|
+
var num = parseInt(hex || numStr, hex ? 16 : 10);
|
|
960
|
+
if (num == 0 || num > 1114111 || num >= 55296 && num <= 57343) num = 65533;
|
|
961
|
+
return String.fromCodePoint(num);
|
|
962
|
+
});
|
|
963
|
+
},
|
|
964
|
+
/**
|
|
965
|
+
* Try to extract metadata from JSON-LD object.
|
|
966
|
+
* For now, only Schema.org objects of type Article or its subtypes are supported.
|
|
967
|
+
* @return Object with any metadata that could be extracted (possibly none)
|
|
968
|
+
*/
|
|
969
|
+
_getJSONLD(doc) {
|
|
970
|
+
var scripts = this._getAllNodesWithTag(doc, ["script"]);
|
|
971
|
+
var metadata;
|
|
972
|
+
this._forEachNode(scripts, function(jsonLdElement) {
|
|
973
|
+
if (!metadata && jsonLdElement.getAttribute("type") === "application/ld+json") try {
|
|
974
|
+
var content = jsonLdElement.textContent.replace(/^\s*<!\[CDATA\[|\]\]>\s*$/g, "");
|
|
975
|
+
var parsed = JSON.parse(content);
|
|
976
|
+
if (Array.isArray(parsed)) {
|
|
977
|
+
parsed = parsed.find((it) => {
|
|
978
|
+
return it["@type"] && it["@type"].match(this.REGEXPS.jsonLdArticleTypes);
|
|
979
|
+
});
|
|
980
|
+
if (!parsed) return;
|
|
981
|
+
}
|
|
982
|
+
var schemaDotOrgRegex = /^https?\:\/\/schema\.org\/?$/;
|
|
983
|
+
if (!(typeof parsed["@context"] === "string" && parsed["@context"].match(schemaDotOrgRegex) || typeof parsed["@context"] === "object" && typeof parsed["@context"]["@vocab"] == "string" && parsed["@context"]["@vocab"].match(schemaDotOrgRegex))) return;
|
|
984
|
+
if (!parsed["@type"] && Array.isArray(parsed["@graph"])) parsed = parsed["@graph"].find((it) => {
|
|
985
|
+
return (it["@type"] || "").match(this.REGEXPS.jsonLdArticleTypes);
|
|
986
|
+
});
|
|
987
|
+
if (!parsed || !parsed["@type"] || !parsed["@type"].match(this.REGEXPS.jsonLdArticleTypes)) return;
|
|
988
|
+
metadata = {};
|
|
989
|
+
if (typeof parsed.name === "string" && typeof parsed.headline === "string" && parsed.name !== parsed.headline) {
|
|
990
|
+
var title = this._getArticleTitle();
|
|
991
|
+
var nameMatches = this._textSimilarity(parsed.name, title) > .75;
|
|
992
|
+
if (this._textSimilarity(parsed.headline, title) > .75 && !nameMatches) metadata.title = parsed.headline;
|
|
993
|
+
else metadata.title = parsed.name;
|
|
994
|
+
} else if (typeof parsed.name === "string") metadata.title = parsed.name.trim();
|
|
995
|
+
else if (typeof parsed.headline === "string") metadata.title = parsed.headline.trim();
|
|
996
|
+
if (parsed.author) {
|
|
997
|
+
if (typeof parsed.author.name === "string") metadata.byline = parsed.author.name.trim();
|
|
998
|
+
else if (Array.isArray(parsed.author) && parsed.author[0] && typeof parsed.author[0].name === "string") metadata.byline = parsed.author.filter(function(author) {
|
|
999
|
+
return author && typeof author.name === "string";
|
|
1000
|
+
}).map(function(author) {
|
|
1001
|
+
return author.name.trim();
|
|
1002
|
+
}).join(", ");
|
|
1003
|
+
}
|
|
1004
|
+
if (typeof parsed.description === "string") metadata.excerpt = parsed.description.trim();
|
|
1005
|
+
if (parsed.publisher && typeof parsed.publisher.name === "string") metadata.siteName = parsed.publisher.name.trim();
|
|
1006
|
+
if (typeof parsed.datePublished === "string") metadata.datePublished = parsed.datePublished.trim();
|
|
1007
|
+
} catch (err) {
|
|
1008
|
+
this.log(err.message);
|
|
1009
|
+
}
|
|
1010
|
+
});
|
|
1011
|
+
return metadata ? metadata : {};
|
|
1012
|
+
},
|
|
1013
|
+
/**
|
|
1014
|
+
* Attempts to get excerpt and byline metadata for the article.
|
|
1015
|
+
*
|
|
1016
|
+
* @param {Object} jsonld — object containing any metadata that
|
|
1017
|
+
* could be extracted from JSON-LD object.
|
|
1018
|
+
*
|
|
1019
|
+
* @return Object with optional "excerpt" and "byline" properties
|
|
1020
|
+
*/
|
|
1021
|
+
_getArticleMetadata(jsonld) {
|
|
1022
|
+
var metadata = {};
|
|
1023
|
+
var values = {};
|
|
1024
|
+
var metaElements = this._doc.getElementsByTagName("meta");
|
|
1025
|
+
var propertyPattern = /\s*(article|dc|dcterm|og|twitter)\s*:\s*(author|creator|description|published_time|title|site_name)\s*/gi;
|
|
1026
|
+
var namePattern = /^\s*(?:(dc|dcterm|og|twitter|parsely|weibo:(article|webpage))\s*[-\.:]\s*)?(author|creator|pub-date|description|title|site_name)\s*$/i;
|
|
1027
|
+
this._forEachNode(metaElements, function(element) {
|
|
1028
|
+
var elementName = element.getAttribute("name");
|
|
1029
|
+
var elementProperty = element.getAttribute("property");
|
|
1030
|
+
var content = element.getAttribute("content");
|
|
1031
|
+
if (!content) return;
|
|
1032
|
+
var matches = null;
|
|
1033
|
+
var name = null;
|
|
1034
|
+
if (elementProperty) {
|
|
1035
|
+
matches = elementProperty.match(propertyPattern);
|
|
1036
|
+
if (matches) {
|
|
1037
|
+
name = matches[0].toLowerCase().replace(/\s/g, "");
|
|
1038
|
+
values[name] = content.trim();
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
if (!matches && elementName && namePattern.test(elementName)) {
|
|
1042
|
+
name = elementName;
|
|
1043
|
+
if (content) {
|
|
1044
|
+
name = name.toLowerCase().replace(/\s/g, "").replace(/\./g, ":");
|
|
1045
|
+
values[name] = content.trim();
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
});
|
|
1049
|
+
metadata.title = jsonld.title || values["dc:title"] || values["dcterm:title"] || values["og:title"] || values["weibo:article:title"] || values["weibo:webpage:title"] || values.title || values["twitter:title"] || values["parsely-title"];
|
|
1050
|
+
if (!metadata.title) metadata.title = this._getArticleTitle();
|
|
1051
|
+
const articleAuthor = typeof values["article:author"] === "string" && !this._isUrl(values["article:author"]) ? values["article:author"] : void 0;
|
|
1052
|
+
metadata.byline = jsonld.byline || values["dc:creator"] || values["dcterm:creator"] || values.author || values["parsely-author"] || articleAuthor;
|
|
1053
|
+
metadata.excerpt = jsonld.excerpt || values["dc:description"] || values["dcterm:description"] || values["og:description"] || values["weibo:article:description"] || values["weibo:webpage:description"] || values.description || values["twitter:description"];
|
|
1054
|
+
metadata.siteName = jsonld.siteName || values["og:site_name"];
|
|
1055
|
+
metadata.publishedTime = jsonld.datePublished || values["article:published_time"] || values["parsely-pub-date"] || null;
|
|
1056
|
+
metadata.title = this._unescapeHtmlEntities(metadata.title);
|
|
1057
|
+
metadata.byline = this._unescapeHtmlEntities(metadata.byline);
|
|
1058
|
+
metadata.excerpt = this._unescapeHtmlEntities(metadata.excerpt);
|
|
1059
|
+
metadata.siteName = this._unescapeHtmlEntities(metadata.siteName);
|
|
1060
|
+
metadata.publishedTime = this._unescapeHtmlEntities(metadata.publishedTime);
|
|
1061
|
+
return metadata;
|
|
1062
|
+
},
|
|
1063
|
+
/**
|
|
1064
|
+
* Check if node is image, or if node contains exactly only one image
|
|
1065
|
+
* whether as a direct child or as its descendants.
|
|
1066
|
+
*
|
|
1067
|
+
* @param Element
|
|
1068
|
+
**/
|
|
1069
|
+
_isSingleImage(node) {
|
|
1070
|
+
while (node) {
|
|
1071
|
+
if (node.tagName === "IMG") return true;
|
|
1072
|
+
if (node.children.length !== 1 || node.textContent.trim() !== "") return false;
|
|
1073
|
+
node = node.children[0];
|
|
1074
|
+
}
|
|
1075
|
+
return false;
|
|
1076
|
+
},
|
|
1077
|
+
/**
|
|
1078
|
+
* Find all <noscript> that are located after <img> nodes, and which contain only one
|
|
1079
|
+
* <img> element. Replace the first image with the image from inside the <noscript> tag,
|
|
1080
|
+
* and remove the <noscript> tag. This improves the quality of the images we use on
|
|
1081
|
+
* some sites (e.g. Medium).
|
|
1082
|
+
*
|
|
1083
|
+
* @param Element
|
|
1084
|
+
**/
|
|
1085
|
+
_unwrapNoscriptImages(doc) {
|
|
1086
|
+
var imgs = Array.from(doc.getElementsByTagName("img"));
|
|
1087
|
+
this._forEachNode(imgs, function(img) {
|
|
1088
|
+
for (var i = 0; i < img.attributes.length; i++) {
|
|
1089
|
+
var attr = img.attributes[i];
|
|
1090
|
+
switch (attr.name) {
|
|
1091
|
+
case "src":
|
|
1092
|
+
case "srcset":
|
|
1093
|
+
case "data-src":
|
|
1094
|
+
case "data-srcset": return;
|
|
1095
|
+
}
|
|
1096
|
+
if (/\.(jpg|jpeg|png|webp)/i.test(attr.value)) return;
|
|
1097
|
+
}
|
|
1098
|
+
img.remove();
|
|
1099
|
+
});
|
|
1100
|
+
var noscripts = Array.from(doc.getElementsByTagName("noscript"));
|
|
1101
|
+
this._forEachNode(noscripts, function(noscript) {
|
|
1102
|
+
if (!this._isSingleImage(noscript)) return;
|
|
1103
|
+
var tmp = doc.createElement("div");
|
|
1104
|
+
tmp.innerHTML = noscript.innerHTML;
|
|
1105
|
+
var prevElement = noscript.previousElementSibling;
|
|
1106
|
+
if (prevElement && this._isSingleImage(prevElement)) {
|
|
1107
|
+
var prevImg = prevElement;
|
|
1108
|
+
if (prevImg.tagName !== "IMG") prevImg = prevElement.getElementsByTagName("img")[0];
|
|
1109
|
+
var newImg = tmp.getElementsByTagName("img")[0];
|
|
1110
|
+
for (var i = 0; i < prevImg.attributes.length; i++) {
|
|
1111
|
+
var attr = prevImg.attributes[i];
|
|
1112
|
+
if (attr.value === "") continue;
|
|
1113
|
+
if (attr.name === "src" || attr.name === "srcset" || /\.(jpg|jpeg|png|webp)/i.test(attr.value)) {
|
|
1114
|
+
if (newImg.getAttribute(attr.name) === attr.value) continue;
|
|
1115
|
+
var attrName = attr.name;
|
|
1116
|
+
if (newImg.hasAttribute(attrName)) attrName = "data-old-" + attrName;
|
|
1117
|
+
newImg.setAttribute(attrName, attr.value);
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
noscript.parentNode.replaceChild(tmp.firstElementChild, prevElement);
|
|
1121
|
+
}
|
|
1122
|
+
});
|
|
1123
|
+
},
|
|
1124
|
+
/**
|
|
1125
|
+
* Removes script tags from the document.
|
|
1126
|
+
*
|
|
1127
|
+
* @param Element
|
|
1128
|
+
**/
|
|
1129
|
+
_removeScripts(doc) {
|
|
1130
|
+
this._removeNodes(this._getAllNodesWithTag(doc, ["script", "noscript"]));
|
|
1131
|
+
},
|
|
1132
|
+
/**
|
|
1133
|
+
* Check if this node has only whitespace and a single element with given tag
|
|
1134
|
+
* Returns false if the DIV node contains non-empty text nodes
|
|
1135
|
+
* or if it contains no element with given tag or more than 1 element.
|
|
1136
|
+
*
|
|
1137
|
+
* @param Element
|
|
1138
|
+
* @param string tag of child element
|
|
1139
|
+
**/
|
|
1140
|
+
_hasSingleTagInsideElement(element, tag) {
|
|
1141
|
+
if (element.children.length != 1 || element.children[0].tagName !== tag) return false;
|
|
1142
|
+
return !this._someNode(element.childNodes, function(node) {
|
|
1143
|
+
return node.nodeType === this.TEXT_NODE && this.REGEXPS.hasContent.test(node.textContent);
|
|
1144
|
+
});
|
|
1145
|
+
},
|
|
1146
|
+
_isElementWithoutContent(node) {
|
|
1147
|
+
return node.nodeType === this.ELEMENT_NODE && !node.textContent.trim().length && (!node.children.length || node.children.length == node.getElementsByTagName("br").length + node.getElementsByTagName("hr").length);
|
|
1148
|
+
},
|
|
1149
|
+
/**
|
|
1150
|
+
* Determine whether element has any children block level elements.
|
|
1151
|
+
*
|
|
1152
|
+
* @param Element
|
|
1153
|
+
*/
|
|
1154
|
+
_hasChildBlockElement(element) {
|
|
1155
|
+
return this._someNode(element.childNodes, function(node) {
|
|
1156
|
+
return this.DIV_TO_P_ELEMS.has(node.tagName) || this._hasChildBlockElement(node);
|
|
1157
|
+
});
|
|
1158
|
+
},
|
|
1159
|
+
/***
|
|
1160
|
+
* Determine if a node qualifies as phrasing content.
|
|
1161
|
+
* https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/Content_categories#Phrasing_content
|
|
1162
|
+
**/
|
|
1163
|
+
_isPhrasingContent(node) {
|
|
1164
|
+
return node.nodeType === this.TEXT_NODE || this.PHRASING_ELEMS.includes(node.tagName) || (node.tagName === "A" || node.tagName === "DEL" || node.tagName === "INS") && this._everyNode(node.childNodes, this._isPhrasingContent);
|
|
1165
|
+
},
|
|
1166
|
+
_isWhitespace(node) {
|
|
1167
|
+
return node.nodeType === this.TEXT_NODE && node.textContent.trim().length === 0 || node.nodeType === this.ELEMENT_NODE && node.tagName === "BR";
|
|
1168
|
+
},
|
|
1169
|
+
/**
|
|
1170
|
+
* Get the inner text of a node - cross browser compatibly.
|
|
1171
|
+
* This also strips out any excess whitespace to be found.
|
|
1172
|
+
*
|
|
1173
|
+
* @param Element
|
|
1174
|
+
* @param Boolean normalizeSpaces (default: true)
|
|
1175
|
+
* @return string
|
|
1176
|
+
**/
|
|
1177
|
+
_getInnerText(e, normalizeSpaces) {
|
|
1178
|
+
normalizeSpaces = typeof normalizeSpaces === "undefined" ? true : normalizeSpaces;
|
|
1179
|
+
var textContent = e.textContent.trim();
|
|
1180
|
+
if (normalizeSpaces) return textContent.replace(this.REGEXPS.normalize, " ");
|
|
1181
|
+
return textContent;
|
|
1182
|
+
},
|
|
1183
|
+
/**
|
|
1184
|
+
* Get the number of times a string s appears in the node e.
|
|
1185
|
+
*
|
|
1186
|
+
* @param Element
|
|
1187
|
+
* @param string - what to split on. Default is ","
|
|
1188
|
+
* @return number (integer)
|
|
1189
|
+
**/
|
|
1190
|
+
_getCharCount(e, s) {
|
|
1191
|
+
s = s || ",";
|
|
1192
|
+
return this._getInnerText(e).split(s).length - 1;
|
|
1193
|
+
},
|
|
1194
|
+
/**
|
|
1195
|
+
* Remove the style attribute on every e and under.
|
|
1196
|
+
* TODO: Test if getElementsByTagName(*) is faster.
|
|
1197
|
+
*
|
|
1198
|
+
* @param Element
|
|
1199
|
+
* @return void
|
|
1200
|
+
**/
|
|
1201
|
+
_cleanStyles(e) {
|
|
1202
|
+
if (!e || e.tagName.toLowerCase() === "svg") return;
|
|
1203
|
+
for (var i = 0; i < this.PRESENTATIONAL_ATTRIBUTES.length; i++) e.removeAttribute(this.PRESENTATIONAL_ATTRIBUTES[i]);
|
|
1204
|
+
if (this.DEPRECATED_SIZE_ATTRIBUTE_ELEMS.includes(e.tagName)) {
|
|
1205
|
+
e.removeAttribute("width");
|
|
1206
|
+
e.removeAttribute("height");
|
|
1207
|
+
}
|
|
1208
|
+
var cur = e.firstElementChild;
|
|
1209
|
+
while (cur !== null) {
|
|
1210
|
+
this._cleanStyles(cur);
|
|
1211
|
+
cur = cur.nextElementSibling;
|
|
1212
|
+
}
|
|
1213
|
+
},
|
|
1214
|
+
/**
|
|
1215
|
+
* Get the density of links as a percentage of the content
|
|
1216
|
+
* This is the amount of text that is inside a link divided by the total text in the node.
|
|
1217
|
+
*
|
|
1218
|
+
* @param Element
|
|
1219
|
+
* @return number (float)
|
|
1220
|
+
**/
|
|
1221
|
+
_getLinkDensity(element) {
|
|
1222
|
+
var textLength = this._getInnerText(element).length;
|
|
1223
|
+
if (textLength === 0) return 0;
|
|
1224
|
+
var linkLength = 0;
|
|
1225
|
+
this._forEachNode(element.getElementsByTagName("a"), function(linkNode) {
|
|
1226
|
+
var href = linkNode.getAttribute("href");
|
|
1227
|
+
var coefficient = href && this.REGEXPS.hashUrl.test(href) ? .3 : 1;
|
|
1228
|
+
linkLength += this._getInnerText(linkNode).length * coefficient;
|
|
1229
|
+
});
|
|
1230
|
+
return linkLength / textLength;
|
|
1231
|
+
},
|
|
1232
|
+
/**
|
|
1233
|
+
* Get an elements class/id weight. Uses regular expressions to tell if this
|
|
1234
|
+
* element looks good or bad.
|
|
1235
|
+
*
|
|
1236
|
+
* @param Element
|
|
1237
|
+
* @return number (Integer)
|
|
1238
|
+
**/
|
|
1239
|
+
_getClassWeight(e) {
|
|
1240
|
+
if (!this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) return 0;
|
|
1241
|
+
var weight = 0;
|
|
1242
|
+
if (typeof e.className === "string" && e.className !== "") {
|
|
1243
|
+
if (this.REGEXPS.negative.test(e.className)) weight -= 25;
|
|
1244
|
+
if (this.REGEXPS.positive.test(e.className)) weight += 25;
|
|
1245
|
+
}
|
|
1246
|
+
if (typeof e.id === "string" && e.id !== "") {
|
|
1247
|
+
if (this.REGEXPS.negative.test(e.id)) weight -= 25;
|
|
1248
|
+
if (this.REGEXPS.positive.test(e.id)) weight += 25;
|
|
1249
|
+
}
|
|
1250
|
+
return weight;
|
|
1251
|
+
},
|
|
1252
|
+
/**
|
|
1253
|
+
* Clean a node of all elements of type "tag".
|
|
1254
|
+
* (Unless it's a youtube/vimeo video. People love movies.)
|
|
1255
|
+
*
|
|
1256
|
+
* @param Element
|
|
1257
|
+
* @param string tag to clean
|
|
1258
|
+
* @return void
|
|
1259
|
+
**/
|
|
1260
|
+
_clean(e, tag) {
|
|
1261
|
+
var isEmbed = [
|
|
1262
|
+
"object",
|
|
1263
|
+
"embed",
|
|
1264
|
+
"iframe"
|
|
1265
|
+
].includes(tag);
|
|
1266
|
+
this._removeNodes(this._getAllNodesWithTag(e, [tag]), function(element) {
|
|
1267
|
+
if (isEmbed) {
|
|
1268
|
+
for (var i = 0; i < element.attributes.length; i++) if (this._allowedVideoRegex.test(element.attributes[i].value)) return false;
|
|
1269
|
+
if (element.tagName === "object" && this._allowedVideoRegex.test(element.innerHTML)) return false;
|
|
1270
|
+
}
|
|
1271
|
+
return true;
|
|
1272
|
+
});
|
|
1273
|
+
},
|
|
1274
|
+
/**
|
|
1275
|
+
* Check if a given node has one of its ancestor tag name matching the
|
|
1276
|
+
* provided one.
|
|
1277
|
+
* @param HTMLElement node
|
|
1278
|
+
* @param String tagName
|
|
1279
|
+
* @param Number maxDepth
|
|
1280
|
+
* @param Function filterFn a filter to invoke to determine whether this node 'counts'
|
|
1281
|
+
* @return Boolean
|
|
1282
|
+
*/
|
|
1283
|
+
_hasAncestorTag(node, tagName, maxDepth, filterFn) {
|
|
1284
|
+
maxDepth = maxDepth || 3;
|
|
1285
|
+
tagName = tagName.toUpperCase();
|
|
1286
|
+
var depth = 0;
|
|
1287
|
+
while (node.parentNode) {
|
|
1288
|
+
if (maxDepth > 0 && depth > maxDepth) return false;
|
|
1289
|
+
if (node.parentNode.tagName === tagName && (!filterFn || filterFn(node.parentNode))) return true;
|
|
1290
|
+
node = node.parentNode;
|
|
1291
|
+
depth++;
|
|
1292
|
+
}
|
|
1293
|
+
return false;
|
|
1294
|
+
},
|
|
1295
|
+
/**
|
|
1296
|
+
* Return an object indicating how many rows and columns this table has.
|
|
1297
|
+
*/
|
|
1298
|
+
_getRowAndColumnCount(table) {
|
|
1299
|
+
var rows = 0;
|
|
1300
|
+
var columns = 0;
|
|
1301
|
+
var trs = table.getElementsByTagName("tr");
|
|
1302
|
+
for (var i = 0; i < trs.length; i++) {
|
|
1303
|
+
var rowspan = trs[i].getAttribute("rowspan") || 0;
|
|
1304
|
+
if (rowspan) rowspan = parseInt(rowspan, 10);
|
|
1305
|
+
rows += rowspan || 1;
|
|
1306
|
+
var columnsInThisRow = 0;
|
|
1307
|
+
var cells = trs[i].getElementsByTagName("td");
|
|
1308
|
+
for (var j = 0; j < cells.length; j++) {
|
|
1309
|
+
var colspan = cells[j].getAttribute("colspan") || 0;
|
|
1310
|
+
if (colspan) colspan = parseInt(colspan, 10);
|
|
1311
|
+
columnsInThisRow += colspan || 1;
|
|
1312
|
+
}
|
|
1313
|
+
columns = Math.max(columns, columnsInThisRow);
|
|
1314
|
+
}
|
|
1315
|
+
return {
|
|
1316
|
+
rows,
|
|
1317
|
+
columns
|
|
1318
|
+
};
|
|
1319
|
+
},
|
|
1320
|
+
/**
|
|
1321
|
+
* Look for 'data' (as opposed to 'layout') tables, for which we use
|
|
1322
|
+
* similar checks as
|
|
1323
|
+
* https://searchfox.org/mozilla-central/rev/f82d5c549f046cb64ce5602bfd894b7ae807c8f8/accessible/generic/TableAccessible.cpp#19
|
|
1324
|
+
*/
|
|
1325
|
+
_markDataTables(root) {
|
|
1326
|
+
var tables = root.getElementsByTagName("table");
|
|
1327
|
+
for (var i = 0; i < tables.length; i++) {
|
|
1328
|
+
var table = tables[i];
|
|
1329
|
+
if (table.getAttribute("role") == "presentation") {
|
|
1330
|
+
table._readabilityDataTable = false;
|
|
1331
|
+
continue;
|
|
1332
|
+
}
|
|
1333
|
+
if (table.getAttribute("datatable") == "0") {
|
|
1334
|
+
table._readabilityDataTable = false;
|
|
1335
|
+
continue;
|
|
1336
|
+
}
|
|
1337
|
+
if (table.getAttribute("summary")) {
|
|
1338
|
+
table._readabilityDataTable = true;
|
|
1339
|
+
continue;
|
|
1340
|
+
}
|
|
1341
|
+
var caption = table.getElementsByTagName("caption")[0];
|
|
1342
|
+
if (caption && caption.childNodes.length) {
|
|
1343
|
+
table._readabilityDataTable = true;
|
|
1344
|
+
continue;
|
|
1345
|
+
}
|
|
1346
|
+
var dataTableDescendants = [
|
|
1347
|
+
"col",
|
|
1348
|
+
"colgroup",
|
|
1349
|
+
"tfoot",
|
|
1350
|
+
"thead",
|
|
1351
|
+
"th"
|
|
1352
|
+
];
|
|
1353
|
+
var descendantExists = function(tag) {
|
|
1354
|
+
return !!table.getElementsByTagName(tag)[0];
|
|
1355
|
+
};
|
|
1356
|
+
if (dataTableDescendants.some(descendantExists)) {
|
|
1357
|
+
this.log("Data table because found data-y descendant");
|
|
1358
|
+
table._readabilityDataTable = true;
|
|
1359
|
+
continue;
|
|
1360
|
+
}
|
|
1361
|
+
if (table.getElementsByTagName("table")[0]) {
|
|
1362
|
+
table._readabilityDataTable = false;
|
|
1363
|
+
continue;
|
|
1364
|
+
}
|
|
1365
|
+
var sizeInfo = this._getRowAndColumnCount(table);
|
|
1366
|
+
if (sizeInfo.columns == 1 || sizeInfo.rows == 1) {
|
|
1367
|
+
table._readabilityDataTable = false;
|
|
1368
|
+
continue;
|
|
1369
|
+
}
|
|
1370
|
+
if (sizeInfo.rows >= 10 || sizeInfo.columns > 4) {
|
|
1371
|
+
table._readabilityDataTable = true;
|
|
1372
|
+
continue;
|
|
1373
|
+
}
|
|
1374
|
+
table._readabilityDataTable = sizeInfo.rows * sizeInfo.columns > 10;
|
|
1375
|
+
}
|
|
1376
|
+
},
|
|
1377
|
+
_fixLazyImages(root) {
|
|
1378
|
+
this._forEachNode(this._getAllNodesWithTag(root, [
|
|
1379
|
+
"img",
|
|
1380
|
+
"picture",
|
|
1381
|
+
"figure"
|
|
1382
|
+
]), function(elem) {
|
|
1383
|
+
if (elem.src && this.REGEXPS.b64DataUrl.test(elem.src)) {
|
|
1384
|
+
var parts = this.REGEXPS.b64DataUrl.exec(elem.src);
|
|
1385
|
+
if (parts[1] === "image/svg+xml") return;
|
|
1386
|
+
var srcCouldBeRemoved = false;
|
|
1387
|
+
for (var i = 0; i < elem.attributes.length; i++) {
|
|
1388
|
+
var attr = elem.attributes[i];
|
|
1389
|
+
if (attr.name === "src") continue;
|
|
1390
|
+
if (/\.(jpg|jpeg|png|webp)/i.test(attr.value)) {
|
|
1391
|
+
srcCouldBeRemoved = true;
|
|
1392
|
+
break;
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
if (srcCouldBeRemoved) {
|
|
1396
|
+
var b64starts = parts[0].length;
|
|
1397
|
+
if (elem.src.length - b64starts < 133) elem.removeAttribute("src");
|
|
1398
|
+
}
|
|
1399
|
+
}
|
|
1400
|
+
if ((elem.src || elem.srcset && elem.srcset != "null") && !elem.className.toLowerCase().includes("lazy")) return;
|
|
1401
|
+
for (var j = 0; j < elem.attributes.length; j++) {
|
|
1402
|
+
attr = elem.attributes[j];
|
|
1403
|
+
if (attr.name === "src" || attr.name === "srcset" || attr.name === "alt") continue;
|
|
1404
|
+
var copyTo = null;
|
|
1405
|
+
if (/\.(jpg|jpeg|png|webp)\s+\d/.test(attr.value)) copyTo = "srcset";
|
|
1406
|
+
else if (/^\s*\S+\.(jpg|jpeg|png|webp)\S*\s*$/.test(attr.value)) copyTo = "src";
|
|
1407
|
+
if (copyTo) {
|
|
1408
|
+
if (elem.tagName === "IMG" || elem.tagName === "PICTURE") elem.setAttribute(copyTo, attr.value);
|
|
1409
|
+
else if (elem.tagName === "FIGURE" && !this._getAllNodesWithTag(elem, ["img", "picture"]).length) {
|
|
1410
|
+
var img = this._doc.createElement("img");
|
|
1411
|
+
img.setAttribute(copyTo, attr.value);
|
|
1412
|
+
elem.appendChild(img);
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
});
|
|
1417
|
+
},
|
|
1418
|
+
_getTextDensity(e, tags) {
|
|
1419
|
+
var textLength = this._getInnerText(e, true).length;
|
|
1420
|
+
if (textLength === 0) return 0;
|
|
1421
|
+
var childrenLength = 0;
|
|
1422
|
+
var children = this._getAllNodesWithTag(e, tags);
|
|
1423
|
+
this._forEachNode(children, (child) => childrenLength += this._getInnerText(child, true).length);
|
|
1424
|
+
return childrenLength / textLength;
|
|
1425
|
+
},
|
|
1426
|
+
/**
|
|
1427
|
+
* Clean an element of all tags of type "tag" if they look fishy.
|
|
1428
|
+
* "Fishy" is an algorithm based on content length, classnames, link density, number of images & embeds, etc.
|
|
1429
|
+
*
|
|
1430
|
+
* @return void
|
|
1431
|
+
**/
|
|
1432
|
+
_cleanConditionally(e, tag) {
|
|
1433
|
+
if (!this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) return;
|
|
1434
|
+
this._removeNodes(this._getAllNodesWithTag(e, [tag]), function(node) {
|
|
1435
|
+
var isDataTable = function(t) {
|
|
1436
|
+
return t._readabilityDataTable;
|
|
1437
|
+
};
|
|
1438
|
+
var isList = tag === "ul" || tag === "ol";
|
|
1439
|
+
if (!isList) {
|
|
1440
|
+
var listLength = 0;
|
|
1441
|
+
var listNodes = this._getAllNodesWithTag(node, ["ul", "ol"]);
|
|
1442
|
+
this._forEachNode(listNodes, (list) => listLength += this._getInnerText(list).length);
|
|
1443
|
+
isList = listLength / this._getInnerText(node).length > .9;
|
|
1444
|
+
}
|
|
1445
|
+
if (tag === "table" && isDataTable(node)) return false;
|
|
1446
|
+
if (this._hasAncestorTag(node, "table", -1, isDataTable)) return false;
|
|
1447
|
+
if (this._hasAncestorTag(node, "code")) return false;
|
|
1448
|
+
if ([...node.getElementsByTagName("table")].some((tbl) => tbl._readabilityDataTable)) return false;
|
|
1449
|
+
var weight = this._getClassWeight(node);
|
|
1450
|
+
this.log("Cleaning Conditionally", node);
|
|
1451
|
+
if (weight + 0 < 0) return true;
|
|
1452
|
+
if (this._getCharCount(node, ",") < 10) {
|
|
1453
|
+
var p = node.getElementsByTagName("p").length;
|
|
1454
|
+
var img = node.getElementsByTagName("img").length;
|
|
1455
|
+
var li = node.getElementsByTagName("li").length - 100;
|
|
1456
|
+
var input = node.getElementsByTagName("input").length;
|
|
1457
|
+
var headingDensity = this._getTextDensity(node, [
|
|
1458
|
+
"h1",
|
|
1459
|
+
"h2",
|
|
1460
|
+
"h3",
|
|
1461
|
+
"h4",
|
|
1462
|
+
"h5",
|
|
1463
|
+
"h6"
|
|
1464
|
+
]);
|
|
1465
|
+
var embedCount = 0;
|
|
1466
|
+
var embeds = this._getAllNodesWithTag(node, [
|
|
1467
|
+
"object",
|
|
1468
|
+
"embed",
|
|
1469
|
+
"iframe"
|
|
1470
|
+
]);
|
|
1471
|
+
for (var i = 0; i < embeds.length; i++) {
|
|
1472
|
+
for (var j = 0; j < embeds[i].attributes.length; j++) if (this._allowedVideoRegex.test(embeds[i].attributes[j].value)) return false;
|
|
1473
|
+
if (embeds[i].tagName === "object" && this._allowedVideoRegex.test(embeds[i].innerHTML)) return false;
|
|
1474
|
+
embedCount++;
|
|
1475
|
+
}
|
|
1476
|
+
var innerText = this._getInnerText(node);
|
|
1477
|
+
if (this.REGEXPS.adWords.test(innerText) || this.REGEXPS.loadingWords.test(innerText)) return true;
|
|
1478
|
+
var contentLength = innerText.length;
|
|
1479
|
+
var linkDensity = this._getLinkDensity(node);
|
|
1480
|
+
var textishTags = [
|
|
1481
|
+
"SPAN",
|
|
1482
|
+
"LI",
|
|
1483
|
+
"TD"
|
|
1484
|
+
].concat(Array.from(this.DIV_TO_P_ELEMS));
|
|
1485
|
+
var textDensity = this._getTextDensity(node, textishTags);
|
|
1486
|
+
var isFigureChild = this._hasAncestorTag(node, "figure");
|
|
1487
|
+
const shouldRemoveNode = () => {
|
|
1488
|
+
const errs = [];
|
|
1489
|
+
if (!isFigureChild && img > 1 && p / img < .5) errs.push(`Bad p to img ratio (img=${img}, p=${p})`);
|
|
1490
|
+
if (!isList && li > p) errs.push(`Too many li's outside of a list. (li=${li} > p=${p})`);
|
|
1491
|
+
if (input > Math.floor(p / 3)) errs.push(`Too many inputs per p. (input=${input}, p=${p})`);
|
|
1492
|
+
if (!isList && !isFigureChild && headingDensity < .9 && contentLength < 25 && (img === 0 || img > 2) && linkDensity > 0) errs.push(`Suspiciously short. (headingDensity=${headingDensity}, img=${img}, linkDensity=${linkDensity})`);
|
|
1493
|
+
if (!isList && weight < 25 && linkDensity > .2 + this._linkDensityModifier) errs.push(`Low weight and a little linky. (linkDensity=${linkDensity})`);
|
|
1494
|
+
if (weight >= 25 && linkDensity > .5 + this._linkDensityModifier) errs.push(`High weight and mostly links. (linkDensity=${linkDensity})`);
|
|
1495
|
+
if (embedCount === 1 && contentLength < 75 || embedCount > 1) errs.push(`Suspicious embed. (embedCount=${embedCount}, contentLength=${contentLength})`);
|
|
1496
|
+
if (img === 0 && textDensity === 0) errs.push(`No useful content. (img=${img}, textDensity=${textDensity})`);
|
|
1497
|
+
if (errs.length) {
|
|
1498
|
+
this.log("Checks failed", errs);
|
|
1499
|
+
return true;
|
|
1500
|
+
}
|
|
1501
|
+
return false;
|
|
1502
|
+
};
|
|
1503
|
+
var haveToRemove = shouldRemoveNode();
|
|
1504
|
+
if (isList && haveToRemove) {
|
|
1505
|
+
for (var x = 0; x < node.children.length; x++) if (node.children[x].children.length > 1) return haveToRemove;
|
|
1506
|
+
if (img == node.getElementsByTagName("li").length) return false;
|
|
1507
|
+
}
|
|
1508
|
+
return haveToRemove;
|
|
1509
|
+
}
|
|
1510
|
+
return false;
|
|
1511
|
+
});
|
|
1512
|
+
},
|
|
1513
|
+
/**
|
|
1514
|
+
* Clean out elements that match the specified conditions
|
|
1515
|
+
*
|
|
1516
|
+
* @param Element
|
|
1517
|
+
* @param Function determines whether a node should be removed
|
|
1518
|
+
* @return void
|
|
1519
|
+
**/
|
|
1520
|
+
_cleanMatchedNodes(e, filter) {
|
|
1521
|
+
var endOfSearchMarkerNode = this._getNextNode(e, true);
|
|
1522
|
+
var next = this._getNextNode(e);
|
|
1523
|
+
while (next && next != endOfSearchMarkerNode) if (filter.call(this, next, next.className + " " + next.id)) next = this._removeAndGetNext(next);
|
|
1524
|
+
else next = this._getNextNode(next);
|
|
1525
|
+
},
|
|
1526
|
+
/**
|
|
1527
|
+
* Clean out spurious headers from an Element.
|
|
1528
|
+
*
|
|
1529
|
+
* @param Element
|
|
1530
|
+
* @return void
|
|
1531
|
+
**/
|
|
1532
|
+
_cleanHeaders(e) {
|
|
1533
|
+
let headingNodes = this._getAllNodesWithTag(e, ["h1", "h2"]);
|
|
1534
|
+
this._removeNodes(headingNodes, function(node) {
|
|
1535
|
+
let shouldRemove = this._getClassWeight(node) < 0;
|
|
1536
|
+
if (shouldRemove) this.log("Removing header with low class weight:", node);
|
|
1537
|
+
return shouldRemove;
|
|
1538
|
+
});
|
|
1539
|
+
},
|
|
1540
|
+
/**
|
|
1541
|
+
* Check if this node is an H1 or H2 element whose content is mostly
|
|
1542
|
+
* the same as the article title.
|
|
1543
|
+
*
|
|
1544
|
+
* @param Element the node to check.
|
|
1545
|
+
* @return boolean indicating whether this is a title-like header.
|
|
1546
|
+
*/
|
|
1547
|
+
_headerDuplicatesTitle(node) {
|
|
1548
|
+
if (node.tagName != "H1" && node.tagName != "H2") return false;
|
|
1549
|
+
var heading = this._getInnerText(node, false);
|
|
1550
|
+
this.log("Evaluating similarity of header:", heading, this._articleTitle);
|
|
1551
|
+
return this._textSimilarity(this._articleTitle, heading) > .75;
|
|
1552
|
+
},
|
|
1553
|
+
_flagIsActive(flag) {
|
|
1554
|
+
return (this._flags & flag) > 0;
|
|
1555
|
+
},
|
|
1556
|
+
_removeFlag(flag) {
|
|
1557
|
+
this._flags = this._flags & ~flag;
|
|
1558
|
+
},
|
|
1559
|
+
_isProbablyVisible(node) {
|
|
1560
|
+
return (!node.style || node.style.display != "none") && (!node.style || node.style.visibility != "hidden") && !node.hasAttribute("hidden") && (!node.hasAttribute("aria-hidden") || node.getAttribute("aria-hidden") != "true" || node.className && node.className.includes && node.className.includes("fallback-image"));
|
|
1561
|
+
},
|
|
1562
|
+
/**
|
|
1563
|
+
* Runs readability.
|
|
1564
|
+
*
|
|
1565
|
+
* Workflow:
|
|
1566
|
+
* 1. Prep the document by removing script tags, css, etc.
|
|
1567
|
+
* 2. Build readability's DOM tree.
|
|
1568
|
+
* 3. Grab the article content from the current dom tree.
|
|
1569
|
+
* 4. Replace the current DOM tree with the new one.
|
|
1570
|
+
* 5. Read peacefully.
|
|
1571
|
+
*
|
|
1572
|
+
* @return void
|
|
1573
|
+
**/
|
|
1574
|
+
parse() {
|
|
1575
|
+
if (this._maxElemsToParse > 0) {
|
|
1576
|
+
var numTags = this._doc.getElementsByTagName("*").length;
|
|
1577
|
+
if (numTags > this._maxElemsToParse) throw new Error("Aborting parsing document; " + numTags + " elements found");
|
|
1578
|
+
}
|
|
1579
|
+
this._unwrapNoscriptImages(this._doc);
|
|
1580
|
+
var jsonLd = this._disableJSONLD ? {} : this._getJSONLD(this._doc);
|
|
1581
|
+
this._removeScripts(this._doc);
|
|
1582
|
+
this._prepDocument();
|
|
1583
|
+
var metadata = this._getArticleMetadata(jsonLd);
|
|
1584
|
+
this._metadata = metadata;
|
|
1585
|
+
this._articleTitle = metadata.title;
|
|
1586
|
+
var articleContent = this._grabArticle();
|
|
1587
|
+
if (!articleContent) return null;
|
|
1588
|
+
this.log("Grabbed: " + articleContent.innerHTML);
|
|
1589
|
+
this._postProcessContent(articleContent);
|
|
1590
|
+
if (!metadata.excerpt) {
|
|
1591
|
+
var paragraphs = articleContent.getElementsByTagName("p");
|
|
1592
|
+
if (paragraphs.length) metadata.excerpt = paragraphs[0].textContent.trim();
|
|
1593
|
+
}
|
|
1594
|
+
var textContent = articleContent.textContent;
|
|
1595
|
+
return {
|
|
1596
|
+
title: this._articleTitle,
|
|
1597
|
+
byline: metadata.byline || this._articleByline,
|
|
1598
|
+
dir: this._articleDir,
|
|
1599
|
+
lang: this._articleLang,
|
|
1600
|
+
content: this._serializer(articleContent),
|
|
1601
|
+
textContent,
|
|
1602
|
+
length: textContent.length,
|
|
1603
|
+
excerpt: metadata.excerpt,
|
|
1604
|
+
siteName: metadata.siteName || this._articleSiteName,
|
|
1605
|
+
publishedTime: metadata.publishedTime
|
|
1606
|
+
};
|
|
1607
|
+
}
|
|
1608
|
+
};
|
|
1609
|
+
if (typeof module === "object") module.exports = Readability;
|
|
1610
|
+
}));
|
|
1611
|
+
//#endregion
|
|
1612
|
+
//#region ../../node_modules/.pnpm/@mozilla+readability@0.6.0/node_modules/@mozilla/readability/Readability-readerable.js
|
|
1613
|
+
var require_Readability_readerable = /* @__PURE__ */ __commonJSMin(((exports, module) => {
|
|
1614
|
+
var REGEXPS = {
|
|
1615
|
+
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
|
|
1616
|
+
okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i
|
|
1617
|
+
};
|
|
1618
|
+
function isNodeVisible(node) {
|
|
1619
|
+
return (!node.style || node.style.display != "none") && !node.hasAttribute("hidden") && (!node.hasAttribute("aria-hidden") || node.getAttribute("aria-hidden") != "true" || node.className && node.className.includes && node.className.includes("fallback-image"));
|
|
1620
|
+
}
|
|
1621
|
+
/**
|
|
1622
|
+
* Decides whether or not the document is reader-able without parsing the whole thing.
|
|
1623
|
+
* @param {Object} options Configuration object.
|
|
1624
|
+
* @param {number} [options.minContentLength=140] The minimum node content length used to decide if the document is readerable.
|
|
1625
|
+
* @param {number} [options.minScore=20] The minumum cumulated 'score' used to determine if the document is readerable.
|
|
1626
|
+
* @param {Function} [options.visibilityChecker=isNodeVisible] The function used to determine if a node is visible.
|
|
1627
|
+
* @return {boolean} Whether or not we suspect Readability.parse() will suceeed at returning an article object.
|
|
1628
|
+
*/
|
|
1629
|
+
function isProbablyReaderable(doc, options = {}) {
|
|
1630
|
+
if (typeof options == "function") options = { visibilityChecker: options };
|
|
1631
|
+
options = Object.assign({
|
|
1632
|
+
minScore: 20,
|
|
1633
|
+
minContentLength: 140,
|
|
1634
|
+
visibilityChecker: isNodeVisible
|
|
1635
|
+
}, options);
|
|
1636
|
+
var nodes = doc.querySelectorAll("p, pre, article");
|
|
1637
|
+
var brNodes = doc.querySelectorAll("div > br");
|
|
1638
|
+
if (brNodes.length) {
|
|
1639
|
+
var set = new Set(nodes);
|
|
1640
|
+
[].forEach.call(brNodes, function(node) {
|
|
1641
|
+
set.add(node.parentNode);
|
|
1642
|
+
});
|
|
1643
|
+
nodes = Array.from(set);
|
|
1644
|
+
}
|
|
1645
|
+
var score = 0;
|
|
1646
|
+
return [].some.call(nodes, function(node) {
|
|
1647
|
+
if (!options.visibilityChecker(node)) return false;
|
|
1648
|
+
var matchString = node.className + " " + node.id;
|
|
1649
|
+
if (REGEXPS.unlikelyCandidates.test(matchString) && !REGEXPS.okMaybeItsACandidate.test(matchString)) return false;
|
|
1650
|
+
if (node.matches("li p")) return false;
|
|
1651
|
+
var textContentLength = node.textContent.trim().length;
|
|
1652
|
+
if (textContentLength < options.minContentLength) return false;
|
|
1653
|
+
score += Math.sqrt(textContentLength - options.minContentLength);
|
|
1654
|
+
if (score > options.minScore) return true;
|
|
1655
|
+
return false;
|
|
1656
|
+
});
|
|
1657
|
+
}
|
|
1658
|
+
if (typeof module === "object") module.exports = isProbablyReaderable;
|
|
1659
|
+
}));
|
|
1660
|
+
//#endregion
|
|
1661
|
+
//#region ../../node_modules/.pnpm/@mozilla+readability@0.6.0/node_modules/@mozilla/readability/index.js
|
|
1662
|
+
var require_readability = /* @__PURE__ */ __commonJSMin(((exports, module) => {
|
|
1663
|
+
module.exports = {
|
|
1664
|
+
Readability: require_Readability(),
|
|
1665
|
+
isProbablyReaderable: require_Readability_readerable()
|
|
1666
|
+
};
|
|
1667
|
+
}));
|
|
1668
|
+
//#endregion
|
|
1669
|
+
export default require_readability();
|
|
1670
|
+
export {};
|