@pablopunk/tau 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/auth-storage-Dk3CUY3q.js +1814 -0
  2. package/dist/bin.js +23 -0
  3. package/dist/cli-entry-DcDpQ30L.js +13 -0
  4. package/dist/client-C91-b2CL.js +1 -0
  5. package/dist/client-y2WkNuZj.js +7977 -0
  6. package/dist/dist-BFP19NOO.js +432 -0
  7. package/dist/dist-Bh1bQHZA.js +38890 -0
  8. package/dist/dist-Cq73_r65.js +3 -0
  9. package/dist/esm-CMGh_WhC.js +9446 -0
  10. package/dist/execute-BeGXhU1O.js +1757 -0
  11. package/dist/fake-claude-connectors-DSP9UC8d.js +1 -0
  12. package/dist/fake-claude-connectors-sDLeP10T.js +107 -0
  13. package/dist/fake-claude-sdk-CvqW_1iQ.js +571 -0
  14. package/dist/fake-claude-thread-script-2iHHwUZM.js +48 -0
  15. package/dist/last-exit-write-ksHeHGrC.js +9 -0
  16. package/dist/latest-version-di9hD7cY.js +106 -0
  17. package/dist/lib-DIX7q_HJ.js +1460 -0
  18. package/dist/machines-behind-BjGgmSs_.js +257 -0
  19. package/dist/main-D-Tg9b5Y.js +1610 -0
  20. package/dist/manager-B_WHbb1U.js +589 -0
  21. package/dist/names-cTrEXB6w.js +35 -0
  22. package/dist/node-BpwblPFP.js +56269 -0
  23. package/dist/pdfjs-nTcHjDPT.js +69856 -0
  24. package/dist/photon_rs-DBr7W_rK.js +4252 -0
  25. package/dist/promise.try-CAAOYvMP.js +1426 -0
  26. package/dist/readability-DsDNO63a.js +1670 -0
  27. package/dist/real-deps-f96UmLBz.js +89 -0
  28. package/dist/rolldown-runtime-1p2KPIdQ.js +44 -0
  29. package/dist/runtime-i-oEtgAl.js +2721 -0
  30. package/dist/sdk-missing-CHbVbY1G.js +10 -0
  31. package/dist/sdk-port-BGw9ORUR.js +215 -0
  32. package/dist/serve-Cmy0rUWr.js +2932 -0
  33. package/dist/serve-Dmzn5V0l.js +1 -0
  34. package/dist/service-command-C74K0saj.js +34 -0
  35. package/dist/service-command-DqcstJ8e.js +1 -0
  36. package/dist/shell-CQa4e7rx.js +643 -0
  37. package/dist/skills/builtin/tau/SKILL.md +51 -0
  38. package/dist/src-DO9LUlhi.js +44088 -0
  39. package/dist/streamableHttp-BBaRYV9h.js +1 -0
  40. package/dist/streamableHttp-DPAoMlJd.js +1931 -0
  41. package/dist/tool-DBZK2lci.js +14335 -0
  42. package/dist/tool-server-BjP37EJg.js +40 -0
  43. package/dist/turndown.cjs-Bo1xLg2S.js +15620 -0
  44. package/dist/turndown.cjs-jpUF50AR.js +3 -0
  45. package/dist/types-C4M3CRp2.js +7592 -0
  46. package/dist/undici-8cI97lnP.js +27110 -0
  47. package/dist/wait-for-lock-B1Qx8GWg.js +1 -0
  48. package/dist/wait-for-lock-CrSl2mi1.js +25 -0
  49. package/dist/watchdog.js +47 -0
  50. package/dist/web/dist/assets/addon-fit-DIOBYJe3.js +1 -0
  51. package/dist/web/dist/assets/geist-cyrillic-ext-wght-normal-DjL33-gN.woff2 +0 -0
  52. package/dist/web/dist/assets/geist-cyrillic-wght-normal-BEAKL7Jp.woff2 +0 -0
  53. package/dist/web/dist/assets/geist-latin-ext-wght-normal-DC-KSUi6.woff2 +0 -0
  54. package/dist/web/dist/assets/geist-latin-wght-normal-BgDaEnEv.woff2 +0 -0
  55. package/dist/web/dist/assets/geist-vietnamese-wght-normal-6IgcOCM7.woff2 +0 -0
  56. package/dist/web/dist/assets/index-BktLKZHR.js +108 -0
  57. package/dist/web/dist/assets/index-D_Aa3Oou.css +2 -0
  58. package/dist/web/dist/assets/jetbrains-mono-cyrillic-wght-normal-D73BlboJ.woff2 +0 -0
  59. package/dist/web/dist/assets/jetbrains-mono-greek-wght-normal-Bw9x6K1M.woff2 +0 -0
  60. package/dist/web/dist/assets/jetbrains-mono-latin-ext-wght-normal-DBQx-q_a.woff2 +0 -0
  61. package/dist/web/dist/assets/jetbrains-mono-latin-wght-normal-B9CIFXIH.woff2 +0 -0
  62. package/dist/web/dist/assets/jetbrains-mono-vietnamese-wght-normal-Bt-aOZkq.woff2 +0 -0
  63. package/dist/web/dist/assets/wght-DMLdVLkb.css +1 -0
  64. package/dist/web/dist/assets/xterm-BrP-ENHg.css +1 -0
  65. package/dist/web/dist/assets/xterm-CerOhKYV.js +16 -0
  66. package/dist/web/dist/favicon.svg +14 -0
  67. package/dist/web/dist/index.html +25 -0
  68. package/package.json +53 -4
  69. package/README.md +0 -3
@@ -0,0 +1,1670 @@
1
+ import { t as __commonJSMin } from "./rolldown-runtime-1p2KPIdQ.js";
2
+ //#region ../../node_modules/.pnpm/@mozilla+readability@0.6.0/node_modules/@mozilla/readability/Readability.js
3
+ var require_Readability = /* @__PURE__ */ __commonJSMin(((exports, module) => {
4
+ /**
5
+ * Public constructor.
6
+ * @param {HTMLDocument} doc The document to parse.
7
+ * @param {Object} options The options object.
8
+ */
9
+ function Readability(doc, options) {
10
+ if (options && options.documentElement) {
11
+ doc = options;
12
+ options = arguments[2];
13
+ } else if (!doc || !doc.documentElement) throw new Error("First argument to Readability constructor should be a document object.");
14
+ options = options || {};
15
+ this._doc = doc;
16
+ this._docJSDOMParser = this._doc.firstChild.__JSDOMParser__;
17
+ this._articleTitle = null;
18
+ this._articleByline = null;
19
+ this._articleDir = null;
20
+ this._articleSiteName = null;
21
+ this._attempts = [];
22
+ this._metadata = {};
23
+ this._debug = !!options.debug;
24
+ this._maxElemsToParse = options.maxElemsToParse || this.DEFAULT_MAX_ELEMS_TO_PARSE;
25
+ this._nbTopCandidates = options.nbTopCandidates || this.DEFAULT_N_TOP_CANDIDATES;
26
+ this._charThreshold = options.charThreshold || this.DEFAULT_CHAR_THRESHOLD;
27
+ this._classesToPreserve = this.CLASSES_TO_PRESERVE.concat(options.classesToPreserve || []);
28
+ this._keepClasses = !!options.keepClasses;
29
+ this._serializer = options.serializer || function(el) {
30
+ return el.innerHTML;
31
+ };
32
+ this._disableJSONLD = !!options.disableJSONLD;
33
+ this._allowedVideoRegex = options.allowedVideoRegex || this.REGEXPS.videos;
34
+ this._linkDensityModifier = options.linkDensityModifier || 0;
35
+ this._flags = this.FLAG_STRIP_UNLIKELYS | this.FLAG_WEIGHT_CLASSES | this.FLAG_CLEAN_CONDITIONALLY;
36
+ if (this._debug) {
37
+ let logNode = function(node) {
38
+ if (node.nodeType == node.TEXT_NODE) return `${node.nodeName} ("${node.textContent}")`;
39
+ let attrPairs = Array.from(node.attributes || [], function(attr) {
40
+ return `${attr.name}="${attr.value}"`;
41
+ }).join(" ");
42
+ return `<${node.localName} ${attrPairs}>`;
43
+ };
44
+ this.log = function() {
45
+ if (typeof console !== "undefined") {
46
+ let args = Array.from(arguments, (arg) => {
47
+ if (arg && arg.nodeType == this.ELEMENT_NODE) return logNode(arg);
48
+ return arg;
49
+ });
50
+ args.unshift("Reader: (Readability)");
51
+ console.log(...args);
52
+ } else if (typeof dump !== "undefined") {
53
+ var msg = Array.prototype.map.call(arguments, function(x) {
54
+ return x && x.nodeName ? logNode(x) : x;
55
+ }).join(" ");
56
+ dump("Reader: (Readability) " + msg + "\n");
57
+ }
58
+ };
59
+ } else this.log = function() {};
60
+ }
61
+ Readability.prototype = {
62
+ FLAG_STRIP_UNLIKELYS: 1,
63
+ FLAG_WEIGHT_CLASSES: 2,
64
+ FLAG_CLEAN_CONDITIONALLY: 4,
65
+ ELEMENT_NODE: 1,
66
+ TEXT_NODE: 3,
67
+ DEFAULT_MAX_ELEMS_TO_PARSE: 0,
68
+ DEFAULT_N_TOP_CANDIDATES: 5,
69
+ DEFAULT_TAGS_TO_SCORE: "section,h2,h3,h4,h5,h6,p,td,pre".toUpperCase().split(","),
70
+ DEFAULT_CHAR_THRESHOLD: 500,
71
+ REGEXPS: {
72
+ unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
73
+ okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i,
74
+ positive: /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,
75
+ negative: /-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget/i,
76
+ extraneous: /print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,
77
+ byline: /byline|author|dateline|writtenby|p-author/i,
78
+ replaceFonts: /<(\/?)font[^>]*>/gi,
79
+ normalize: /\s{2,}/g,
80
+ videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,
81
+ shareElements: /(\b|_)(share|sharedaddy)(\b|_)/i,
82
+ nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,
83
+ prevLink: /(prev|earl|old|new|<|«)/i,
84
+ tokenize: /\W+/g,
85
+ whitespace: /^\s*$/,
86
+ hasContent: /\S$/,
87
+ hashUrl: /^#.+/,
88
+ srcsetUrl: /(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))/g,
89
+ b64DataUrl: /^data:\s*([^\s;,]+)\s*;\s*base64\s*,/i,
90
+ commas: /\u002C|\u060C|\uFE50|\uFE10|\uFE11|\u2E41|\u2E34|\u2E32|\uFF0C/g,
91
+ jsonLdArticleTypes: /^Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|AskPublicNewsArticle|BackgroundNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|SatiricalArticle|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference$/,
92
+ adWords: /^(ad(vertising|vertisement)?|pub(licité)?|werb(ung)?|广告|Реклама|Anuncio)$/iu,
93
+ loadingWords: /^((loading|正在加载|Загрузка|chargement|cargando)(…|\.\.\.)?)$/iu
94
+ },
95
+ UNLIKELY_ROLES: [
96
+ "menu",
97
+ "menubar",
98
+ "complementary",
99
+ "navigation",
100
+ "alert",
101
+ "alertdialog",
102
+ "dialog"
103
+ ],
104
+ DIV_TO_P_ELEMS: /* @__PURE__ */ new Set([
105
+ "BLOCKQUOTE",
106
+ "DL",
107
+ "DIV",
108
+ "IMG",
109
+ "OL",
110
+ "P",
111
+ "PRE",
112
+ "TABLE",
113
+ "UL"
114
+ ]),
115
+ ALTER_TO_DIV_EXCEPTIONS: [
116
+ "DIV",
117
+ "ARTICLE",
118
+ "SECTION",
119
+ "P",
120
+ "OL",
121
+ "UL"
122
+ ],
123
+ PRESENTATIONAL_ATTRIBUTES: [
124
+ "align",
125
+ "background",
126
+ "bgcolor",
127
+ "border",
128
+ "cellpadding",
129
+ "cellspacing",
130
+ "frame",
131
+ "hspace",
132
+ "rules",
133
+ "style",
134
+ "valign",
135
+ "vspace"
136
+ ],
137
+ DEPRECATED_SIZE_ATTRIBUTE_ELEMS: [
138
+ "TABLE",
139
+ "TH",
140
+ "TD",
141
+ "HR",
142
+ "PRE"
143
+ ],
144
+ PHRASING_ELEMS: [
145
+ "ABBR",
146
+ "AUDIO",
147
+ "B",
148
+ "BDO",
149
+ "BR",
150
+ "BUTTON",
151
+ "CITE",
152
+ "CODE",
153
+ "DATA",
154
+ "DATALIST",
155
+ "DFN",
156
+ "EM",
157
+ "EMBED",
158
+ "I",
159
+ "IMG",
160
+ "INPUT",
161
+ "KBD",
162
+ "LABEL",
163
+ "MARK",
164
+ "MATH",
165
+ "METER",
166
+ "NOSCRIPT",
167
+ "OBJECT",
168
+ "OUTPUT",
169
+ "PROGRESS",
170
+ "Q",
171
+ "RUBY",
172
+ "SAMP",
173
+ "SCRIPT",
174
+ "SELECT",
175
+ "SMALL",
176
+ "SPAN",
177
+ "STRONG",
178
+ "SUB",
179
+ "SUP",
180
+ "TEXTAREA",
181
+ "TIME",
182
+ "VAR",
183
+ "WBR"
184
+ ],
185
+ CLASSES_TO_PRESERVE: ["page"],
186
+ HTML_ESCAPE_MAP: {
187
+ lt: "<",
188
+ gt: ">",
189
+ amp: "&",
190
+ quot: "\"",
191
+ apos: "'"
192
+ },
193
+ /**
194
+ * Run any post-process modifications to article content as necessary.
195
+ *
196
+ * @param Element
197
+ * @return void
198
+ **/
199
+ _postProcessContent(articleContent) {
200
+ this._fixRelativeUris(articleContent);
201
+ this._simplifyNestedElements(articleContent);
202
+ if (!this._keepClasses) this._cleanClasses(articleContent);
203
+ },
204
+ /**
205
+ * Iterates over a NodeList, calls `filterFn` for each node and removes node
206
+ * if function returned `true`.
207
+ *
208
+ * If function is not passed, removes all the nodes in node list.
209
+ *
210
+ * @param NodeList nodeList The nodes to operate on
211
+ * @param Function filterFn the function to use as a filter
212
+ * @return void
213
+ */
214
+ _removeNodes(nodeList, filterFn) {
215
+ if (this._docJSDOMParser && nodeList._isLiveNodeList) throw new Error("Do not pass live node lists to _removeNodes");
216
+ for (var i = nodeList.length - 1; i >= 0; i--) {
217
+ var node = nodeList[i];
218
+ var parentNode = node.parentNode;
219
+ if (parentNode) {
220
+ if (!filterFn || filterFn.call(this, node, i, nodeList)) parentNode.removeChild(node);
221
+ }
222
+ }
223
+ },
224
+ /**
225
+ * Iterates over a NodeList, and calls _setNodeTag for each node.
226
+ *
227
+ * @param NodeList nodeList The nodes to operate on
228
+ * @param String newTagName the new tag name to use
229
+ * @return void
230
+ */
231
+ _replaceNodeTags(nodeList, newTagName) {
232
+ if (this._docJSDOMParser && nodeList._isLiveNodeList) throw new Error("Do not pass live node lists to _replaceNodeTags");
233
+ for (const node of nodeList) this._setNodeTag(node, newTagName);
234
+ },
235
+ /**
236
+ * Iterate over a NodeList, which doesn't natively fully implement the Array
237
+ * interface.
238
+ *
239
+ * For convenience, the current object context is applied to the provided
240
+ * iterate function.
241
+ *
242
+ * @param NodeList nodeList The NodeList.
243
+ * @param Function fn The iterate function.
244
+ * @return void
245
+ */
246
+ _forEachNode(nodeList, fn) {
247
+ Array.prototype.forEach.call(nodeList, fn, this);
248
+ },
249
+ /**
250
+ * Iterate over a NodeList, and return the first node that passes
251
+ * the supplied test function
252
+ *
253
+ * For convenience, the current object context is applied to the provided
254
+ * test function.
255
+ *
256
+ * @param NodeList nodeList The NodeList.
257
+ * @param Function fn The test function.
258
+ * @return void
259
+ */
260
+ _findNode(nodeList, fn) {
261
+ return Array.prototype.find.call(nodeList, fn, this);
262
+ },
263
+ /**
264
+ * Iterate over a NodeList, return true if any of the provided iterate
265
+ * function calls returns true, false otherwise.
266
+ *
267
+ * For convenience, the current object context is applied to the
268
+ * provided iterate function.
269
+ *
270
+ * @param NodeList nodeList The NodeList.
271
+ * @param Function fn The iterate function.
272
+ * @return Boolean
273
+ */
274
+ _someNode(nodeList, fn) {
275
+ return Array.prototype.some.call(nodeList, fn, this);
276
+ },
277
+ /**
278
+ * Iterate over a NodeList, return true if all of the provided iterate
279
+ * function calls return true, false otherwise.
280
+ *
281
+ * For convenience, the current object context is applied to the
282
+ * provided iterate function.
283
+ *
284
+ * @param NodeList nodeList The NodeList.
285
+ * @param Function fn The iterate function.
286
+ * @return Boolean
287
+ */
288
+ _everyNode(nodeList, fn) {
289
+ return Array.prototype.every.call(nodeList, fn, this);
290
+ },
291
+ _getAllNodesWithTag(node, tagNames) {
292
+ if (node.querySelectorAll) return node.querySelectorAll(tagNames.join(","));
293
+ return [].concat.apply([], tagNames.map(function(tag) {
294
+ var collection = node.getElementsByTagName(tag);
295
+ return Array.isArray(collection) ? collection : Array.from(collection);
296
+ }));
297
+ },
298
+ /**
299
+ * Removes the class="" attribute from every element in the given
300
+ * subtree, except those that match CLASSES_TO_PRESERVE and
301
+ * the classesToPreserve array from the options object.
302
+ *
303
+ * @param Element
304
+ * @return void
305
+ */
306
+ _cleanClasses(node) {
307
+ var classesToPreserve = this._classesToPreserve;
308
+ var className = (node.getAttribute("class") || "").split(/\s+/).filter((cls) => classesToPreserve.includes(cls)).join(" ");
309
+ if (className) node.setAttribute("class", className);
310
+ else node.removeAttribute("class");
311
+ for (node = node.firstElementChild; node; node = node.nextElementSibling) this._cleanClasses(node);
312
+ },
313
+ /**
314
+ * Tests whether a string is a URL or not.
315
+ *
316
+ * @param {string} str The string to test
317
+ * @return {boolean} true if str is a URL, false if not
318
+ */
319
+ _isUrl(str) {
320
+ try {
321
+ new URL(str);
322
+ return true;
323
+ } catch {
324
+ return false;
325
+ }
326
+ },
327
+ /**
328
+ * Converts each <a> and <img> uri in the given element to an absolute URI,
329
+ * ignoring #ref URIs.
330
+ *
331
+ * @param Element
332
+ * @return void
333
+ */
334
+ _fixRelativeUris(articleContent) {
335
+ var baseURI = this._doc.baseURI;
336
+ var documentURI = this._doc.documentURI;
337
+ function toAbsoluteURI(uri) {
338
+ if (baseURI == documentURI && uri.charAt(0) == "#") return uri;
339
+ try {
340
+ return new URL(uri, baseURI).href;
341
+ } catch (ex) {}
342
+ return uri;
343
+ }
344
+ var links = this._getAllNodesWithTag(articleContent, ["a"]);
345
+ this._forEachNode(links, function(link) {
346
+ var href = link.getAttribute("href");
347
+ if (href) {
348
+ if (href.indexOf("javascript:") === 0) {
349
+ if (link.childNodes.length === 1 && link.childNodes[0].nodeType === this.TEXT_NODE) {
350
+ var text = this._doc.createTextNode(link.textContent);
351
+ link.parentNode.replaceChild(text, link);
352
+ } else {
353
+ var container = this._doc.createElement("span");
354
+ while (link.firstChild) container.appendChild(link.firstChild);
355
+ link.parentNode.replaceChild(container, link);
356
+ }
357
+ } else link.setAttribute("href", toAbsoluteURI(href));
358
+ }
359
+ });
360
+ var medias = this._getAllNodesWithTag(articleContent, [
361
+ "img",
362
+ "picture",
363
+ "figure",
364
+ "video",
365
+ "audio",
366
+ "source"
367
+ ]);
368
+ this._forEachNode(medias, function(media) {
369
+ var src = media.getAttribute("src");
370
+ var poster = media.getAttribute("poster");
371
+ var srcset = media.getAttribute("srcset");
372
+ if (src) media.setAttribute("src", toAbsoluteURI(src));
373
+ if (poster) media.setAttribute("poster", toAbsoluteURI(poster));
374
+ if (srcset) {
375
+ var newSrcset = srcset.replace(this.REGEXPS.srcsetUrl, function(_, p1, p2, p3) {
376
+ return toAbsoluteURI(p1) + (p2 || "") + p3;
377
+ });
378
+ media.setAttribute("srcset", newSrcset);
379
+ }
380
+ });
381
+ },
382
+ _simplifyNestedElements(articleContent) {
383
+ var node = articleContent;
384
+ while (node) {
385
+ if (node.parentNode && ["DIV", "SECTION"].includes(node.tagName) && !(node.id && node.id.startsWith("readability"))) {
386
+ if (this._isElementWithoutContent(node)) {
387
+ node = this._removeAndGetNext(node);
388
+ continue;
389
+ } else if (this._hasSingleTagInsideElement(node, "DIV") || this._hasSingleTagInsideElement(node, "SECTION")) {
390
+ var child = node.children[0];
391
+ for (var i = 0; i < node.attributes.length; i++) child.setAttributeNode(node.attributes[i].cloneNode());
392
+ node.parentNode.replaceChild(child, node);
393
+ node = child;
394
+ continue;
395
+ }
396
+ }
397
+ node = this._getNextNode(node);
398
+ }
399
+ },
400
+ /**
401
+ * Get the article title as an H1.
402
+ *
403
+ * @return string
404
+ **/
405
+ _getArticleTitle() {
406
+ var doc = this._doc;
407
+ var curTitle = "";
408
+ var origTitle = "";
409
+ try {
410
+ curTitle = origTitle = doc.title.trim();
411
+ if (typeof curTitle !== "string") curTitle = origTitle = this._getInnerText(doc.getElementsByTagName("title")[0]);
412
+ } catch (e) {}
413
+ var titleHadHierarchicalSeparators = false;
414
+ function wordCount(str) {
415
+ return str.split(/\s+/).length;
416
+ }
417
+ if (/ [\|\-\\\/>»] /.test(curTitle)) {
418
+ titleHadHierarchicalSeparators = / [\\\/>»] /.test(curTitle);
419
+ let allSeparators = Array.from(origTitle.matchAll(/ [\|\-\\\/>»] /gi));
420
+ curTitle = origTitle.substring(0, allSeparators.pop().index);
421
+ if (wordCount(curTitle) < 3) curTitle = origTitle.replace(/^[^\|\-\\\/>»]*[\|\-\\\/>»]/gi, "");
422
+ } else if (curTitle.includes(": ")) {
423
+ var headings = this._getAllNodesWithTag(doc, ["h1", "h2"]);
424
+ var trimmedTitle = curTitle.trim();
425
+ if (!this._someNode(headings, function(heading) {
426
+ return heading.textContent.trim() === trimmedTitle;
427
+ })) {
428
+ curTitle = origTitle.substring(origTitle.lastIndexOf(":") + 1);
429
+ if (wordCount(curTitle) < 3) curTitle = origTitle.substring(origTitle.indexOf(":") + 1);
430
+ else if (wordCount(origTitle.substr(0, origTitle.indexOf(":"))) > 5) curTitle = origTitle;
431
+ }
432
+ } else if (curTitle.length > 150 || curTitle.length < 15) {
433
+ var hOnes = doc.getElementsByTagName("h1");
434
+ if (hOnes.length === 1) curTitle = this._getInnerText(hOnes[0]);
435
+ }
436
+ curTitle = curTitle.trim().replace(this.REGEXPS.normalize, " ");
437
+ var curTitleWordCount = wordCount(curTitle);
438
+ if (curTitleWordCount <= 4 && (!titleHadHierarchicalSeparators || curTitleWordCount != wordCount(origTitle.replace(/[\|\-\\\/>»]+/g, "")) - 1)) curTitle = origTitle;
439
+ return curTitle;
440
+ },
441
+ /**
442
+ * Prepare the HTML document for readability to scrape it.
443
+ * This includes things like stripping javascript, CSS, and handling terrible markup.
444
+ *
445
+ * @return void
446
+ **/
447
+ _prepDocument() {
448
+ var doc = this._doc;
449
+ this._removeNodes(this._getAllNodesWithTag(doc, ["style"]));
450
+ if (doc.body) this._replaceBrs(doc.body);
451
+ this._replaceNodeTags(this._getAllNodesWithTag(doc, ["font"]), "SPAN");
452
+ },
453
+ /**
454
+ * Finds the next node, starting from the given node, and ignoring
455
+ * whitespace in between. If the given node is an element, the same node is
456
+ * returned.
457
+ */
458
+ _nextNode(node) {
459
+ var next = node;
460
+ while (next && next.nodeType != this.ELEMENT_NODE && this.REGEXPS.whitespace.test(next.textContent)) next = next.nextSibling;
461
+ return next;
462
+ },
463
+ /**
464
+ * Replaces 2 or more successive <br> elements with a single <p>.
465
+ * Whitespace between <br> elements are ignored. For example:
466
+ * <div>foo<br>bar<br> <br><br>abc</div>
467
+ * will become:
468
+ * <div>foo<br>bar<p>abc</p></div>
469
+ */
470
+ _replaceBrs(elem) {
471
+ this._forEachNode(this._getAllNodesWithTag(elem, ["br"]), function(br) {
472
+ var next = br.nextSibling;
473
+ var replaced = false;
474
+ while ((next = this._nextNode(next)) && next.tagName == "BR") {
475
+ replaced = true;
476
+ var brSibling = next.nextSibling;
477
+ next.remove();
478
+ next = brSibling;
479
+ }
480
+ if (replaced) {
481
+ var p = this._doc.createElement("p");
482
+ br.parentNode.replaceChild(p, br);
483
+ next = p.nextSibling;
484
+ while (next) {
485
+ if (next.tagName == "BR") {
486
+ var nextElem = this._nextNode(next.nextSibling);
487
+ if (nextElem && nextElem.tagName == "BR") break;
488
+ }
489
+ if (!this._isPhrasingContent(next)) break;
490
+ var sibling = next.nextSibling;
491
+ p.appendChild(next);
492
+ next = sibling;
493
+ }
494
+ while (p.lastChild && this._isWhitespace(p.lastChild)) p.lastChild.remove();
495
+ if (p.parentNode.tagName === "P") this._setNodeTag(p.parentNode, "DIV");
496
+ }
497
+ });
498
+ },
499
+ _setNodeTag(node, tag) {
500
+ this.log("_setNodeTag", node, tag);
501
+ if (this._docJSDOMParser) {
502
+ node.localName = tag.toLowerCase();
503
+ node.tagName = tag.toUpperCase();
504
+ return node;
505
+ }
506
+ var replacement = node.ownerDocument.createElement(tag);
507
+ while (node.firstChild) replacement.appendChild(node.firstChild);
508
+ node.parentNode.replaceChild(replacement, node);
509
+ if (node.readability) replacement.readability = node.readability;
510
+ for (var i = 0; i < node.attributes.length; i++) replacement.setAttributeNode(node.attributes[i].cloneNode());
511
+ return replacement;
512
+ },
513
+ /**
514
+ * Prepare the article node for display. Clean out any inline styles,
515
+ * iframes, forms, strip extraneous <p> tags, etc.
516
+ *
517
+ * @param Element
518
+ * @return void
519
+ **/
520
+ _prepArticle(articleContent) {
521
+ this._cleanStyles(articleContent);
522
+ this._markDataTables(articleContent);
523
+ this._fixLazyImages(articleContent);
524
+ this._cleanConditionally(articleContent, "form");
525
+ this._cleanConditionally(articleContent, "fieldset");
526
+ this._clean(articleContent, "object");
527
+ this._clean(articleContent, "embed");
528
+ this._clean(articleContent, "footer");
529
+ this._clean(articleContent, "link");
530
+ this._clean(articleContent, "aside");
531
+ var shareElementThreshold = this.DEFAULT_CHAR_THRESHOLD;
532
+ this._forEachNode(articleContent.children, function(topCandidate) {
533
+ this._cleanMatchedNodes(topCandidate, function(node, matchString) {
534
+ return this.REGEXPS.shareElements.test(matchString) && node.textContent.length < shareElementThreshold;
535
+ });
536
+ });
537
+ this._clean(articleContent, "iframe");
538
+ this._clean(articleContent, "input");
539
+ this._clean(articleContent, "textarea");
540
+ this._clean(articleContent, "select");
541
+ this._clean(articleContent, "button");
542
+ this._cleanHeaders(articleContent);
543
+ this._cleanConditionally(articleContent, "table");
544
+ this._cleanConditionally(articleContent, "ul");
545
+ this._cleanConditionally(articleContent, "div");
546
+ this._replaceNodeTags(this._getAllNodesWithTag(articleContent, ["h1"]), "h2");
547
+ this._removeNodes(this._getAllNodesWithTag(articleContent, ["p"]), function(paragraph) {
548
+ return this._getAllNodesWithTag(paragraph, [
549
+ "img",
550
+ "embed",
551
+ "object",
552
+ "iframe"
553
+ ]).length === 0 && !this._getInnerText(paragraph, false);
554
+ });
555
+ this._forEachNode(this._getAllNodesWithTag(articleContent, ["br"]), function(br) {
556
+ var next = this._nextNode(br.nextSibling);
557
+ if (next && next.tagName == "P") br.remove();
558
+ });
559
+ this._forEachNode(this._getAllNodesWithTag(articleContent, ["table"]), function(table) {
560
+ var tbody = this._hasSingleTagInsideElement(table, "TBODY") ? table.firstElementChild : table;
561
+ if (this._hasSingleTagInsideElement(tbody, "TR")) {
562
+ var row = tbody.firstElementChild;
563
+ if (this._hasSingleTagInsideElement(row, "TD")) {
564
+ var cell = row.firstElementChild;
565
+ cell = this._setNodeTag(cell, this._everyNode(cell.childNodes, this._isPhrasingContent) ? "P" : "DIV");
566
+ table.parentNode.replaceChild(cell, table);
567
+ }
568
+ }
569
+ });
570
+ },
571
+ /**
572
+ * Initialize a node with the readability object. Also checks the
573
+ * className/id for special names to add to its score.
574
+ *
575
+ * @param Element
576
+ * @return void
577
+ **/
578
+ _initializeNode(node) {
579
+ node.readability = { contentScore: 0 };
580
+ switch (node.tagName) {
581
+ case "DIV":
582
+ node.readability.contentScore += 5;
583
+ break;
584
+ case "PRE":
585
+ case "TD":
586
+ case "BLOCKQUOTE":
587
+ node.readability.contentScore += 3;
588
+ break;
589
+ case "ADDRESS":
590
+ case "OL":
591
+ case "UL":
592
+ case "DL":
593
+ case "DD":
594
+ case "DT":
595
+ case "LI":
596
+ case "FORM":
597
+ node.readability.contentScore -= 3;
598
+ break;
599
+ case "H1":
600
+ case "H2":
601
+ case "H3":
602
+ case "H4":
603
+ case "H5":
604
+ case "H6":
605
+ case "TH": node.readability.contentScore -= 5;
606
+ }
607
+ node.readability.contentScore += this._getClassWeight(node);
608
+ },
609
+ _removeAndGetNext(node) {
610
+ var nextNode = this._getNextNode(node, true);
611
+ node.remove();
612
+ return nextNode;
613
+ },
614
+ /**
615
+ * Traverse the DOM from node to node, starting at the node passed in.
616
+ * Pass true for the second parameter to indicate this node itself
617
+ * (and its kids) are going away, and we want the next node over.
618
+ *
619
+ * Calling this in a loop will traverse the DOM depth-first.
620
+ *
621
+ * @param {Element} node
622
+ * @param {boolean} ignoreSelfAndKids
623
+ * @return {Element}
624
+ */
625
+ _getNextNode(node, ignoreSelfAndKids) {
626
+ if (!ignoreSelfAndKids && node.firstElementChild) return node.firstElementChild;
627
+ if (node.nextElementSibling) return node.nextElementSibling;
628
+ do
629
+ node = node.parentNode;
630
+ while (node && !node.nextElementSibling);
631
+ return node && node.nextElementSibling;
632
+ },
633
+ _textSimilarity(textA, textB) {
634
+ var tokensA = textA.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean);
635
+ var tokensB = textB.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean);
636
+ if (!tokensA.length || !tokensB.length) return 0;
637
+ return 1 - tokensB.filter((token) => !tokensA.includes(token)).join(" ").length / tokensB.join(" ").length;
638
+ },
639
+ /**
640
+ * Checks whether an element node contains a valid byline
641
+ *
642
+ * @param node {Element}
643
+ * @param matchString {string}
644
+ * @return boolean
645
+ */
646
+ _isValidByline(node, matchString) {
647
+ var rel = node.getAttribute("rel");
648
+ var itemprop = node.getAttribute("itemprop");
649
+ var bylineLength = node.textContent.trim().length;
650
+ return (rel === "author" || itemprop && itemprop.includes("author") || this.REGEXPS.byline.test(matchString)) && !!bylineLength && bylineLength < 100;
651
+ },
652
+ _getNodeAncestors(node, maxDepth) {
653
+ maxDepth = maxDepth || 0;
654
+ var i = 0, ancestors = [];
655
+ while (node.parentNode) {
656
+ ancestors.push(node.parentNode);
657
+ if (maxDepth && ++i === maxDepth) break;
658
+ node = node.parentNode;
659
+ }
660
+ return ancestors;
661
+ },
662
+ /***
663
+ * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
664
+ * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
665
+ *
666
+ * @param page a document to run upon. Needs to be a full document, complete with body.
667
+ * @return Element
668
+ **/
669
+ _grabArticle(page) {
670
+ this.log("**** grabArticle ****");
671
+ var doc = this._doc;
672
+ var isPaging = page !== null;
673
+ page = page ? page : this._doc.body;
674
+ if (!page) {
675
+ this.log("No body found in document. Abort.");
676
+ return null;
677
+ }
678
+ var pageCacheHtml = page.innerHTML;
679
+ while (true) {
680
+ this.log("Starting grabArticle loop");
681
+ var stripUnlikelyCandidates = this._flagIsActive(this.FLAG_STRIP_UNLIKELYS);
682
+ var elementsToScore = [];
683
+ var node = this._doc.documentElement;
684
+ let shouldRemoveTitleHeader = true;
685
+ while (node) {
686
+ if (node.tagName === "HTML") this._articleLang = node.getAttribute("lang");
687
+ var matchString = node.className + " " + node.id;
688
+ if (!this._isProbablyVisible(node)) {
689
+ this.log("Removing hidden node - " + matchString);
690
+ node = this._removeAndGetNext(node);
691
+ continue;
692
+ }
693
+ if (node.getAttribute("aria-modal") == "true" && node.getAttribute("role") == "dialog") {
694
+ node = this._removeAndGetNext(node);
695
+ continue;
696
+ }
697
+ if (!this._articleByline && !this._metadata.byline && this._isValidByline(node, matchString)) {
698
+ var endOfSearchMarkerNode = this._getNextNode(node, true);
699
+ var next = this._getNextNode(node);
700
+ var itemPropNameNode = null;
701
+ while (next && next != endOfSearchMarkerNode) {
702
+ var itemprop = next.getAttribute("itemprop");
703
+ if (itemprop && itemprop.includes("name")) {
704
+ itemPropNameNode = next;
705
+ break;
706
+ } else next = this._getNextNode(next);
707
+ }
708
+ this._articleByline = (itemPropNameNode ?? node).textContent.trim();
709
+ node = this._removeAndGetNext(node);
710
+ continue;
711
+ }
712
+ if (shouldRemoveTitleHeader && this._headerDuplicatesTitle(node)) {
713
+ this.log("Removing header: ", node.textContent.trim(), this._articleTitle.trim());
714
+ shouldRemoveTitleHeader = false;
715
+ node = this._removeAndGetNext(node);
716
+ continue;
717
+ }
718
+ if (stripUnlikelyCandidates) {
719
+ if (this.REGEXPS.unlikelyCandidates.test(matchString) && !this.REGEXPS.okMaybeItsACandidate.test(matchString) && !this._hasAncestorTag(node, "table") && !this._hasAncestorTag(node, "code") && node.tagName !== "BODY" && node.tagName !== "A") {
720
+ this.log("Removing unlikely candidate - " + matchString);
721
+ node = this._removeAndGetNext(node);
722
+ continue;
723
+ }
724
+ if (this.UNLIKELY_ROLES.includes(node.getAttribute("role"))) {
725
+ this.log("Removing content with role " + node.getAttribute("role") + " - " + matchString);
726
+ node = this._removeAndGetNext(node);
727
+ continue;
728
+ }
729
+ }
730
+ if ((node.tagName === "DIV" || node.tagName === "SECTION" || node.tagName === "HEADER" || node.tagName === "H1" || node.tagName === "H2" || node.tagName === "H3" || node.tagName === "H4" || node.tagName === "H5" || node.tagName === "H6") && this._isElementWithoutContent(node)) {
731
+ node = this._removeAndGetNext(node);
732
+ continue;
733
+ }
734
+ if (this.DEFAULT_TAGS_TO_SCORE.includes(node.tagName)) elementsToScore.push(node);
735
+ if (node.tagName === "DIV") {
736
+ var p = null;
737
+ var childNode = node.firstChild;
738
+ while (childNode) {
739
+ var nextSibling = childNode.nextSibling;
740
+ if (this._isPhrasingContent(childNode)) {
741
+ if (p !== null) p.appendChild(childNode);
742
+ else if (!this._isWhitespace(childNode)) {
743
+ p = doc.createElement("p");
744
+ node.replaceChild(p, childNode);
745
+ p.appendChild(childNode);
746
+ }
747
+ } else if (p !== null) {
748
+ while (p.lastChild && this._isWhitespace(p.lastChild)) p.lastChild.remove();
749
+ p = null;
750
+ }
751
+ childNode = nextSibling;
752
+ }
753
+ if (this._hasSingleTagInsideElement(node, "P") && this._getLinkDensity(node) < .25) {
754
+ var newNode = node.children[0];
755
+ node.parentNode.replaceChild(newNode, node);
756
+ node = newNode;
757
+ elementsToScore.push(node);
758
+ } else if (!this._hasChildBlockElement(node)) {
759
+ node = this._setNodeTag(node, "P");
760
+ elementsToScore.push(node);
761
+ }
762
+ }
763
+ node = this._getNextNode(node);
764
+ }
765
+ /**
766
+ * Loop through all paragraphs, and assign a score to them based on how content-y they look.
767
+ * Then add their score to their parent node.
768
+ *
769
+ * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
770
+ **/
771
+ var candidates = [];
772
+ this._forEachNode(elementsToScore, function(elementToScore) {
773
+ if (!elementToScore.parentNode || typeof elementToScore.parentNode.tagName === "undefined") return;
774
+ var innerText = this._getInnerText(elementToScore);
775
+ if (innerText.length < 25) return;
776
+ var ancestors = this._getNodeAncestors(elementToScore, 5);
777
+ if (ancestors.length === 0) return;
778
+ var contentScore = 0;
779
+ contentScore += 1;
780
+ contentScore += innerText.split(this.REGEXPS.commas).length;
781
+ contentScore += Math.min(Math.floor(innerText.length / 100), 3);
782
+ this._forEachNode(ancestors, function(ancestor, level) {
783
+ if (!ancestor.tagName || !ancestor.parentNode || typeof ancestor.parentNode.tagName === "undefined") return;
784
+ if (typeof ancestor.readability === "undefined") {
785
+ this._initializeNode(ancestor);
786
+ candidates.push(ancestor);
787
+ }
788
+ if (level === 0) var scoreDivider = 1;
789
+ else if (level === 1) scoreDivider = 2;
790
+ else scoreDivider = level * 3;
791
+ ancestor.readability.contentScore += contentScore / scoreDivider;
792
+ });
793
+ });
794
+ var topCandidates = [];
795
+ for (var c = 0, cl = candidates.length; c < cl; c += 1) {
796
+ var candidate = candidates[c];
797
+ var candidateScore = candidate.readability.contentScore * (1 - this._getLinkDensity(candidate));
798
+ candidate.readability.contentScore = candidateScore;
799
+ this.log("Candidate:", candidate, "with score " + candidateScore);
800
+ for (var t = 0; t < this._nbTopCandidates; t++) {
801
+ var aTopCandidate = topCandidates[t];
802
+ if (!aTopCandidate || candidateScore > aTopCandidate.readability.contentScore) {
803
+ topCandidates.splice(t, 0, candidate);
804
+ if (topCandidates.length > this._nbTopCandidates) topCandidates.pop();
805
+ break;
806
+ }
807
+ }
808
+ }
809
+ var topCandidate = topCandidates[0] || null;
810
+ var neededToCreateTopCandidate = false;
811
+ var parentOfTopCandidate;
812
+ if (topCandidate === null || topCandidate.tagName === "BODY") {
813
+ topCandidate = doc.createElement("DIV");
814
+ neededToCreateTopCandidate = true;
815
+ while (page.firstChild) {
816
+ this.log("Moving child out:", page.firstChild);
817
+ topCandidate.appendChild(page.firstChild);
818
+ }
819
+ page.appendChild(topCandidate);
820
+ this._initializeNode(topCandidate);
821
+ } else if (topCandidate) {
822
+ var alternativeCandidateAncestors = [];
823
+ for (var i = 1; i < topCandidates.length; i++) if (topCandidates[i].readability.contentScore / topCandidate.readability.contentScore >= .75) alternativeCandidateAncestors.push(this._getNodeAncestors(topCandidates[i]));
824
+ var MINIMUM_TOPCANDIDATES = 3;
825
+ if (alternativeCandidateAncestors.length >= MINIMUM_TOPCANDIDATES) {
826
+ parentOfTopCandidate = topCandidate.parentNode;
827
+ while (parentOfTopCandidate.tagName !== "BODY") {
828
+ var listsContainingThisAncestor = 0;
829
+ for (var ancestorIndex = 0; ancestorIndex < alternativeCandidateAncestors.length && listsContainingThisAncestor < MINIMUM_TOPCANDIDATES; ancestorIndex++) listsContainingThisAncestor += Number(alternativeCandidateAncestors[ancestorIndex].includes(parentOfTopCandidate));
830
+ if (listsContainingThisAncestor >= MINIMUM_TOPCANDIDATES) {
831
+ topCandidate = parentOfTopCandidate;
832
+ break;
833
+ }
834
+ parentOfTopCandidate = parentOfTopCandidate.parentNode;
835
+ }
836
+ }
837
+ if (!topCandidate.readability) this._initializeNode(topCandidate);
838
+ parentOfTopCandidate = topCandidate.parentNode;
839
+ var lastScore = topCandidate.readability.contentScore;
840
+ var scoreThreshold = lastScore / 3;
841
+ while (parentOfTopCandidate.tagName !== "BODY") {
842
+ if (!parentOfTopCandidate.readability) {
843
+ parentOfTopCandidate = parentOfTopCandidate.parentNode;
844
+ continue;
845
+ }
846
+ var parentScore = parentOfTopCandidate.readability.contentScore;
847
+ if (parentScore < scoreThreshold) break;
848
+ if (parentScore > lastScore) {
849
+ topCandidate = parentOfTopCandidate;
850
+ break;
851
+ }
852
+ lastScore = parentOfTopCandidate.readability.contentScore;
853
+ parentOfTopCandidate = parentOfTopCandidate.parentNode;
854
+ }
855
+ parentOfTopCandidate = topCandidate.parentNode;
856
+ while (parentOfTopCandidate.tagName != "BODY" && parentOfTopCandidate.children.length == 1) {
857
+ topCandidate = parentOfTopCandidate;
858
+ parentOfTopCandidate = topCandidate.parentNode;
859
+ }
860
+ if (!topCandidate.readability) this._initializeNode(topCandidate);
861
+ }
862
+ var articleContent = doc.createElement("DIV");
863
+ if (isPaging) articleContent.id = "readability-content";
864
+ var siblingScoreThreshold = Math.max(10, topCandidate.readability.contentScore * .2);
865
+ parentOfTopCandidate = topCandidate.parentNode;
866
+ var siblings = parentOfTopCandidate.children;
867
+ for (var s = 0, sl = siblings.length; s < sl; s++) {
868
+ var sibling = siblings[s];
869
+ var append = false;
870
+ this.log("Looking at sibling node:", sibling, sibling.readability ? "with score " + sibling.readability.contentScore : "");
871
+ this.log("Sibling has score", sibling.readability ? sibling.readability.contentScore : "Unknown");
872
+ if (sibling === topCandidate) append = true;
873
+ else {
874
+ var contentBonus = 0;
875
+ if (sibling.className === topCandidate.className && topCandidate.className !== "") contentBonus += topCandidate.readability.contentScore * .2;
876
+ if (sibling.readability && sibling.readability.contentScore + contentBonus >= siblingScoreThreshold) append = true;
877
+ else if (sibling.nodeName === "P") {
878
+ var linkDensity = this._getLinkDensity(sibling);
879
+ var nodeContent = this._getInnerText(sibling);
880
+ var nodeLength = nodeContent.length;
881
+ if (nodeLength > 80 && linkDensity < .25) append = true;
882
+ else if (nodeLength < 80 && nodeLength > 0 && linkDensity === 0 && nodeContent.search(/\.( |$)/) !== -1) append = true;
883
+ }
884
+ }
885
+ if (append) {
886
+ this.log("Appending node:", sibling);
887
+ if (!this.ALTER_TO_DIV_EXCEPTIONS.includes(sibling.nodeName)) {
888
+ this.log("Altering sibling:", sibling, "to div.");
889
+ sibling = this._setNodeTag(sibling, "DIV");
890
+ }
891
+ articleContent.appendChild(sibling);
892
+ siblings = parentOfTopCandidate.children;
893
+ s -= 1;
894
+ sl -= 1;
895
+ }
896
+ }
897
+ if (this._debug) this.log("Article content pre-prep: " + articleContent.innerHTML);
898
+ this._prepArticle(articleContent);
899
+ if (this._debug) this.log("Article content post-prep: " + articleContent.innerHTML);
900
+ if (neededToCreateTopCandidate) {
901
+ topCandidate.id = "readability-page-1";
902
+ topCandidate.className = "page";
903
+ } else {
904
+ var div = doc.createElement("DIV");
905
+ div.id = "readability-page-1";
906
+ div.className = "page";
907
+ while (articleContent.firstChild) div.appendChild(articleContent.firstChild);
908
+ articleContent.appendChild(div);
909
+ }
910
+ if (this._debug) this.log("Article content after paging: " + articleContent.innerHTML);
911
+ var parseSuccessful = true;
912
+ var textLength = this._getInnerText(articleContent, true).length;
913
+ if (textLength < this._charThreshold) {
914
+ parseSuccessful = false;
915
+ page.innerHTML = pageCacheHtml;
916
+ this._attempts.push({
917
+ articleContent,
918
+ textLength
919
+ });
920
+ if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) this._removeFlag(this.FLAG_STRIP_UNLIKELYS);
921
+ else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) this._removeFlag(this.FLAG_WEIGHT_CLASSES);
922
+ else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY);
923
+ else {
924
+ this._attempts.sort(function(a, b) {
925
+ return b.textLength - a.textLength;
926
+ });
927
+ if (!this._attempts[0].textLength) return null;
928
+ articleContent = this._attempts[0].articleContent;
929
+ parseSuccessful = true;
930
+ }
931
+ }
932
+ if (parseSuccessful) {
933
+ var ancestors = [parentOfTopCandidate, topCandidate].concat(this._getNodeAncestors(parentOfTopCandidate));
934
+ this._someNode(ancestors, function(ancestor) {
935
+ if (!ancestor.tagName) return false;
936
+ var articleDir = ancestor.getAttribute("dir");
937
+ if (articleDir) {
938
+ this._articleDir = articleDir;
939
+ return true;
940
+ }
941
+ return false;
942
+ });
943
+ return articleContent;
944
+ }
945
+ }
946
+ },
947
+ /**
948
+ * Converts some of the common HTML entities in string to their corresponding characters.
949
+ *
950
+ * @param str {string} - a string to unescape.
951
+ * @return string without HTML entity.
952
+ */
953
+ _unescapeHtmlEntities(str) {
954
+ if (!str) return str;
955
+ var htmlEscapeMap = this.HTML_ESCAPE_MAP;
956
+ return str.replace(/&(quot|amp|apos|lt|gt);/g, function(_, tag) {
957
+ return htmlEscapeMap[tag];
958
+ }).replace(/&#(?:x([0-9a-f]+)|([0-9]+));/gi, function(_, hex, numStr) {
959
+ var num = parseInt(hex || numStr, hex ? 16 : 10);
960
+ if (num == 0 || num > 1114111 || num >= 55296 && num <= 57343) num = 65533;
961
+ return String.fromCodePoint(num);
962
+ });
963
+ },
964
+ /**
965
+ * Try to extract metadata from JSON-LD object.
966
+ * For now, only Schema.org objects of type Article or its subtypes are supported.
967
+ * @return Object with any metadata that could be extracted (possibly none)
968
+ */
969
+ _getJSONLD(doc) {
970
+ var scripts = this._getAllNodesWithTag(doc, ["script"]);
971
+ var metadata;
972
+ this._forEachNode(scripts, function(jsonLdElement) {
973
+ if (!metadata && jsonLdElement.getAttribute("type") === "application/ld+json") try {
974
+ var content = jsonLdElement.textContent.replace(/^\s*<!\[CDATA\[|\]\]>\s*$/g, "");
975
+ var parsed = JSON.parse(content);
976
+ if (Array.isArray(parsed)) {
977
+ parsed = parsed.find((it) => {
978
+ return it["@type"] && it["@type"].match(this.REGEXPS.jsonLdArticleTypes);
979
+ });
980
+ if (!parsed) return;
981
+ }
982
+ var schemaDotOrgRegex = /^https?\:\/\/schema\.org\/?$/;
983
+ if (!(typeof parsed["@context"] === "string" && parsed["@context"].match(schemaDotOrgRegex) || typeof parsed["@context"] === "object" && typeof parsed["@context"]["@vocab"] == "string" && parsed["@context"]["@vocab"].match(schemaDotOrgRegex))) return;
984
+ if (!parsed["@type"] && Array.isArray(parsed["@graph"])) parsed = parsed["@graph"].find((it) => {
985
+ return (it["@type"] || "").match(this.REGEXPS.jsonLdArticleTypes);
986
+ });
987
+ if (!parsed || !parsed["@type"] || !parsed["@type"].match(this.REGEXPS.jsonLdArticleTypes)) return;
988
+ metadata = {};
989
+ if (typeof parsed.name === "string" && typeof parsed.headline === "string" && parsed.name !== parsed.headline) {
990
+ var title = this._getArticleTitle();
991
+ var nameMatches = this._textSimilarity(parsed.name, title) > .75;
992
+ if (this._textSimilarity(parsed.headline, title) > .75 && !nameMatches) metadata.title = parsed.headline;
993
+ else metadata.title = parsed.name;
994
+ } else if (typeof parsed.name === "string") metadata.title = parsed.name.trim();
995
+ else if (typeof parsed.headline === "string") metadata.title = parsed.headline.trim();
996
+ if (parsed.author) {
997
+ if (typeof parsed.author.name === "string") metadata.byline = parsed.author.name.trim();
998
+ else if (Array.isArray(parsed.author) && parsed.author[0] && typeof parsed.author[0].name === "string") metadata.byline = parsed.author.filter(function(author) {
999
+ return author && typeof author.name === "string";
1000
+ }).map(function(author) {
1001
+ return author.name.trim();
1002
+ }).join(", ");
1003
+ }
1004
+ if (typeof parsed.description === "string") metadata.excerpt = parsed.description.trim();
1005
+ if (parsed.publisher && typeof parsed.publisher.name === "string") metadata.siteName = parsed.publisher.name.trim();
1006
+ if (typeof parsed.datePublished === "string") metadata.datePublished = parsed.datePublished.trim();
1007
+ } catch (err) {
1008
+ this.log(err.message);
1009
+ }
1010
+ });
1011
+ return metadata ? metadata : {};
1012
+ },
1013
+ /**
1014
+ * Attempts to get excerpt and byline metadata for the article.
1015
+ *
1016
+ * @param {Object} jsonld — object containing any metadata that
1017
+ * could be extracted from JSON-LD object.
1018
+ *
1019
+ * @return Object with optional "excerpt" and "byline" properties
1020
+ */
1021
+ _getArticleMetadata(jsonld) {
1022
+ var metadata = {};
1023
+ var values = {};
1024
+ var metaElements = this._doc.getElementsByTagName("meta");
1025
+ var propertyPattern = /\s*(article|dc|dcterm|og|twitter)\s*:\s*(author|creator|description|published_time|title|site_name)\s*/gi;
1026
+ var namePattern = /^\s*(?:(dc|dcterm|og|twitter|parsely|weibo:(article|webpage))\s*[-\.:]\s*)?(author|creator|pub-date|description|title|site_name)\s*$/i;
1027
+ this._forEachNode(metaElements, function(element) {
1028
+ var elementName = element.getAttribute("name");
1029
+ var elementProperty = element.getAttribute("property");
1030
+ var content = element.getAttribute("content");
1031
+ if (!content) return;
1032
+ var matches = null;
1033
+ var name = null;
1034
+ if (elementProperty) {
1035
+ matches = elementProperty.match(propertyPattern);
1036
+ if (matches) {
1037
+ name = matches[0].toLowerCase().replace(/\s/g, "");
1038
+ values[name] = content.trim();
1039
+ }
1040
+ }
1041
+ if (!matches && elementName && namePattern.test(elementName)) {
1042
+ name = elementName;
1043
+ if (content) {
1044
+ name = name.toLowerCase().replace(/\s/g, "").replace(/\./g, ":");
1045
+ values[name] = content.trim();
1046
+ }
1047
+ }
1048
+ });
1049
+ metadata.title = jsonld.title || values["dc:title"] || values["dcterm:title"] || values["og:title"] || values["weibo:article:title"] || values["weibo:webpage:title"] || values.title || values["twitter:title"] || values["parsely-title"];
1050
+ if (!metadata.title) metadata.title = this._getArticleTitle();
1051
+ const articleAuthor = typeof values["article:author"] === "string" && !this._isUrl(values["article:author"]) ? values["article:author"] : void 0;
1052
+ metadata.byline = jsonld.byline || values["dc:creator"] || values["dcterm:creator"] || values.author || values["parsely-author"] || articleAuthor;
1053
+ metadata.excerpt = jsonld.excerpt || values["dc:description"] || values["dcterm:description"] || values["og:description"] || values["weibo:article:description"] || values["weibo:webpage:description"] || values.description || values["twitter:description"];
1054
+ metadata.siteName = jsonld.siteName || values["og:site_name"];
1055
+ metadata.publishedTime = jsonld.datePublished || values["article:published_time"] || values["parsely-pub-date"] || null;
1056
+ metadata.title = this._unescapeHtmlEntities(metadata.title);
1057
+ metadata.byline = this._unescapeHtmlEntities(metadata.byline);
1058
+ metadata.excerpt = this._unescapeHtmlEntities(metadata.excerpt);
1059
+ metadata.siteName = this._unescapeHtmlEntities(metadata.siteName);
1060
+ metadata.publishedTime = this._unescapeHtmlEntities(metadata.publishedTime);
1061
+ return metadata;
1062
+ },
1063
+ /**
1064
+ * Check if node is image, or if node contains exactly only one image
1065
+ * whether as a direct child or as its descendants.
1066
+ *
1067
+ * @param Element
1068
+ **/
1069
+ _isSingleImage(node) {
1070
+ while (node) {
1071
+ if (node.tagName === "IMG") return true;
1072
+ if (node.children.length !== 1 || node.textContent.trim() !== "") return false;
1073
+ node = node.children[0];
1074
+ }
1075
+ return false;
1076
+ },
1077
+ /**
1078
+ * Find all <noscript> that are located after <img> nodes, and which contain only one
1079
+ * <img> element. Replace the first image with the image from inside the <noscript> tag,
1080
+ * and remove the <noscript> tag. This improves the quality of the images we use on
1081
+ * some sites (e.g. Medium).
1082
+ *
1083
+ * @param Element
1084
+ **/
1085
+ _unwrapNoscriptImages(doc) {
1086
+ var imgs = Array.from(doc.getElementsByTagName("img"));
1087
+ this._forEachNode(imgs, function(img) {
1088
+ for (var i = 0; i < img.attributes.length; i++) {
1089
+ var attr = img.attributes[i];
1090
+ switch (attr.name) {
1091
+ case "src":
1092
+ case "srcset":
1093
+ case "data-src":
1094
+ case "data-srcset": return;
1095
+ }
1096
+ if (/\.(jpg|jpeg|png|webp)/i.test(attr.value)) return;
1097
+ }
1098
+ img.remove();
1099
+ });
1100
+ var noscripts = Array.from(doc.getElementsByTagName("noscript"));
1101
+ this._forEachNode(noscripts, function(noscript) {
1102
+ if (!this._isSingleImage(noscript)) return;
1103
+ var tmp = doc.createElement("div");
1104
+ tmp.innerHTML = noscript.innerHTML;
1105
+ var prevElement = noscript.previousElementSibling;
1106
+ if (prevElement && this._isSingleImage(prevElement)) {
1107
+ var prevImg = prevElement;
1108
+ if (prevImg.tagName !== "IMG") prevImg = prevElement.getElementsByTagName("img")[0];
1109
+ var newImg = tmp.getElementsByTagName("img")[0];
1110
+ for (var i = 0; i < prevImg.attributes.length; i++) {
1111
+ var attr = prevImg.attributes[i];
1112
+ if (attr.value === "") continue;
1113
+ if (attr.name === "src" || attr.name === "srcset" || /\.(jpg|jpeg|png|webp)/i.test(attr.value)) {
1114
+ if (newImg.getAttribute(attr.name) === attr.value) continue;
1115
+ var attrName = attr.name;
1116
+ if (newImg.hasAttribute(attrName)) attrName = "data-old-" + attrName;
1117
+ newImg.setAttribute(attrName, attr.value);
1118
+ }
1119
+ }
1120
+ noscript.parentNode.replaceChild(tmp.firstElementChild, prevElement);
1121
+ }
1122
+ });
1123
+ },
1124
+ /**
1125
+ * Removes script tags from the document.
1126
+ *
1127
+ * @param Element
1128
+ **/
1129
+ _removeScripts(doc) {
1130
+ this._removeNodes(this._getAllNodesWithTag(doc, ["script", "noscript"]));
1131
+ },
1132
+ /**
1133
+ * Check if this node has only whitespace and a single element with given tag
1134
+ * Returns false if the DIV node contains non-empty text nodes
1135
+ * or if it contains no element with given tag or more than 1 element.
1136
+ *
1137
+ * @param Element
1138
+ * @param string tag of child element
1139
+ **/
1140
+ _hasSingleTagInsideElement(element, tag) {
1141
+ if (element.children.length != 1 || element.children[0].tagName !== tag) return false;
1142
+ return !this._someNode(element.childNodes, function(node) {
1143
+ return node.nodeType === this.TEXT_NODE && this.REGEXPS.hasContent.test(node.textContent);
1144
+ });
1145
+ },
1146
+ _isElementWithoutContent(node) {
1147
+ return node.nodeType === this.ELEMENT_NODE && !node.textContent.trim().length && (!node.children.length || node.children.length == node.getElementsByTagName("br").length + node.getElementsByTagName("hr").length);
1148
+ },
1149
+ /**
1150
+ * Determine whether element has any children block level elements.
1151
+ *
1152
+ * @param Element
1153
+ */
1154
+ _hasChildBlockElement(element) {
1155
+ return this._someNode(element.childNodes, function(node) {
1156
+ return this.DIV_TO_P_ELEMS.has(node.tagName) || this._hasChildBlockElement(node);
1157
+ });
1158
+ },
1159
+ /***
1160
+ * Determine if a node qualifies as phrasing content.
1161
+ * https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/Content_categories#Phrasing_content
1162
+ **/
1163
+ _isPhrasingContent(node) {
1164
+ return node.nodeType === this.TEXT_NODE || this.PHRASING_ELEMS.includes(node.tagName) || (node.tagName === "A" || node.tagName === "DEL" || node.tagName === "INS") && this._everyNode(node.childNodes, this._isPhrasingContent);
1165
+ },
1166
+ _isWhitespace(node) {
1167
+ return node.nodeType === this.TEXT_NODE && node.textContent.trim().length === 0 || node.nodeType === this.ELEMENT_NODE && node.tagName === "BR";
1168
+ },
1169
+ /**
1170
+ * Get the inner text of a node - cross browser compatibly.
1171
+ * This also strips out any excess whitespace to be found.
1172
+ *
1173
+ * @param Element
1174
+ * @param Boolean normalizeSpaces (default: true)
1175
+ * @return string
1176
+ **/
1177
+ _getInnerText(e, normalizeSpaces) {
1178
+ normalizeSpaces = typeof normalizeSpaces === "undefined" ? true : normalizeSpaces;
1179
+ var textContent = e.textContent.trim();
1180
+ if (normalizeSpaces) return textContent.replace(this.REGEXPS.normalize, " ");
1181
+ return textContent;
1182
+ },
1183
+ /**
1184
+ * Get the number of times a string s appears in the node e.
1185
+ *
1186
+ * @param Element
1187
+ * @param string - what to split on. Default is ","
1188
+ * @return number (integer)
1189
+ **/
1190
+ _getCharCount(e, s) {
1191
+ s = s || ",";
1192
+ return this._getInnerText(e).split(s).length - 1;
1193
+ },
1194
+ /**
1195
+ * Remove the style attribute on every e and under.
1196
+ * TODO: Test if getElementsByTagName(*) is faster.
1197
+ *
1198
+ * @param Element
1199
+ * @return void
1200
+ **/
1201
+ _cleanStyles(e) {
1202
+ if (!e || e.tagName.toLowerCase() === "svg") return;
1203
+ for (var i = 0; i < this.PRESENTATIONAL_ATTRIBUTES.length; i++) e.removeAttribute(this.PRESENTATIONAL_ATTRIBUTES[i]);
1204
+ if (this.DEPRECATED_SIZE_ATTRIBUTE_ELEMS.includes(e.tagName)) {
1205
+ e.removeAttribute("width");
1206
+ e.removeAttribute("height");
1207
+ }
1208
+ var cur = e.firstElementChild;
1209
+ while (cur !== null) {
1210
+ this._cleanStyles(cur);
1211
+ cur = cur.nextElementSibling;
1212
+ }
1213
+ },
1214
+ /**
1215
+ * Get the density of links as a percentage of the content
1216
+ * This is the amount of text that is inside a link divided by the total text in the node.
1217
+ *
1218
+ * @param Element
1219
+ * @return number (float)
1220
+ **/
1221
+ _getLinkDensity(element) {
1222
+ var textLength = this._getInnerText(element).length;
1223
+ if (textLength === 0) return 0;
1224
+ var linkLength = 0;
1225
+ this._forEachNode(element.getElementsByTagName("a"), function(linkNode) {
1226
+ var href = linkNode.getAttribute("href");
1227
+ var coefficient = href && this.REGEXPS.hashUrl.test(href) ? .3 : 1;
1228
+ linkLength += this._getInnerText(linkNode).length * coefficient;
1229
+ });
1230
+ return linkLength / textLength;
1231
+ },
1232
+ /**
1233
+ * Get an elements class/id weight. Uses regular expressions to tell if this
1234
+ * element looks good or bad.
1235
+ *
1236
+ * @param Element
1237
+ * @return number (Integer)
1238
+ **/
1239
+ _getClassWeight(e) {
1240
+ if (!this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) return 0;
1241
+ var weight = 0;
1242
+ if (typeof e.className === "string" && e.className !== "") {
1243
+ if (this.REGEXPS.negative.test(e.className)) weight -= 25;
1244
+ if (this.REGEXPS.positive.test(e.className)) weight += 25;
1245
+ }
1246
+ if (typeof e.id === "string" && e.id !== "") {
1247
+ if (this.REGEXPS.negative.test(e.id)) weight -= 25;
1248
+ if (this.REGEXPS.positive.test(e.id)) weight += 25;
1249
+ }
1250
+ return weight;
1251
+ },
1252
+ /**
1253
+ * Clean a node of all elements of type "tag".
1254
+ * (Unless it's a youtube/vimeo video. People love movies.)
1255
+ *
1256
+ * @param Element
1257
+ * @param string tag to clean
1258
+ * @return void
1259
+ **/
1260
+ _clean(e, tag) {
1261
+ var isEmbed = [
1262
+ "object",
1263
+ "embed",
1264
+ "iframe"
1265
+ ].includes(tag);
1266
+ this._removeNodes(this._getAllNodesWithTag(e, [tag]), function(element) {
1267
+ if (isEmbed) {
1268
+ for (var i = 0; i < element.attributes.length; i++) if (this._allowedVideoRegex.test(element.attributes[i].value)) return false;
1269
+ if (element.tagName === "object" && this._allowedVideoRegex.test(element.innerHTML)) return false;
1270
+ }
1271
+ return true;
1272
+ });
1273
+ },
1274
+ /**
1275
+ * Check if a given node has one of its ancestor tag name matching the
1276
+ * provided one.
1277
+ * @param HTMLElement node
1278
+ * @param String tagName
1279
+ * @param Number maxDepth
1280
+ * @param Function filterFn a filter to invoke to determine whether this node 'counts'
1281
+ * @return Boolean
1282
+ */
1283
+ _hasAncestorTag(node, tagName, maxDepth, filterFn) {
1284
+ maxDepth = maxDepth || 3;
1285
+ tagName = tagName.toUpperCase();
1286
+ var depth = 0;
1287
+ while (node.parentNode) {
1288
+ if (maxDepth > 0 && depth > maxDepth) return false;
1289
+ if (node.parentNode.tagName === tagName && (!filterFn || filterFn(node.parentNode))) return true;
1290
+ node = node.parentNode;
1291
+ depth++;
1292
+ }
1293
+ return false;
1294
+ },
1295
+ /**
1296
+ * Return an object indicating how many rows and columns this table has.
1297
+ */
1298
+ _getRowAndColumnCount(table) {
1299
+ var rows = 0;
1300
+ var columns = 0;
1301
+ var trs = table.getElementsByTagName("tr");
1302
+ for (var i = 0; i < trs.length; i++) {
1303
+ var rowspan = trs[i].getAttribute("rowspan") || 0;
1304
+ if (rowspan) rowspan = parseInt(rowspan, 10);
1305
+ rows += rowspan || 1;
1306
+ var columnsInThisRow = 0;
1307
+ var cells = trs[i].getElementsByTagName("td");
1308
+ for (var j = 0; j < cells.length; j++) {
1309
+ var colspan = cells[j].getAttribute("colspan") || 0;
1310
+ if (colspan) colspan = parseInt(colspan, 10);
1311
+ columnsInThisRow += colspan || 1;
1312
+ }
1313
+ columns = Math.max(columns, columnsInThisRow);
1314
+ }
1315
+ return {
1316
+ rows,
1317
+ columns
1318
+ };
1319
+ },
1320
+ /**
1321
+ * Look for 'data' (as opposed to 'layout') tables, for which we use
1322
+ * similar checks as
1323
+ * https://searchfox.org/mozilla-central/rev/f82d5c549f046cb64ce5602bfd894b7ae807c8f8/accessible/generic/TableAccessible.cpp#19
1324
+ */
1325
+ _markDataTables(root) {
1326
+ var tables = root.getElementsByTagName("table");
1327
+ for (var i = 0; i < tables.length; i++) {
1328
+ var table = tables[i];
1329
+ if (table.getAttribute("role") == "presentation") {
1330
+ table._readabilityDataTable = false;
1331
+ continue;
1332
+ }
1333
+ if (table.getAttribute("datatable") == "0") {
1334
+ table._readabilityDataTable = false;
1335
+ continue;
1336
+ }
1337
+ if (table.getAttribute("summary")) {
1338
+ table._readabilityDataTable = true;
1339
+ continue;
1340
+ }
1341
+ var caption = table.getElementsByTagName("caption")[0];
1342
+ if (caption && caption.childNodes.length) {
1343
+ table._readabilityDataTable = true;
1344
+ continue;
1345
+ }
1346
+ var dataTableDescendants = [
1347
+ "col",
1348
+ "colgroup",
1349
+ "tfoot",
1350
+ "thead",
1351
+ "th"
1352
+ ];
1353
+ var descendantExists = function(tag) {
1354
+ return !!table.getElementsByTagName(tag)[0];
1355
+ };
1356
+ if (dataTableDescendants.some(descendantExists)) {
1357
+ this.log("Data table because found data-y descendant");
1358
+ table._readabilityDataTable = true;
1359
+ continue;
1360
+ }
1361
+ if (table.getElementsByTagName("table")[0]) {
1362
+ table._readabilityDataTable = false;
1363
+ continue;
1364
+ }
1365
+ var sizeInfo = this._getRowAndColumnCount(table);
1366
+ if (sizeInfo.columns == 1 || sizeInfo.rows == 1) {
1367
+ table._readabilityDataTable = false;
1368
+ continue;
1369
+ }
1370
+ if (sizeInfo.rows >= 10 || sizeInfo.columns > 4) {
1371
+ table._readabilityDataTable = true;
1372
+ continue;
1373
+ }
1374
+ table._readabilityDataTable = sizeInfo.rows * sizeInfo.columns > 10;
1375
+ }
1376
+ },
1377
+ _fixLazyImages(root) {
1378
+ this._forEachNode(this._getAllNodesWithTag(root, [
1379
+ "img",
1380
+ "picture",
1381
+ "figure"
1382
+ ]), function(elem) {
1383
+ if (elem.src && this.REGEXPS.b64DataUrl.test(elem.src)) {
1384
+ var parts = this.REGEXPS.b64DataUrl.exec(elem.src);
1385
+ if (parts[1] === "image/svg+xml") return;
1386
+ var srcCouldBeRemoved = false;
1387
+ for (var i = 0; i < elem.attributes.length; i++) {
1388
+ var attr = elem.attributes[i];
1389
+ if (attr.name === "src") continue;
1390
+ if (/\.(jpg|jpeg|png|webp)/i.test(attr.value)) {
1391
+ srcCouldBeRemoved = true;
1392
+ break;
1393
+ }
1394
+ }
1395
+ if (srcCouldBeRemoved) {
1396
+ var b64starts = parts[0].length;
1397
+ if (elem.src.length - b64starts < 133) elem.removeAttribute("src");
1398
+ }
1399
+ }
1400
+ if ((elem.src || elem.srcset && elem.srcset != "null") && !elem.className.toLowerCase().includes("lazy")) return;
1401
+ for (var j = 0; j < elem.attributes.length; j++) {
1402
+ attr = elem.attributes[j];
1403
+ if (attr.name === "src" || attr.name === "srcset" || attr.name === "alt") continue;
1404
+ var copyTo = null;
1405
+ if (/\.(jpg|jpeg|png|webp)\s+\d/.test(attr.value)) copyTo = "srcset";
1406
+ else if (/^\s*\S+\.(jpg|jpeg|png|webp)\S*\s*$/.test(attr.value)) copyTo = "src";
1407
+ if (copyTo) {
1408
+ if (elem.tagName === "IMG" || elem.tagName === "PICTURE") elem.setAttribute(copyTo, attr.value);
1409
+ else if (elem.tagName === "FIGURE" && !this._getAllNodesWithTag(elem, ["img", "picture"]).length) {
1410
+ var img = this._doc.createElement("img");
1411
+ img.setAttribute(copyTo, attr.value);
1412
+ elem.appendChild(img);
1413
+ }
1414
+ }
1415
+ }
1416
+ });
1417
+ },
1418
+ _getTextDensity(e, tags) {
1419
+ var textLength = this._getInnerText(e, true).length;
1420
+ if (textLength === 0) return 0;
1421
+ var childrenLength = 0;
1422
+ var children = this._getAllNodesWithTag(e, tags);
1423
+ this._forEachNode(children, (child) => childrenLength += this._getInnerText(child, true).length);
1424
+ return childrenLength / textLength;
1425
+ },
1426
+ /**
1427
+ * Clean an element of all tags of type "tag" if they look fishy.
1428
+ * "Fishy" is an algorithm based on content length, classnames, link density, number of images & embeds, etc.
1429
+ *
1430
+ * @return void
1431
+ **/
1432
+ _cleanConditionally(e, tag) {
1433
+ if (!this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) return;
1434
+ this._removeNodes(this._getAllNodesWithTag(e, [tag]), function(node) {
1435
+ var isDataTable = function(t) {
1436
+ return t._readabilityDataTable;
1437
+ };
1438
+ var isList = tag === "ul" || tag === "ol";
1439
+ if (!isList) {
1440
+ var listLength = 0;
1441
+ var listNodes = this._getAllNodesWithTag(node, ["ul", "ol"]);
1442
+ this._forEachNode(listNodes, (list) => listLength += this._getInnerText(list).length);
1443
+ isList = listLength / this._getInnerText(node).length > .9;
1444
+ }
1445
+ if (tag === "table" && isDataTable(node)) return false;
1446
+ if (this._hasAncestorTag(node, "table", -1, isDataTable)) return false;
1447
+ if (this._hasAncestorTag(node, "code")) return false;
1448
+ if ([...node.getElementsByTagName("table")].some((tbl) => tbl._readabilityDataTable)) return false;
1449
+ var weight = this._getClassWeight(node);
1450
+ this.log("Cleaning Conditionally", node);
1451
+ if (weight + 0 < 0) return true;
1452
+ if (this._getCharCount(node, ",") < 10) {
1453
+ var p = node.getElementsByTagName("p").length;
1454
+ var img = node.getElementsByTagName("img").length;
1455
+ var li = node.getElementsByTagName("li").length - 100;
1456
+ var input = node.getElementsByTagName("input").length;
1457
+ var headingDensity = this._getTextDensity(node, [
1458
+ "h1",
1459
+ "h2",
1460
+ "h3",
1461
+ "h4",
1462
+ "h5",
1463
+ "h6"
1464
+ ]);
1465
+ var embedCount = 0;
1466
+ var embeds = this._getAllNodesWithTag(node, [
1467
+ "object",
1468
+ "embed",
1469
+ "iframe"
1470
+ ]);
1471
+ for (var i = 0; i < embeds.length; i++) {
1472
+ for (var j = 0; j < embeds[i].attributes.length; j++) if (this._allowedVideoRegex.test(embeds[i].attributes[j].value)) return false;
1473
+ if (embeds[i].tagName === "object" && this._allowedVideoRegex.test(embeds[i].innerHTML)) return false;
1474
+ embedCount++;
1475
+ }
1476
+ var innerText = this._getInnerText(node);
1477
+ if (this.REGEXPS.adWords.test(innerText) || this.REGEXPS.loadingWords.test(innerText)) return true;
1478
+ var contentLength = innerText.length;
1479
+ var linkDensity = this._getLinkDensity(node);
1480
+ var textishTags = [
1481
+ "SPAN",
1482
+ "LI",
1483
+ "TD"
1484
+ ].concat(Array.from(this.DIV_TO_P_ELEMS));
1485
+ var textDensity = this._getTextDensity(node, textishTags);
1486
+ var isFigureChild = this._hasAncestorTag(node, "figure");
1487
+ const shouldRemoveNode = () => {
1488
+ const errs = [];
1489
+ if (!isFigureChild && img > 1 && p / img < .5) errs.push(`Bad p to img ratio (img=${img}, p=${p})`);
1490
+ if (!isList && li > p) errs.push(`Too many li's outside of a list. (li=${li} > p=${p})`);
1491
+ if (input > Math.floor(p / 3)) errs.push(`Too many inputs per p. (input=${input}, p=${p})`);
1492
+ if (!isList && !isFigureChild && headingDensity < .9 && contentLength < 25 && (img === 0 || img > 2) && linkDensity > 0) errs.push(`Suspiciously short. (headingDensity=${headingDensity}, img=${img}, linkDensity=${linkDensity})`);
1493
+ if (!isList && weight < 25 && linkDensity > .2 + this._linkDensityModifier) errs.push(`Low weight and a little linky. (linkDensity=${linkDensity})`);
1494
+ if (weight >= 25 && linkDensity > .5 + this._linkDensityModifier) errs.push(`High weight and mostly links. (linkDensity=${linkDensity})`);
1495
+ if (embedCount === 1 && contentLength < 75 || embedCount > 1) errs.push(`Suspicious embed. (embedCount=${embedCount}, contentLength=${contentLength})`);
1496
+ if (img === 0 && textDensity === 0) errs.push(`No useful content. (img=${img}, textDensity=${textDensity})`);
1497
+ if (errs.length) {
1498
+ this.log("Checks failed", errs);
1499
+ return true;
1500
+ }
1501
+ return false;
1502
+ };
1503
+ var haveToRemove = shouldRemoveNode();
1504
+ if (isList && haveToRemove) {
1505
+ for (var x = 0; x < node.children.length; x++) if (node.children[x].children.length > 1) return haveToRemove;
1506
+ if (img == node.getElementsByTagName("li").length) return false;
1507
+ }
1508
+ return haveToRemove;
1509
+ }
1510
+ return false;
1511
+ });
1512
+ },
1513
+ /**
1514
+ * Clean out elements that match the specified conditions
1515
+ *
1516
+ * @param Element
1517
+ * @param Function determines whether a node should be removed
1518
+ * @return void
1519
+ **/
1520
+ _cleanMatchedNodes(e, filter) {
1521
+ var endOfSearchMarkerNode = this._getNextNode(e, true);
1522
+ var next = this._getNextNode(e);
1523
+ while (next && next != endOfSearchMarkerNode) if (filter.call(this, next, next.className + " " + next.id)) next = this._removeAndGetNext(next);
1524
+ else next = this._getNextNode(next);
1525
+ },
1526
+ /**
1527
+ * Clean out spurious headers from an Element.
1528
+ *
1529
+ * @param Element
1530
+ * @return void
1531
+ **/
1532
+ _cleanHeaders(e) {
1533
+ let headingNodes = this._getAllNodesWithTag(e, ["h1", "h2"]);
1534
+ this._removeNodes(headingNodes, function(node) {
1535
+ let shouldRemove = this._getClassWeight(node) < 0;
1536
+ if (shouldRemove) this.log("Removing header with low class weight:", node);
1537
+ return shouldRemove;
1538
+ });
1539
+ },
1540
+ /**
1541
+ * Check if this node is an H1 or H2 element whose content is mostly
1542
+ * the same as the article title.
1543
+ *
1544
+ * @param Element the node to check.
1545
+ * @return boolean indicating whether this is a title-like header.
1546
+ */
1547
+ _headerDuplicatesTitle(node) {
1548
+ if (node.tagName != "H1" && node.tagName != "H2") return false;
1549
+ var heading = this._getInnerText(node, false);
1550
+ this.log("Evaluating similarity of header:", heading, this._articleTitle);
1551
+ return this._textSimilarity(this._articleTitle, heading) > .75;
1552
+ },
1553
+ _flagIsActive(flag) {
1554
+ return (this._flags & flag) > 0;
1555
+ },
1556
+ _removeFlag(flag) {
1557
+ this._flags = this._flags & ~flag;
1558
+ },
1559
+ _isProbablyVisible(node) {
1560
+ return (!node.style || node.style.display != "none") && (!node.style || node.style.visibility != "hidden") && !node.hasAttribute("hidden") && (!node.hasAttribute("aria-hidden") || node.getAttribute("aria-hidden") != "true" || node.className && node.className.includes && node.className.includes("fallback-image"));
1561
+ },
1562
+ /**
1563
+ * Runs readability.
1564
+ *
1565
+ * Workflow:
1566
+ * 1. Prep the document by removing script tags, css, etc.
1567
+ * 2. Build readability's DOM tree.
1568
+ * 3. Grab the article content from the current dom tree.
1569
+ * 4. Replace the current DOM tree with the new one.
1570
+ * 5. Read peacefully.
1571
+ *
1572
+ * @return void
1573
+ **/
1574
+ parse() {
1575
+ if (this._maxElemsToParse > 0) {
1576
+ var numTags = this._doc.getElementsByTagName("*").length;
1577
+ if (numTags > this._maxElemsToParse) throw new Error("Aborting parsing document; " + numTags + " elements found");
1578
+ }
1579
+ this._unwrapNoscriptImages(this._doc);
1580
+ var jsonLd = this._disableJSONLD ? {} : this._getJSONLD(this._doc);
1581
+ this._removeScripts(this._doc);
1582
+ this._prepDocument();
1583
+ var metadata = this._getArticleMetadata(jsonLd);
1584
+ this._metadata = metadata;
1585
+ this._articleTitle = metadata.title;
1586
+ var articleContent = this._grabArticle();
1587
+ if (!articleContent) return null;
1588
+ this.log("Grabbed: " + articleContent.innerHTML);
1589
+ this._postProcessContent(articleContent);
1590
+ if (!metadata.excerpt) {
1591
+ var paragraphs = articleContent.getElementsByTagName("p");
1592
+ if (paragraphs.length) metadata.excerpt = paragraphs[0].textContent.trim();
1593
+ }
1594
+ var textContent = articleContent.textContent;
1595
+ return {
1596
+ title: this._articleTitle,
1597
+ byline: metadata.byline || this._articleByline,
1598
+ dir: this._articleDir,
1599
+ lang: this._articleLang,
1600
+ content: this._serializer(articleContent),
1601
+ textContent,
1602
+ length: textContent.length,
1603
+ excerpt: metadata.excerpt,
1604
+ siteName: metadata.siteName || this._articleSiteName,
1605
+ publishedTime: metadata.publishedTime
1606
+ };
1607
+ }
1608
+ };
1609
+ if (typeof module === "object") module.exports = Readability;
1610
+ }));
1611
+ //#endregion
1612
+ //#region ../../node_modules/.pnpm/@mozilla+readability@0.6.0/node_modules/@mozilla/readability/Readability-readerable.js
1613
+ var require_Readability_readerable = /* @__PURE__ */ __commonJSMin(((exports, module) => {
1614
+ var REGEXPS = {
1615
+ unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
1616
+ okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i
1617
+ };
1618
+ function isNodeVisible(node) {
1619
+ return (!node.style || node.style.display != "none") && !node.hasAttribute("hidden") && (!node.hasAttribute("aria-hidden") || node.getAttribute("aria-hidden") != "true" || node.className && node.className.includes && node.className.includes("fallback-image"));
1620
+ }
1621
+ /**
1622
+ * Decides whether or not the document is reader-able without parsing the whole thing.
1623
+ * @param {Object} options Configuration object.
1624
+ * @param {number} [options.minContentLength=140] The minimum node content length used to decide if the document is readerable.
1625
+ * @param {number} [options.minScore=20] The minumum cumulated 'score' used to determine if the document is readerable.
1626
+ * @param {Function} [options.visibilityChecker=isNodeVisible] The function used to determine if a node is visible.
1627
+ * @return {boolean} Whether or not we suspect Readability.parse() will suceeed at returning an article object.
1628
+ */
1629
+ function isProbablyReaderable(doc, options = {}) {
1630
+ if (typeof options == "function") options = { visibilityChecker: options };
1631
+ options = Object.assign({
1632
+ minScore: 20,
1633
+ minContentLength: 140,
1634
+ visibilityChecker: isNodeVisible
1635
+ }, options);
1636
+ var nodes = doc.querySelectorAll("p, pre, article");
1637
+ var brNodes = doc.querySelectorAll("div > br");
1638
+ if (brNodes.length) {
1639
+ var set = new Set(nodes);
1640
+ [].forEach.call(brNodes, function(node) {
1641
+ set.add(node.parentNode);
1642
+ });
1643
+ nodes = Array.from(set);
1644
+ }
1645
+ var score = 0;
1646
+ return [].some.call(nodes, function(node) {
1647
+ if (!options.visibilityChecker(node)) return false;
1648
+ var matchString = node.className + " " + node.id;
1649
+ if (REGEXPS.unlikelyCandidates.test(matchString) && !REGEXPS.okMaybeItsACandidate.test(matchString)) return false;
1650
+ if (node.matches("li p")) return false;
1651
+ var textContentLength = node.textContent.trim().length;
1652
+ if (textContentLength < options.minContentLength) return false;
1653
+ score += Math.sqrt(textContentLength - options.minContentLength);
1654
+ if (score > options.minScore) return true;
1655
+ return false;
1656
+ });
1657
+ }
1658
+ if (typeof module === "object") module.exports = isProbablyReaderable;
1659
+ }));
1660
+ //#endregion
1661
+ //#region ../../node_modules/.pnpm/@mozilla+readability@0.6.0/node_modules/@mozilla/readability/index.js
1662
+ var require_readability = /* @__PURE__ */ __commonJSMin(((exports, module) => {
1663
+ module.exports = {
1664
+ Readability: require_Readability(),
1665
+ isProbablyReaderable: require_Readability_readerable()
1666
+ };
1667
+ }));
1668
+ //#endregion
1669
+ export default require_readability();
1670
+ export {};