extract-webpage 1.2.5

package/src/html-to-content/extract-content/extract-content-mercury-utils.ts

@@ -0,0 +1,696 @@
+// @ts-nocheck
+/**
+ * @module research/extractor/html-to-content/extract-content/extract-content-mercury-utils
+ * @description Research library module.
+ */
+//HTML-to-Content Postlight  Utilities
+
+const SPACER_RE = new RegExp("transparent|spacer|blank", "i");
+const KEEP_CLASS = "parser-keep";
+const STRIP_OUTPUT_TAGS = [
+  "title",
+  "script",
+  "noscript",
+  "link",
+  "style",
+  "hr",
+  "embed",
+  "iframe",
+  "object",
+];
+const WHITELIST_ATTRS = [
+  "src",
+  "srcset",
+  "sizes",
+  "type",
+  "href",
+  "class",
+  "id",
+  "alt",
+  "xlink:href",
+  "width",
+  "height",
+];
+const WHITELIST_ATTRS_RE = new RegExp(`^(${WHITELIST_ATTRS.join("|")})$`, "i");
+const REMOVE_EMPTY_TAGS = ["p"];
+const REMOVE_EMPTY_SELECTORS = REMOVE_EMPTY_TAGS.map(
+  (tag) => `${tag}:empty`
+).join(",");
+const UNLIKELY_CANDIDATES_BLACKLIST = [
+  "ad-break",
+  "adbox",
+  "advert",
+  "addthis",
+  "agegate",
+  "aux",
+  "blogger-labels",
+  "combx",
+  "comment",
+  "conversation",
+  "disqus",
+  "entry-unrelated",
+  "extra",
+  "foot",
+  "header",
+  "hidden",
+  "loader",
+  "login",
+  "menu",
+  "meta",
+  "nav",
+  "outbrain",
+  "pager",
+  "pagination",
+  "predicta",
+  "presence_control_external",
+  "popup",
+  "printfriendly",
+  "related",
+  "remove",
+  "remark",
+  "rss",
+  "share",
+  "shoutbox",
+  "sidebar",
+  "sociable",
+  "sponsor",
+  "taboola",
+  "tools",
+];
+const UNLIKELY_CANDIDATES_WHITELIST = [
+  "and",
+  "article",
+  "body",
+  "blogindex",
+  "column",
+  "content",
+  "entry-content-asset",
+  "format",
+  "hfeed",
+  "hentry",
+  "hatom",
+  "main",
+  "page",
+  "posts",
+  "shadow",
+];
+const DIV_TO_P_BLOCK_TAGS = [
+  "a",
+  "blockquote",
+  "dl",
+  "div",
+  "img",
+  "p",
+  "pre",
+  "table",
+].join(",");
+const NON_TOP_CANDIDATE_TAGS = [
+  "br",
+  "b",
+  "i",
+  "label",
+  "hr",
+  "area",
+  "base",
+  "basefont",
+  "input",
+  "img",
+  "link",
+  "meta",
+];
+const NON_TOP_CANDIDATE_TAGS_RE = new RegExp(
+  `^(${NON_TOP_CANDIDATE_TAGS.join("|")})$`,
+  "i"
+);
+const HNEWS_CONTENT_SELECTORS = [
+  [".hentry", ".entry-content"],
+  ["entry", ".entry-content"],
+  [".entry", ".entry_content"],
+  [".post", ".postbody"],
+  [".post", ".post_body"],
+  [".post", ".post-body"],
+];
+const PHOTO_HINTS = ["figure", "photo", "image", "caption"];
+const PHOTO_HINTS_RE = new RegExp(PHOTO_HINTS.join("|"), "i");
+const POSITIVE_SCORE_HINTS = [
+  "article",
+  "articlecontent",
+  "instapaper_body",
+  "blog",
+  "body",
+  "content",
+  "entry-content-asset",
+  "entry",
+  "hentry",
+  "main",
+  "Normal",
+  "page",
+  "pagination",
+  "permalink",
+  "post",
+  "story",
+  "text",
+  "[-_]copy",
+  "\\Bcopy",
+];
+const POSITIVE_SCORE_RE = new RegExp(POSITIVE_SCORE_HINTS.join("|"), "i");
+const READABILITY_ASSET = new RegExp("entry-content-asset", "i");
+const NEGATIVE_SCORE_HINTS = [
+  "adbox",
+  "advert",
+  "author",
+  "bio",
+  "bookmark",
+  "bottom",
+  "byline",
+  "clear",
+  "com-",
+  "combx",
+  "comment",
+  "comment\\B",
+  "contact",
+  "copy",
+  "credit",
+  "crumb",
+  "date",
+  "deck",
+  "excerpt",
+  "featured",
+  "foot",
+  "footer",
+  "footnote",
+  "graf",
+  "head",
+  "info",
+  "infotext",
+  "instapaper_ignore",
+  "jump",
+  "linebreak",
+  "link",
+  "masthead",
+  "media",
+  "meta",
+  "modal",
+  "outbrain",
+  "promo",
+  "pr_",
+  "related",
+  "respond",
+  "roundcontent",
+  "scroll",
+  "secondary",
+  "share",
+  "shopping",
+  "shoutbox",
+  "side",
+  "sidebar",
+  "sponsor",
+  "stamp",
+  "sub",
+  "summary",
+  "tags",
+  "tools",
+  "widget",
+];
+const NEGATIVE_SCORE_RE = new RegExp(NEGATIVE_SCORE_HINTS.join("|"), "i");
+const IS_WP_SELECTOR = "meta[name=generator][value^=WordPress]";
+const DIGIT_RE = new RegExp("[0-9]");
+const EXTRANEOUS_LINK_HINTS = [
+  "print",
+  "archive",
+  "comment",
+  "discuss",
+  "e-mail",
+  "email",
+  "share",
+  "reply",
+  "all",
+  "login",
+  "sign",
+  "single",
+  "adx",
+  "entry-unrelated",
+];
+const BLOCK_LEVEL_TAGS = [
+  "article",
+  "aside",
+  "blockquote",
+  "body",
+  "br",
+  "button",
+  "canvas",
+  "caption",
+  "col",
+  "colgroup",
+  "dd",
+  "div",
+  "dl",
+  "dt",
+  "embed",
+  "fieldset",
+  "figcaption",
+  "figure",
+  "footer",
+  "form",
+  "h1",
+  "h2",
+  "h3",
+  "h4",
+  "h5",
+  "h6",
+  "header",
+  "hgroup",
+  "hr",
+  "li",
+  "map",
+  "object",
+  "ol",
+  "output",
+  "p",
+  "pre",
+  "progress",
+  "section",
+  "table",
+  "tbody",
+  "textarea",
+  "tfoot",
+  "th",
+  "thead",
+  "tr",
+  "ul",
+  "video",
+];
+const BLOCK_LEVEL_TAGS_RE = new RegExp(
+  `^(${BLOCK_LEVEL_TAGS.join("|")})$`,
+  "i"
+);
+const candidatesBlacklist = UNLIKELY_CANDIDATES_BLACKLIST.join("|");
+const CANDIDATES_BLACKLIST = new RegExp(candidatesBlacklist, "i");
+const candidatesWhitelist = UNLIKELY_CANDIDATES_WHITELIST.join("|");
+const CANDIDATES_WHITELIST = new RegExp(candidatesWhitelist, "i");
+
+
+function normalizeSpaces(text) {
+  return text.replace(/\s{2,}(?![^<>]*<\/(pre|code|textarea)>)/g, " ").trim();
+}
+
+
+function paragraphize(node, document, br = false) {
+  if (br) {
+    let sibling = node.nextSibling;
+    const p = document.createElement("p");
+
+    while (
+      sibling &&
+      !(sibling.tagName && BLOCK_LEVEL_TAGS_RE.test(sibling.tagName))
+    ) {
+      const { nextSibling } = sibling;
+      p.appendChild(sibling);
+      sibling = nextSibling;
+    }
+
+    node.parentNode?.replaceChild(p, node);
+    node.remove();
+    return document;
+  }
+
+  return document;
+}
+
+
+function getAttrs(node) {
+  return node.attributes
+    ? Array.from(node.attributes).reduce((acc, attr) => {
+        acc[attr.name] = attr.value;
+        return acc;
+      }, {})
+    : {};
+}
+
+
+function convertNodeTo(node, document, tag = "p") {
+  if (!node) {
+    return document;
+  }
+  const attrs = getAttrs(node);
+
+  const attribString = Object.keys(attrs)
+    .map((key) => `${key}="${attrs[key]}"`)
+    .join(" ");
+  const html =
+    node.tagName.toLowerCase() === "noscript"
+      ? node.textContent
+      : node.innerHTML;
+
+  const newElement = document.createElement(tag);
+  newElement.innerHTML = html;
+  Object.keys(attrs).forEach((key) => newElement.setAttribute(key, attrs[key]));
+
+  node.parentNode?.replaceChild(newElement, node);
+  return document;
+}
+
+
+function brsToPs(document) {
+  let collapsing = false;
+  document.querySelectorAll("br").forEach((element) => {
+    const nextElement = element.nextElementSibling;
+
+    if (nextElement && nextElement.tagName.toLowerCase() === "br") {
+      collapsing = true;
+      element.remove();
+    } else if (collapsing) {
+      collapsing = false;
+      paragraphize(element, document, true);
+    }
+  });
+
+  return document;
+}
+
+
+function convertDivs(document) {
+  document.querySelectorAll("div").forEach((div) => {
+    const convertible = div.querySelectorAll(DIV_TO_P_BLOCK_TAGS).length === 0;
+
+    if (convertible) {
+      convertNodeTo(div, document, "p");
+    }
+  });
+
+  return document;
+}
+
+
+function convertSpans(document) {
+  document.querySelectorAll("span").forEach((span) => {
+    const convertible = !span.closest("p, div, li, figcaption");
+    if (convertible) {
+      convertNodeTo(span, document, "p");
+    }
+  });
+
+  return document;
+}
+
+
+function convertToParagraphs(document) {
+  document = brsToPs(document);
+  document = convertDivs(document);
+  document = convertSpans(document);
+
+  return document;
+}
+
+
+function cleanForHeight(img, document) {
+  const height = parseInt(img.getAttribute("height"), 10);
+  const width = parseInt(img.getAttribute("width"), 10) || 20;
+
+  if ((height || 20) < 10 || width < 10) {
+    img.remove();
+  } else if (height) {
+    img.removeAttribute("height");
+  }
+
+  return document;
+}
+
+
+function removeSpacers(img, document) {
+  if (SPACER_RE.test(img.getAttribute("src"))) {
+    img.remove();
+  }
+
+  return document;
+}
+
+
+function cleanImages(article, document) {
+  article.querySelectorAll("img").forEach((img) => {
+    cleanForHeight(img, document);
+    removeSpacers(img, document);
+  });
+
+  return document;
+}
+
+
+function stripJunkTags(article, document, tags = []) {
+  if (tags.length === 0) {
+    tags = STRIP_OUTPUT_TAGS;
+  }
+
+  article.querySelectorAll(tags.join(",")).forEach((element) => {
+    if (!element.classList.contains(KEEP_CLASS)) {
+      element.remove();
+    }
+  });
+
+  return document;
+}
+
+
+function cleanHOnes(article, document) {
+  const hOnes = article.querySelectorAll("h1");
+
+  if (hOnes.length < 3) {
+    hOnes.forEach((node) => node.remove());
+  } else {
+    hOnes.forEach((node) => {
+      convertNodeTo(node, document, "h2");
+    });
+  }
+
+  return document;
+}
+
+
+function cleanAttributes(article, document) {
+  return removeAllButWhitelist(
+    article.parentNode ? article.parentNode : article,
+    document
+  );
+}
+
+function removeAllButWhitelist(article) {
+  article.querySelectorAll("*").forEach((node) => {
+    const attrs = getAttrs(node);
+
+    setAttrs(
+      node,
+      Object.keys(attrs).reduce((acc, attr) => {
+        if (WHITELIST_ATTRS_RE.test(attr)) {
+          return { ...acc, [attr]: attrs[attr] };
+        }
+
+        return acc;
+      }, {})
+    );
+  });
+
+  article.querySelectorAll(`.${KEEP_CLASS}`).forEach((node) => {
+    node.classList.remove(KEEP_CLASS);
+  });
+
+  return article;
+}
+
+
+function removeEmpty(article) {
+  article.querySelectorAll("p").forEach((p) => {
+    if (
+      p.querySelectorAll("iframe, img").length === 0 &&
+      p.textContent.trim() === ""
+    )
+      p.remove();
+  });
+
+  return article;
+}
+
+
+function removeUnlessContent(node, weight) {
+  if (node.classList.contains("entry-content-asset")) {
+    return;
+  }
+
+  const content = normalizeSpaces(node.textContent);
+
+  if (content.match(/,/g)?.length < 10) {
+    const pCount = node.querySelectorAll("p").length;
+    const inputCount = node.querySelectorAll("input").length;
+
+    if (inputCount > pCount / 3) {
+      node.remove();
+      return;
+    }
+
+    const contentLength = content.length;
+    const imgCount = node.querySelectorAll("img").length;
+
+    if (contentLength < 25 && imgCount === 0) {
+      node.remove();
+      return;
+    }
+
+    const density = linkDensity(node);
+
+    if (weight < 25 && density > 0.2 && contentLength > 75) {
+      node.remove();
+      return;
+    }
+
+    if (weight >= 25 && density > 0.5) {
+      const tagName = node.tagName.toLowerCase();
+      const nodeIsList = tagName === "ol" || tagName === "ul";
+      if (nodeIsList) {
+        const previousNode = node.previousElementSibling;
+        if (
+          previousNode &&
+          normalizeSpaces(previousNode.textContent).slice(-1) === ":"
+        ) {
+          return;
+        }
+      }
+
+      node.remove();
+      return;
+    }
+
+    const scriptCount = node.querySelectorAll("script").length;
+
+    if (scriptCount > 0 && contentLength < 150) {
+      node.remove();
+    }
+  }
+}
+
+
+function rewriteTopLevel(article, document) {
+  // document.documentElement.outerHTML = `<div>${document.documentElement.innerHTML}</div>`;
+  document.body.outerHTML = `<div>${document.body.innerHTML}</div>`;
+
+  return document;
+}
+
+
+function textLength(text) {
+  return text.trim().replace(/\s+/g, " ").length;
+}
+
+
+function linkDensity(node) {
+  const totalTextLength = textLength(node.textContent);
+
+  const linkText = Array.from(node.querySelectorAll("a"))
+    .map((a) => a.textContent)
+    .join(" ");
+  const linkLength = textLength(linkText);
+
+  if (totalTextLength > 0) {
+    return linkLength / totalTextLength;
+  }
+  if (totalTextLength === 0 && linkLength > 0) {
+    return 1;
+  }
+
+  return 0;
+}
+
+
+function stripTags(text, document) {
+  const span = document.createElement("span");
+  span.innerHTML = text;
+  const cleanText = span.textContent;
+  return cleanText === "" ? text : cleanText;
+}
+
+
+function stripUnlikelyCandidates(document) {
+  document.querySelectorAll("*:not(a)").forEach((node) => {
+    const classes = node.getAttribute("class");
+    const id = node.getAttribute("id");
+    if (!id && !classes) return;
+
+    const classAndId = `${classes || ""} ${id || ""}`;
+    if (CANDIDATES_WHITELIST.test(classAndId)) {
+      return;
+    }
+    if (CANDIDATES_BLACKLIST.test(classAndId)) {
+      node.remove();
+    }
+  });
+
+  return document;
+}
+
+
+function withinComment(node) {
+  let parent = node.parentNode;
+  while (parent) {
+    const attrs = getAttrs(parent);
+    const { class: nodeClass, id } = attrs;
+    const classAndId = `${nodeClass} ${id}`;
+    if (classAndId.includes("comment")) {
+      return true;
+    }
+    parent = parent.parentNode;
+  }
+
+  return false;
+}
+
+
+function nodeIsSufficient(node) {
+  return node.textContent.trim().length >= 100;
+}
+
+
+function isWordpress(document) {
+  return document.querySelectorAll(IS_WP_SELECTOR).length > 0;
+}
+
+
+function setAttr(node, attr, val) {
+  node.setAttribute(attr, val);
+  return node;
+}
+
+
+function setAttrs(node, attrs) {
+  while (node.attributes.length > 0) {
+    node.removeAttribute(node.attributes[0].name);
+  }
+
+  Object.keys(attrs).forEach((key) => {
+    node.setAttribute(key, attrs[key]);
+  });
+
+  return node;
+}
+
+export {
+  normalizeSpaces,
+  paragraphize,
+  getAttrs,
+  convertNodeTo,
+  brsToPs,
+  convertToParagraphs,
+  cleanImages,
+  stripJunkTags,
+  cleanHOnes,
+  cleanAttributes,
+  removeEmpty,
+  rewriteTopLevel,
+  textLength,
+  linkDensity,
+  stripTags,
+  stripUnlikelyCandidates,
+  withinComment,
+  nodeIsSufficient,
+  isWordpress,
+  setAttr,
+  removeUnlessContent,
+  setAttrs,
+};