extract-webpage 1.2.5

package/src/html-to-content/extract-content/extract-content-mercury.ts

@@ -0,0 +1,830 @@
+// @ts-nocheck
+/**
+ * @module research/extractor/html-to-content/extract-content/extract-content-mercury
+ * @description Research library module.
+ */
+import { parseHTML } from "linkedom";
+import {
+  convertNodeTo,
+  stripUnlikelyCandidates,
+  convertToParagraphs,
+  cleanAttributes,
+  cleanHOnes,
+  cleanImages,
+  removeEmpty,
+  rewriteTopLevel,
+  stripJunkTags,
+  textLength,
+  linkDensity,
+  removeUnlessContent,
+  nodeIsSufficient,
+} from "./extract-content-mercury-utils";
+
+/**
+ *  ### HTML-to-Main-Content Extractor #2
+ *
+ * 1. The algorithm starts by loading the HTML content using linkedom, a lightweight DOM parser for Node.js.
+ * 2. It then applies a series of cleaning and scoring techniques to identify the main content of
+ * the page, starting with stripping unlikely candidates (e.g., elements with class names like "comment"
+ *  or "sidebar").
+ * 3. The HTML is converted into a series of paragraph elements, which are then scored based on various
+ *  factors such as text length, number of commas, and the presence of certain class names or IDs.
+ * 4. The algorithm assigns scores to parent and grandparent elements based on the scores of their
+ * children, with parents receiving the full score and grandparents receiving half.
+ * 5. After scoring, the algorithm finds the top candidate element by selecting the node with the
+ * highest score.
+ * 6. The top candidate's siblings are then examined to see if they should be included in the main
+ * content, based on their scores and other factors like link density.
+ * 7. The algorithm then cleans the selected content by removing unnecessary tags, attributes, and empty
+ * elements.
+ * 8. It also handles special cases like cleaning up header tags, images, and other potentially irrelevant
+ *  content.
+ * 9. Throughout the process, the algorithm uses various regular expressions and scoring heuristics to
+ * identify positive and negative indicators of content relevance.
+ * 10. Finally, the cleaned and extracted content is returned as an HTML string, representing the main
+ * body of the article or webpage.
+ *
+ * [Article Extraction Benchmark](https://trafilatura.readthedocs.io/en/latest/evaluation.html)
+ *
+ * @param {string} html - The HTML content to extract from.
+ * @param {Object} [opts] - The options for content extraction.
+ * @param {boolean} opts.stripUnlikelyCandidates default=true - Remove elements that match non-article-
+ * like criteria first (e.g., elements with a classname of "comment").
+ * @param {boolean} opts.weightNodes default=true - Modify an element's score based on certain classNames or
+ * IDs (e.g., subtract if a node has a className of 'comment', add if a node has an ID of 'entry-content').
+ * @param {boolean} opts.cleanConditionally default=true - Clean the node to remove superfluous content
+ *  like forms, ads, etc. Initially, pass in the most restrictive options which will return the highest
+ * quality content. On each failure, retry with slightly more lax options.
+ * @returns {string} The extracted content as an HTML string, or null if extraction fails.
+ * @author [vtempest (2025)](https://github.com/vtempest)
+ * Based on [Postlight Mercury Parser (2017-)](https://github.com/postlight/parser/tree/main/src)
+ * @example var url =  "https://en.wikipedia.org/wiki/David_Hilbert"
+ * var html = await (await fetch(url)).text();
+ * var content = extractMainContentFromHTML(html);
+ * console.log(content); // HTML content of main article body
+ * @category Extract
+ */
+export function extractMainContentFromHTML2(html, opts) {
+  opts = {
+    stripUnlikelyCandidates: true,
+    weightNodes: true,
+    cleanConditionally: true,
+    ...opts,
+  };
+
+  if (!html) return;
+  const document = parseHTML(html)?.document;
+
+  if (!document) return;
+
+  var title = document.querySelector("title")?.textContent.trim();
+
+  // Cascade through our extraction-specific opts in an ordered fashion,
+  // turning them off as we try to extract content.
+  let node = getContentNode(document, title, opts);
+
+  if (nodeIsSufficient(node)) {
+    return cleanAndReturnNode(node, document);
+  }
+
+  // We didn't succeed on first pass, one by one disable our
+  // extraction opts and try again.
+  // eslint-disable-next-line no-restricted-syntax
+  for (const key of Reflect.ownKeys(opts).filter((k) => opts[k] === true)) {
+    opts[key] = false;
+    const { document: newDocument } = parseHTML(html);
+
+    node = getContentNode(newDocument, title, opts);
+
+    if (nodeIsSufficient(node)) {
+      break;
+    }
+  }
+
+  return cleanAndReturnNode(node, document);
+}
+
+// A list of tags that should be ignored when trying to find the top candidate
+// for a document.
+const NON_TOP_CANDIDATE_TAGS = [
+  "br",
+  "b",
+  "i",
+  "label",
+  "hr",
+  "area",
+  "base",
+  "basefont",
+  "input",
+  "img",
+  "link",
+  "meta",
+];
+
+const NON_TOP_CANDIDATE_TAGS_RE = new RegExp(
+  `^(${NON_TOP_CANDIDATE_TAGS.join("|")})$`,
+  "i"
+);
+
+const PHOTO_HINTS = ["figure", "photo", "image", "caption"];
+const PHOTO_HINTS_RE = new RegExp(PHOTO_HINTS.join("|"), "i");
+
+// A list of strings that denote a positive scoring for this content as being
+// an article container. Checked against className and id.
+//
+// TODO: Perhaps have these scale based on their odds of being quality?
+const POSITIVE_SCORE_HINTS = [
+  "article",
+  "articlecontent",
+  "instapaper_body",
+  "blog",
+  "body",
+  "content",
+  "entry-content-asset",
+  "entry",
+  "hentry",
+  "main",
+  "Normal",
+  "page",
+  "pagination",
+  "permalink",
+  "post",
+  "story",
+  "text",
+  "[-_]copy", // usatoday
+  "\\Bcopy",
+];
+
+// The above list, joined into a matching regular expression
+const POSITIVE_SCORE_RE = new RegExp(POSITIVE_SCORE_HINTS.join("|"), "i");
+
+// Readability publisher-specific guidelines
+const READABILITY_ASSET = new RegExp("entry-content-asset", "i");
+
+const PARAGRAPH_SCORE_TAGS = new RegExp("^(p|li|span|pre)$", "i");
+const CHILD_CONTENT_TAGS = new RegExp("^(td|blockquote|ol|ul|dl)$", "i");
+const BAD_TAGS = new RegExp("^(address|form)$", "i");
+
+// A list of strings that denote a negative scoring for this content as being
+// an article container. Checked against className and id.
+//
+// TODO: Perhaps have these scale based on their odds of being quality?
+const NEGATIVE_SCORE_HINTS = [
+  "adbox",
+  "advert",
+  "author",
+  "bio",
+  "bookmark",
+  "bottom",
+  "byline",
+  "clear",
+  "com-",
+  "combx",
+  "comment",
+  "comment\\B",
+  "contact",
+  "copy",
+  "credit",
+  "crumb",
+  "date",
+  "deck",
+  "excerpt",
+  "featured", // tnr.com has a featured_content which throws us off
+  "foot",
+  "footer",
+  "footnote",
+  "graf",
+  "head",
+  "info",
+  "infotext", // newscientist.com copyright
+  "instapaper_ignore",
+  "jump",
+  "linebreak",
+  "link",
+  "masthead",
+  "media",
+  "meta",
+  "modal",
+  "outbrain", // slate.com junk
+  "promo",
+  "pr_", // autoblog - press release
+  "related",
+  "respond",
+  "roundcontent", // lifehacker restricted content warning
+  "scroll",
+  "secondary",
+  "share",
+  "shopping",
+  "shoutbox",
+  "side",
+  "sidebar",
+  "sponsor",
+  "stamp",
+  "sub",
+  "summary",
+  "tags",
+  "tools",
+  "widget",
+];
+// The above list, joined into a matching regular expression
+const NEGATIVE_SCORE_RE = new RegExp(NEGATIVE_SCORE_HINTS.join("|"), "i");
+
+// A list of selectors that specify, very clearly, either hNews or other
+// very content-specific style content, like Blogger templates.
+// More examples here: http://microformats.org/wiki/blog-post-formats
+const HNEWS_CONTENT_SELECTORS = [
+  [".hentry", ".entry-content"],
+  ["entry", ".entry-content"],
+  [".entry", ".entry_content"],
+  [".post", ".postbody"],
+  [".post", ".post_body"],
+  [".post", ".post-body"],
+];
+
+/**
+ * Normalizes spaces in a given text string.
+ * @param {string} text - The text to normalize.
+ * @returns {string} The normalized text.
+ */
+function normalizeSpaces(text) {
+  return text.replace(/\s{2,}(?![^<>]*<\/(pre|code|textarea)>)/g, " ").trim();
+}
+
+/**
+ * Cleans and returns the HTML of a given node.
+ * @param {Node} node - The node to clean and return.
+ * @param {Document} document - The document object.
+ * @returns {string|null} The cleaned HTML string or null if no node is provided.
+ */
+function cleanAndReturnNode(node, document) {
+  if (!node) {
+    return null;
+  }
+
+  return normalizeSpaces(node.outerHTML);
+}
+
+/**
+ * Gets the content node from the document.
+ * @param {Document} document - The document object.
+ * @param {string} title - The title of the document.
+ * @param {Object} opts - The options for content extraction.
+ * @returns {Node} The content node.
+ */
+function getContentNode(document, title, opts) {
+  return cleanContent(extractBestNode(document, opts), {
+    document,
+    cleanConditionally: opts.cleanConditionally,
+    title,
+  });
+}
+
+/**
+ * Gets the score of a node.
+ * @param {Node} node - The node to get the score from.
+ * @returns {number|null} The score of the node or null if no score is set.
+ */
+function getScore(node) {
+  return parseFloat(node.getAttribute("score")) || null;
+}
+
+/**
+ * Scores the number of commas in a text.
+ * @param {string} text - The text to score.
+ * @returns {number} The number of commas in the text.
+ */
+function scoreCommas(text) {
+  return (text.match(/,/g) || []).length;
+}
+
+/**
+ * Converts span elements to div elements.
+ * @param {Node} node - The node to convert.
+ * @param {Document} document - The document object.
+ */
+function convertSpans(node, document) {
+  if (node?.tagName?.toLowerCase() === "span") {
+    // convert spans to divs
+    convertNodeTo(node, document, "div");
+  }
+}
+
+/**
+ * Adds a score to a node and its parent elements.
+ * @param {Node} node - The node to add the score to.
+ * @param {Document} document - The document object.
+ * @param {number} score - The score to add.
+ */
+function addScoreTo(node, document, score) {
+  if (node) {
+    convertSpans(node, document);
+    addScore(node, document, score);
+  }
+}
+
+/**
+ * Scores paragraph elements in the document.
+ * @param {Document} document - The document object.
+ * @param {boolean} weightNodes - Whether to weight nodes or not.
+ * @returns {Document} The document with scored paragraphs.
+ */
+function scorePs(document, weightNodes) {
+  document.querySelectorAll("p, pre").forEach((node) => {
+    if (!node.hasAttribute("score")) {
+      // The raw score for this paragraph, before we add any parent/child
+      // scores.
+      node = setScore(
+        node,
+        document,
+        getOrInitScore(node, document, weightNodes)
+      );
+
+      const parent = node.parentNode;
+      const rawScore = scoreNode(node);
+
+      addScoreTo(parent, document, rawScore, weightNodes);
+      if (parent) {
+        // Add half of the individual content score to the
+        // grandparent
+        addScoreTo(parent.parentNode, document, rawScore / 2, weightNodes);
+      }
+    }
+  });
+
+  return document;
+}
+
+/**
+ * Scores the content of the document.
+ * @param {Document} document - The document object.
+ * @param {boolean} weightNodes - Whether to weight nodes or not.
+ * @returns {Document} The document with scored content.
+ */
+function scoreContent(document, weightNodes = true) {
+  // First, look for special hNews based selectors and give them a big
+  // boost, if they exist
+  HNEWS_CONTENT_SELECTORS.forEach(([parentSelector, childSelector]) => {
+    document
+      .querySelectorAll(`${parentSelector} ${childSelector}`)
+      .forEach((node) => {
+        addScore(node.closest(parentSelector), document, 80);
+      });
+  });
+
+  // Doubling this again
+  // Previous solution caused a bug
+  // in which parents weren't retaining
+  // scores. This is not ideal, and
+  // should be fixed.
+  scorePs(document, weightNodes);
+  scorePs(document, weightNodes);
+
+  return document;
+}
+
+/**
+ * Scores the length of text.
+ * @param {number} textLength - The length of the text.
+ * @param {string} tagName - The tag name of the element.
+ * @returns {number} The score based on text length.
+ */
+function scoreLength(textLength, tagName = "p") {
+  const chunks = textLength / 50;
+
+  if (chunks > 0) {
+    let lengthBonus;
+
+    // No idea why p or pre are being tamped down here
+    // but just following the source for now
+    // Not even sure why tagName is included here,
+    // since this is only being called from the context
+    // of scoreParagraph
+    if (new RegExp("^(p|pre)$", "i").test(tagName)) {
+      lengthBonus = chunks - 2;
+    } else {
+      lengthBonus = chunks - 1.25;
+    }
+
+    return Math.min(Math.max(lengthBonus, 0), 3);
+  }
+
+  return 0;
+}
+
+/**
+ * Sets the score attribute of a node.
+ * @param {Node} node - The node to set the score on.
+ * @param {Document} document - The document object.
+ * @param {number} score - The score to set.
+ * @returns {Node} The node with the set score.
+ * @private
+ */
+export function setScore(node, document, score) {
+  node.setAttribute("score", score);
+  return node;
+}
+
+/**
+ * Scores a paragraph node.
+ * @param {Node} node - The paragraph node to score.
+ * @private
+ * @returns {number} The score of the paragraph.
+ */
+export function scoreParagraph(node) {
+  let score = 1;
+  const text = node.textContent.trim();
+  const textLength = text.length;
+
+  // If this paragraph is less than 25 characters, don't count it.
+  if (textLength < 25) {
+    return 0;
+  }
+
+  // Add points for any commas within this paragraph
+  score += scoreCommas(text);
+
+  // For every 50 characters in this paragraph, add another point. Up
+  // to 3 points.
+  score += scoreLength(textLength);
+
+  // Articles can end with short paragraphs when people are being clever
+  // but they can also end with short paragraphs setting up lists of junk
+  // that we strip. This negative tweaks junk setup paragraphs just below
+  // the cutoff threshold.
+  if (text.slice(-1) === ":") {
+    score -= 1;
+  }
+
+  return score;
+}
+
+// Score an individual node. Has some smarts for paragraphs, otherwise
+// just scores based on tag.
+function scoreNode(node) {
+  const tagName = node.tagName?.toLowerCase();
+  // if (!tagName) return 0;
+
+  // TODO: Consider ordering by most likely.
+  // E.g., if divs are a more common tag on a page,
+  // Could save doing that regex test on every node \u2013 AP
+  if (PARAGRAPH_SCORE_TAGS.test(tagName)) {
+    return scoreParagraph(node);
+  }
+  if (tagName === "div") {
+    return 5;
+  }
+  if (CHILD_CONTENT_TAGS.test(tagName)) {
+    return 3;
+  }
+  if (BAD_TAGS.test(tagName)) {
+    return -3;
+  }
+  if (tagName === "th") {
+    return -5;
+  }
+
+  return 0;
+}
+
+function addScore(node, document, amount) {
+  try {
+    const score = getOrInitScore(node, document) + amount;
+    setScore(node, document, score);
+  } catch (e) {
+    // Ignoring; error occurs in scoreNode
+  }
+
+  return node;
+}
+
+// Adds 1/4 of a child's score to its parent
+function addToParent(node, document, score) {
+  const parent = node.parentNode;
+  if (parent) {
+    addScore(parent, document, score * 0.25);
+  }
+
+  return node;
+}
+
+// Using a variety of scoring techniques, extract the content most
+// likely to be article text.
+//
+// If strip_unlikely_candidates is True, remove any elements that
+// match certain criteria first. (Like, does this element have a
+// classname of "comment")
+//
+// If weight_nodes is True, use classNames and IDs to determine the
+// worthiness of nodes.
+//
+// Returns a DOM node
+function extractBestNode(document, opts) {
+  if (opts.stripUnlikelyCandidates) {
+    document = stripUnlikelyCandidates(document);
+  }
+
+  document = convertToParagraphs(document);
+  document = scoreContent(document, opts.weightNodes);
+  const topCandidate = findTopCandidate(document);
+
+  return topCandidate;
+}
+
+// Clean our article content, returning a new, cleaned node.
+function cleanContent(
+  article,
+  { document, cleanConditionally = true, title = "", defaultCleaner = true }
+) {
+  // Rewrite the tag name to div if it's a top level node like body or
+  // html to avoid later complications with multiple body tags.
+  rewriteTopLevel(article, document);
+
+  // Drop small images and spacer images
+  // Only do this is defaultCleaner is set to true;
+  // this can sometimes be too aggressive.
+  if (defaultCleaner) cleanImages(article, document);
+
+  // Drop certain tags like <title>, etc
+  // This is -mostly- for cleanliness, not security.
+  stripJunkTags(article, document);
+
+  // H1 tags are typically the article title, which should be extracted
+  // by the title extractor instead. If there's less than 3 of them (<3),
+  // strip them. Otherwise, turn 'em into H2s.
+  cleanHOnes(article, document);
+
+  // Clean headers
+  cleanHeaders(article, document, title);
+
+  // We used to clean UL's and OL's here, but it was leading to
+  // too many in-article lists being removed. Consider a better
+  // way to detect menus particularly and remove them.
+  // Also optionally running, since it can be overly aggressive.
+  if (defaultCleaner) cleanTags(article, document, cleanConditionally);
+
+  // Remove empty paragraph nodes
+  removeEmpty(article, document);
+
+  // Remove unnecessary attributes
+  cleanAttributes(article, document);
+
+  return article;
+}
+
+// After we've calculated scores, loop through all of the possible
+// candidate nodes we found and find the one with the highest score.
+function findTopCandidate(document) {
+  let candidate;
+  let topScore = 0;
+
+  document.querySelectorAll("[score]").forEach((node) => {
+    // Ignore tags like BR, HR, etc
+    if (NON_TOP_CANDIDATE_TAGS_RE.test(node.tagName)) {
+      return;
+    }
+
+    const score = getScore(node);
+
+    if (score > topScore) {
+      topScore = score;
+      candidate = node;
+    }
+  });
+
+  // If we don't have a candidate, return the body
+  // or whatever the first element is
+  if (!candidate) {
+    return document.body || document.querySelector("*");
+  }
+
+  candidate = mergeSiblings(candidate, topScore, document);
+
+  return candidate;
+}
+
+// gets and returns the score if it exists
+// if not, initializes a score based on
+// the node's tag type
+function getOrInitScore(node, document, weightNodes = true) {
+  let score = getScore(node);
+
+  if (score) {
+    return score;
+  }
+
+  score = scoreNode(node);
+
+  if (weightNodes) {
+    score += getWeight(node);
+  }
+
+  addToParent(node, document, score);
+
+  return score;
+}
+
+// Get the score of a node based on its className and id.
+function getWeight(node) {
+  const classes = node.getAttribute("class");
+  const id = node.getAttribute("id");
+  let score = 0;
+
+  if (id) {
+    // if id exists, try to score on both positive and negative
+    if (POSITIVE_SCORE_RE.test(id)) {
+      score += 25;
+    }
+    if (NEGATIVE_SCORE_RE.test(id)) {
+      score -= 25;
+    }
+  }
+
+  if (classes) {
+    if (score === 0) {
+      // if classes exist and id did not contribute to score
+      // try to score on both positive and negative
+      if (POSITIVE_SCORE_RE.test(classes)) {
+        score += 25;
+      }
+      if (NEGATIVE_SCORE_RE.test(classes)) {
+        score -= 25;
+      }
+    }
+
+    // even if score has been set by id, add score for
+    // possible photo matches
+    // "try to keep photos if we can"
+    if (PHOTO_HINTS_RE.test(classes)) {
+      score += 10;
+    }
+
+    // add 25 if class matches entry-content-asset,
+    // a class apparently instructed for use in the
+    // Readability publisher guidelines
+    // https://www.readability.com/developers/guidelines
+    if (READABILITY_ASSET.test(classes)) {
+      score += 25;
+    }
+  }
+
+  return score;
+}
+
+/**
+ * Checks if a given text appears to have an ending sentence within it.
+ * @param {string} text - The text to check for sentence endings.
+ * @returns {boolean} True if the text appears to have an ending sentence, false otherwise.
+ */
+function hasSentenceEnd(text) {
+  return new RegExp(".( |$)").test(text);
+}
+
+/**
+ * Merges siblings of the top candidate that are decently scored.
+ * This function looks through the siblings of the top candidate to see if any of them
+ * are decently scored. If they are, they may be split parts of the content
+ * (like two divs, a preamble and a body).
+ *
+ * @param {Element} candidate - The top candidate element.
+ * @param {number} topScore - The score of the top candidate.
+ * @param {Document} document - The document object.
+ * @returns {Element} The candidate element, potentially with merged siblings.
+ */
+function mergeSiblings(candidate, topScore, document) {
+  if (!candidate.parentNode) {
+    return candidate;
+  }
+
+  const siblingScoreThreshold = Math.max(10, topScore * 0.25);
+  const wrappingDiv = document.createElement("div");
+
+  Array.from(candidate.parentNode.children).forEach((sibling) => {
+    // Ignore tags like BR, HR, etc
+    if (NON_TOP_CANDIDATE_TAGS_RE.test(sibling.tagName)) {
+      return null;
+    }
+
+    const siblingScore = getScore(sibling);
+    if (siblingScore) {
+      if (sibling === candidate) {
+        wrappingDiv.appendChild(sibling);
+      } else {
+        let contentBonus = 0;
+        const density = linkDensity(sibling);
+
+        // If sibling has a very low link density,
+        // give it a small bonus
+        if (density < 0.05) {
+          contentBonus += 20;
+        }
+
+        // If sibling has a high link density,
+        // give it a penalty
+        if (density >= 0.5) {
+          contentBonus -= 20;
+        }
+
+        // If sibling node has the same class as
+        // candidate, give it a bonus
+        if (sibling.getAttribute("class") === candidate.getAttribute("class")) {
+          contentBonus += topScore * 0.2;
+        }
+
+        const newScore = siblingScore + contentBonus;
+
+        if (newScore >= siblingScoreThreshold) {
+          return wrappingDiv.appendChild(sibling);
+        }
+        if (sibling.tagName === "P") {
+          const siblingContent = sibling.textContent;
+          const siblingContentLength = textLength(siblingContent);
+
+          if (siblingContentLength > 80 && density < 0.25) {
+            return wrappingDiv.appendChild(sibling);
+          }
+          if (
+            siblingContentLength <= 80 &&
+            density === 0 &&
+            hasSentenceEnd(siblingContent)
+          ) {
+            return wrappingDiv.appendChild(sibling);
+          }
+        }
+      }
+    }
+
+    return null;
+  });
+
+  if (
+    wrappingDiv.children.length === 1 &&
+    wrappingDiv?.firstElementChild === candidate
+  ) {
+    return candidate;
+  }
+
+  return wrappingDiv;
+}
+
+function cleanTags(article, document) {
+  const CLEAN_CONDITIONALLY_TAGS = [
+    "ul",
+    "ol",
+    "table",
+    "div",
+    "button",
+    "form",
+  ];
+  CLEAN_CONDITIONALLY_TAGS.forEach((tag) => {
+    article.querySelectorAll(tag).forEach((node) => {
+      const KEEP_CLASS = "parser-keep";
+
+      if (
+        node.classList.contains(KEEP_CLASS) ||
+        node.querySelector(`.${KEEP_CLASS}`)
+      )
+        return;
+
+      let weight = getScore(node);
+      if (!weight) {
+        weight = getOrInitScore(node, document);
+        setScore(node, document, weight);
+      }
+
+      if (weight < 0) {
+        node.parentNode.removeChild(node);
+      } else {
+        removeUnlessContent(node, document, weight);
+      }
+    });
+  });
+
+  return document;
+}
+
+function cleanHeaders(article, document, title = "") {
+  const HEADER_TAGS = ["h2", "h3", "h4", "h5", "h6"];
+  HEADER_TAGS.forEach((tag) => {
+    article.querySelectorAll(tag).forEach((header) => {
+      if (
+        header.previousElementSibling &&
+        header.previousElementSibling.tagName !== "P"
+      ) {
+        header.parentNode.removeChild(header);
+        return;
+      }
+
+      if (normalizeSpaces(header.textContent) === title) {
+        header.parentNode.removeChild(header);
+        return;
+      }
+
+      if (getWeight(header) < 0) {
+        header.parentNode.removeChild(header);
+        return;
+      }
+    });
+  });
+
+  return document;
+}