@purepageio/fetch-engines 0.2.4 → 0.2.6

package/dist/index.cjs

@@ -0,0 +1,1657 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  FetchEngine: () => FetchEngine,
+  HybridEngine: () => HybridEngine,
+  PlaywrightEngine: () => PlaywrightEngine
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/utils/markdown-converter.ts
+var import_turndown = __toESM(require("turndown"), 1);
+var import_turndown_plugin_gfm = require("turndown-plugin-gfm");
+var import_node_html_parser = require("node-html-parser");
+var PREPROCESSING_REMOVE_SELECTORS = [
+  "script:not([type='application/ld+json'])",
+  // Keep JSON-LD
+  "style",
+  "noscript",
+  "iframe:not([title])"
+  // Keep iframes with titles (potential embeds)
+];
+var MAIN_CONTENT_SELECTORS = [
+  // By semantics
+  "article",
+  "main",
+  "[role='main']",
+  "[role='article']",
+  // By common class/id names (more robust patterns)
+  "[class*='article-body']",
+  "[class*='post-content']",
+  "[class*='main-content']",
+  "[class*='entry-content']",
+  "[id*='article-body']",
+  "[id*='main-content']",
+  // Common CMS patterns
+  ".article",
+  ".post",
+  ".content",
+  ".entry",
+  ".blog-post",
+  // Fallback
+  "body"
+];
+var FORUM_COMMENT_SELECTORS = [
+  ".comment",
+  ".comments",
+  ".comtr",
+  '[id^="comment-"]',
+  'div[id^="c_"]'
+];
+var FORUM_THREAD_SELECTORS = [".thread", ".post", '[id^="thread-"]'];
+var FORUM_VOTE_SELECTORS = [".vote", ".score", ".upvote", ".downvote", ".votelinks"];
+var FORUM_MAIN_POST_SELECTORS = [".fatitem", ".submission", ".op", ".original-post"];
+var FORUM_COMMENTS_CONTAINER_SELECTORS = [".comment-tree", ".comments", "#comments"];
+var FORUM_OBVIOUS_NON_CONTENT_SELECTORS = ["header", "footer", ".nav", ".sidebar"];
+var MIN_LINK_DENSITY_TEXT_LENGTH = 50;
+var DEFAULT_LINK_DENSITY_THRESHOLD = 0.4;
+var MIN_FORUM_INDICATOR_COUNT = 3;
+var CODE_BLOCK_LANG_PREFIXES = ["language-", "lang-"];
+var POSTPROCESSING_MAX_CONSECUTIVE_NEWLINES = 2;
+var MarkdownConverter = class {
+  turndownService;
+  constructor() {
+    this.turndownService = new import_turndown.default({
+      headingStyle: "atx",
+      codeBlockStyle: "fenced",
+      bulletListMarker: "-",
+      strongDelimiter: "**",
+      emDelimiter: "*",
+      hr: "---",
+      // Use nodeType check instead of window.HTMLElement
+      keepReplacement: (_content, node) => {
+        if (node.nodeType === 1) {
+          const htmlElement = node;
+          if (htmlElement.getAttribute("role") === "presentation" || htmlElement.classList?.contains("preserve")) {
+            return htmlElement.outerHTML;
+          }
+        }
+        return "";
+      }
+    });
+    this.turndownService.use(import_turndown_plugin_gfm.gfm);
+    this.setupPrioritizedRules();
+  }
+  // --- Public Method ---
+  /**
+   * Converts HTML string to Markdown.
+   * @param html The HTML string to convert.
+   * @param options Conversion options.
+   * @returns The converted Markdown string.
+   */
+  convert(html, options = {}) {
+    const preprocessedHtml = this.preprocessHTML(html);
+    let markdown = this.turndownService.turndown(preprocessedHtml);
+    markdown = this.postprocessMarkdown(markdown, options);
+    return markdown;
+  }
+  // --- Turndown Rule Setup ---
+  setupPrioritizedRules() {
+    this.addContentExtractionRules();
+    this.addStructureRules();
+    this.addBlockRules();
+    this.addInlineRules();
+  }
+  // We rely on preprocessing to remove nav/menus/high-link-density areas.
+  // These rules primarily help Turndown understand the *structure* of the *intended* content.
+  addContentExtractionRules() {
+    this.turndownService.addRule("main-content-marker", {
+      filter: (node) => {
+        if (node.nodeType !== 1) return false;
+        const el = node;
+        const element = node;
+        return el.tagName.toLowerCase() === "main" || ["main", "article"].includes(el.getAttribute("role") || "") || MAIN_CONTENT_SELECTORS.some((selector) => {
+          try {
+            return element.matches(selector) && selector !== "body";
+          } catch {
+            return false;
+          }
+        });
+      },
+      // Just pass content through, this rule is mainly for filter priority/debugging
+      replacement: (content) => content
+    });
+    const unwantedTags = [
+      "script",
+      "style",
+      "noscript",
+      "iframe",
+      "button",
+      "input",
+      "select",
+      "textarea",
+      "form",
+      "canvas",
+      /*'svg' removed */
+      "audio",
+      "video"
+    ];
+    this.turndownService.addRule("remove-unwanted", {
+      filter: unwantedTags,
+      replacement: () => ""
+    });
+  }
+  addStructureRules() {
+    this.turndownService.addRule("article", {
+      filter: "article",
+      replacement: (content) => `
+
+${content}
+
+`
+      // Add separation
+    });
+    this.turndownService.addRule("section", {
+      filter: "section",
+      replacement: (content) => `
+
+${content}
+
+`
+      // Add separation
+    });
+  }
+  addBlockRules() {
+    this.turndownService.addRule("list", {
+      filter: ["ul", "ol"],
+      replacement: (content, node) => {
+        if (node.nodeType !== 1) return content;
+        const parent = node.parentNode;
+        const indent = parent && parent.nodeName.toLowerCase() === "li" ? "  " : "";
+        return "\n" + content.split("\n").map((line) => indent + line.trimEnd()).join("\n").trim() + "\n";
+      }
+    });
+    this.turndownService.addRule("listItem", {
+      filter: "li",
+      // Use standard function for `this` context if needed, or ensure types match
+      replacement: function(content, node, options) {
+        content = content.replace(/^\s+/gm, "").replace(/\n(?!\s*$)/gm, "\n  ");
+        let prefix = options.bulletListMarker + " ";
+        const parentNode = node.parentNode;
+        if (parentNode && parentNode.nodeName === "OL") {
+          try {
+            const start = parentNode.getAttribute("start");
+            const elementNode = node;
+            const parentElement = parentNode;
+            const index = Array.prototype.indexOf.call(parentElement.children, elementNode);
+            prefix = (start ? Number(start) + index : index + 1) + ". ";
+          } catch (e) {
+            console.warn("Could not determine ordered list index:", e);
+            prefix = "1. ";
+          }
+        }
+        const trimmedContent = content.trim();
+        return prefix + trimmedContent + (node.nextSibling && !/\n$/.test(trimmedContent) ? "\n" : "");
+      }
+    });
+    this.turndownService.addRule("blockquote", {
+      filter: "blockquote",
+      replacement: (content) => {
+        const trimmedContent = content.trim();
+        return "\n\n> " + trimmedContent.replace(/\n/g, "\n> ") + "\n\n";
+      }
+    });
+  }
+  addInlineRules() {
+    this.turndownService.addRule("link", {
+      filter: (node, _options) => {
+        return node.nodeType === 1 && node.nodeName === "A" && !!node.getAttribute("href");
+      },
+      replacement: (content, node) => {
+        const element = node;
+        const href = element.getAttribute("href") || "";
+        const title = element.getAttribute("title");
+        const text = content.trim() ? content.trim() : href;
+        let decodedHref = href;
+        try {
+          if (href.includes("%")) {
+            decodedHref = decodeURI(href);
+          }
+        } catch (e) {
+          console.warn(`Failed to decode URI, keeping original: ${href}`, e);
+        }
+        return title ? `[${text}](${decodedHref} "${title}")` : `[${text}](${decodedHref})`;
+      }
+    });
+    this.turndownService.addRule("figure", {
+      filter: "figure",
+      replacement: (content, node) => {
+        if (node.nodeType !== 1) return content;
+        const element = node;
+        const img = element.querySelector("img");
+        const figcaption = element.querySelector("figcaption");
+        let markdown = "";
+        let mainImgMd = "";
+        if (img) {
+          const src = img.getAttribute("src") || "";
+          const alt = img.getAttribute("alt") || "";
+          const title = img.getAttribute("title");
+          mainImgMd = title ? `![${alt}](${src} "${title}")` : `![${alt}](${src})`;
+        }
+        let processedContent = content.trim();
+        if (mainImgMd) {
+          markdown = mainImgMd;
+          const imgPlaceholder = `![${img?.getAttribute("alt") || ""}](${img?.getAttribute("src") || ""})`;
+          processedContent = processedContent.replace(imgPlaceholder, "").trim();
+        }
+        if (figcaption) {
+          const captionText = figcaption.textContent?.trim();
+          if (captionText) {
+            markdown += `
+
+_${captionText}_`;
+            processedContent = processedContent.replace(captionText, "").trim();
+            processedContent = processedContent.replace(/^_+|_+$/g, "").trim();
+          }
+        }
+        if (processedContent) {
+          if (processedContent.length > 10 || /[a-zA-Z0-9]/.test(processedContent)) {
+            markdown += `
+
+${processedContent}`;
+          }
+        }
+        return "\n\n" + markdown.trim() + "\n\n";
+      }
+    });
+    this.turndownService.addRule("image", {
+      filter: (node) => {
+        return node.nodeType === 1 && node.nodeName === "IMG" && !!node.getAttribute("src");
+      },
+      replacement: (_content, node) => {
+        const element = node;
+        const src = element.getAttribute("src") || "";
+        const alt = element.getAttribute("alt") || "";
+        const title = element.getAttribute("title");
+        return title ? `
+
+![${alt}](${src} "${title}")
+
+` : `
+
+![${alt}](${src})
+
+`;
+      }
+    });
+    this.turndownService.addRule("code-block", {
+      filter: (node) => {
+        if (node.nodeType !== 1) return false;
+        const element = node;
+        const isPre = element.tagName.toLowerCase() === "pre";
+        if (!isPre) return false;
+        const hasCodeChild = element.querySelector("code") !== null;
+        const hasCodeClass = /highlight|syntax|code|listing|source/i.test(element.className);
+        const hasLangAttribute = !!element.getAttribute("lang") || !!element.getAttribute("language");
+        return hasCodeChild || hasCodeClass || hasLangAttribute;
+      },
+      replacement: (content, node) => {
+        if (node.nodeType !== 1) return content.trim();
+        const element = node;
+        let language = "";
+        const codeElement = element.querySelector("code");
+        language = element.getAttribute("lang") || element.getAttribute("language") || (codeElement ? codeElement.getAttribute("lang") || codeElement.getAttribute("language") : "") || "";
+        if (!language) {
+          const classes = (element.className + " " + (codeElement?.className || "")).split(" ").filter(Boolean);
+          for (const cls of classes) {
+            for (const prefix of CODE_BLOCK_LANG_PREFIXES) {
+              if (cls.startsWith(prefix)) {
+                language = cls.substring(prefix.length);
+                break;
+              }
+            }
+            if (language) break;
+          }
+        }
+        const cleanedContent = content.trim();
+        return `
+
+\`\`\`${language}
+${cleanedContent}
+\`\`\`
+
+`;
+      }
+    });
+    this.turndownService.addRule("inlineCode", {
+      filter: (node) => node.nodeName === "CODE" && node.parentNode?.nodeName !== "PRE",
+      replacement: (content) => {
+        const trimmed = content.trim();
+        if (!trimmed) return "";
+        let delimiter = "`";
+        if (trimmed.includes("`")) {
+          delimiter = "``";
+          if (trimmed.startsWith("`") || trimmed.endsWith("`")) {
+            return `${delimiter} ${trimmed} ${delimiter}`;
+          }
+        }
+        return delimiter + trimmed + delimiter;
+      }
+    });
+  }
+  // --- HTML Preprocessing ---
+  preprocessHTML(html) {
+    try {
+      html = this.cleanupHtml(html);
+      const root = (0, import_node_html_parser.parse)(html, {
+        comment: false,
+        blockTextElements: { script: true, style: true, noscript: true }
+      });
+      if (root.nodeType === 3) {
+        return root.textContent ?? "";
+      } else if (root.nodeType !== 1) {
+        console.warn("Unexpected root node type after parsing:", root.nodeType);
+        return root.toString();
+      }
+      const rootElement = root;
+      PREPROCESSING_REMOVE_SELECTORS.forEach((selector) => {
+        try {
+          rootElement.querySelectorAll(selector).forEach((el) => el.remove());
+        } catch (e) {
+          console.warn(`Skipping invalid selector during preprocessing: ${selector}`, e);
+        }
+      });
+      this.removeHighLinkDensityElements(rootElement, DEFAULT_LINK_DENSITY_THRESHOLD);
+      const metadata = this.extractDocumentMetadata(rootElement);
+      const isForum = this.detectForumPage(rootElement);
+      let contentElement = rootElement;
+      if (isForum) {
+        contentElement = this.extractForumContentElement(rootElement);
+      } else {
+        contentElement = this.extractArticleContentElement(rootElement);
+      }
+      let contentHtml = contentElement instanceof import_node_html_parser.HTMLElement ? contentElement.outerHTML : contentElement.textContent;
+      contentHtml = this.cleanupContentHtml(contentHtml || "");
+      const metadataString = metadata.length > 0 ? metadata.join("\n\n") + "\n\n---\n\n" : "";
+      return metadataString + contentHtml;
+    } catch (error) {
+      console.error("HTML preprocessing failed:", error);
+      return this.cleanupHtml(html);
+    }
+  }
+  cleanupHtml(html) {
+    return html.replace(/AMIL:\[=-,amilft[^\s]*/g, "").replace(/\{\{\s*[^}\s]+\s*}}/g, "").replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "");
+  }
+  cleanupContentHtml(content) {
+    return content.replace(/\s*data-(?:reactid|reactroot|react-|testid|v-|js-|qa-|cy-)[^=\s]*\s*=\s*(?:"[^"]*"|'[^']*'|\S+)/g, "").replace(/\s*ng-[^=\s]*\s*=\s*(?:"[^"]*"|'[^']*'|\S+)/g, "").replace(/\s*_ngcontent-[^\s]*\s*=""/g, "").replace(/\s*class\s*=\s*"(ng-|mat-)[^"]*"/g, "").replace(/<!--[\s\S]*?-->/g, "").replace(/([ \t])+/g, " ").replace(/\s*\n\s*/g, "\n").trim();
+  }
+  removeHighLinkDensityElements(element, threshold) {
+    const potentialBoilerplate = element.querySelectorAll(
+      "div, nav, ul, aside, section, .sidebar, .widget, .menu, [role='navigation'], [role='menubar']"
+    );
+    for (const el of Array.from(potentialBoilerplate)) {
+      if (!(el instanceof import_node_html_parser.HTMLElement)) continue;
+      const textContent = el.textContent || "";
+      if (textContent.length < MIN_LINK_DENSITY_TEXT_LENGTH) continue;
+      const links = el.querySelectorAll("a");
+      if (links.length < 3) continue;
+      const textLength = textContent.length;
+      let linkTextLength = 0;
+      el.querySelectorAll("a").forEach((link) => {
+        if (link.closest("a") === link) {
+          linkTextLength += link.textContent?.length || 0;
+        }
+      });
+      if (textLength === 0) continue;
+      const density = linkTextLength / textLength;
+      if (density > threshold) {
+        const containsMainContent = el.querySelector('main, article, [role="main"], [role="article"]') !== null;
+        const isMainContent = MAIN_CONTENT_SELECTORS.some((selector) => {
+          try {
+            return el.matches(selector);
+          } catch {
+            return false;
+          }
+        });
+        if (!containsMainContent && !isMainContent) {
+          el.remove();
+        }
+      }
+    }
+  }
+  extractDocumentMetadata(root) {
+    const metadata = [];
+    const addedMeta = /* @__PURE__ */ new Set();
+    const addMeta = (key, value, isTitle = false) => {
+      const cleanedValue = value?.trim();
+      if (cleanedValue && !addedMeta.has(key.toLowerCase())) {
+        if (isTitle) {
+          metadata.unshift(`# ${cleanedValue}`);
+        } else {
+          metadata.push(`**${key}:** ${cleanedValue}`);
+        }
+        addedMeta.add(key.toLowerCase());
+      }
+    };
+    addMeta("Title", root.querySelector("meta[property='og:title']")?.getAttribute("content"), true);
+    addMeta("Title", root.querySelector("meta[name='twitter:title']")?.getAttribute("content"), true);
+    addMeta("Title", root.querySelector("meta[name='DC.title']")?.getAttribute("content"), true);
+    addMeta("Title", root.querySelector("title")?.textContent, true);
+    addMeta("Description", root.querySelector("meta[property='og:description']")?.getAttribute("content"));
+    addMeta("Description", root.querySelector("meta[name='twitter:description']")?.getAttribute("content"));
+    addMeta("Description", root.querySelector("meta[name='description']")?.getAttribute("content"));
+    addMeta("Description", root.querySelector("meta[name='DC.description']")?.getAttribute("content"));
+    addMeta("Author", root.querySelector("meta[name='author']")?.getAttribute("content"));
+    addMeta("Author", root.querySelector("meta[property='article:author']")?.getAttribute("content"));
+    addMeta("Author", root.querySelector("[rel='author']")?.textContent);
+    addMeta("Published", root.querySelector("meta[property='article:published_time']")?.getAttribute("content"));
+    addMeta("Published", root.querySelector("meta[name='publish-date']")?.getAttribute("content"));
+    addMeta("Published", root.querySelector("time[itemprop='datePublished']")?.getAttribute("datetime"));
+    addMeta("Published", root.querySelector("time")?.getAttribute("datetime"));
+    addMeta("URL", root.querySelector("link[rel='canonical']")?.getAttribute("href"));
+    addMeta("URL", root.querySelector("meta[property='og:url']")?.getAttribute("content"));
+    const jsonLdScripts = root.querySelectorAll("script[type='application/ld+json']");
+    if (jsonLdScripts.length > 0) {
+      const jsonLdData = Array.from(jsonLdScripts).map((script) => {
+        try {
+          const textContent = script.textContent;
+          return textContent ? JSON.parse(textContent) : null;
+        } catch (e) {
+          return null;
+        }
+      }).filter((item) => item !== null);
+      if (jsonLdData.length > 0 && !addedMeta.has("json-ld")) {
+        metadata.push("<details><summary>JSON-LD Metadata</summary>\n");
+        metadata.push("```json", JSON.stringify(jsonLdData, null, 2), "```");
+        metadata.push("</details>");
+        addedMeta.add("json-ld");
+      }
+    }
+    return metadata;
+  }
+  detectForumPage(root) {
+    const countMatches = (selectors) => {
+      return selectors.reduce((count, selector) => {
+        try {
+          if (root) {
+            return count + root.querySelectorAll(selector).length;
+          }
+          return count;
+        } catch {
+          return count;
+        }
+      }, 0);
+    };
+    const commentCount = countMatches(FORUM_COMMENT_SELECTORS);
+    const threadCount = countMatches(FORUM_THREAD_SELECTORS);
+    const voteCount = countMatches(FORUM_VOTE_SELECTORS);
+    let isKnownForumHost = false;
+    try {
+      const canonicalUrl = root.querySelector('link[rel="canonical"]')?.getAttribute("href") || root.querySelector('meta[property="og:url"]')?.getAttribute("content");
+      if (canonicalUrl) {
+        const absoluteUrl = new URL(canonicalUrl, "http://example.com").toString();
+        const hostname = new URL(absoluteUrl).hostname.toLowerCase();
+        isKnownForumHost = hostname.includes("reddit.com") || hostname.includes("news.ycombinator.com") || hostname.includes("forum") || hostname.includes("discuss") || hostname.includes("community");
+      }
+    } catch (e) {
+      console.warn("Could not parse URL for forum detection:", e);
+    }
+    return commentCount >= MIN_FORUM_INDICATOR_COUNT || threadCount > 1 || // More than one thread item is stronger indicator
+    voteCount >= MIN_FORUM_INDICATOR_COUNT || isKnownForumHost;
+  }
+  // Tries to find the main content element for an article-like page
+  extractArticleContentElement(root) {
+    let bestCandidate = null;
+    let maxScore = -1;
+    for (const selector of MAIN_CONTENT_SELECTORS) {
+      try {
+        const elements = root.querySelectorAll(selector);
+        for (const element of Array.from(elements)) {
+          if (!(element instanceof import_node_html_parser.HTMLElement)) continue;
+          const textLength = (element.textContent || "").trim().length;
+          if (textLength < 100 && !element.querySelector("img, video, iframe, figure")) continue;
+          let score = textLength;
+          if (["ARTICLE", "MAIN"].includes(element.tagName)) score *= 1.5;
+          if (["main", "article"].includes(element.getAttribute("role") || "")) score *= 1.5;
+          if (["HEADER", "FOOTER", "NAV", "ASIDE"].includes(element.tagName)) score *= 0.3;
+          try {
+            if (
+              /* @ts-expect-error TODO: fix this */
+              element.matches(
+                '.sidebar, .widget, .menu, .nav, .header, .footer, [role="navigation"], [role="complementary"], [role="banner"]'
+              )
+            )
+              score *= 0.2;
+          } catch {
+          }
+          if (this.hasHighLinkDensity(element, 0.6)) {
+            score *= 0.5;
+          }
+          if (element.querySelectorAll("p").length > 2) score *= 1.2;
+          if (element.tagName === "BODY" && maxScore > 200) continue;
+          if (score > maxScore) {
+            maxScore = score;
+            bestCandidate = element;
+          }
+        }
+      } catch (e) {
+      }
+    }
+    return bestCandidate || root;
+  }
+  // Tries to find the main content element(s) for a forum-like page
+  extractForumContentElement(root) {
+    const tempContainer = (0, import_node_html_parser.parse)("<div></div>").firstChild;
+    try {
+      const mainPost = FORUM_MAIN_POST_SELECTORS.map((s) => root.querySelector(s)).find(
+        (el) => el instanceof import_node_html_parser.HTMLElement
+      );
+      if (mainPost) {
+        tempContainer.appendChild(mainPost.clone());
+      }
+    } catch (e) {
+      console.warn("Error finding forum main post:", e);
+    }
+    try {
+      const commentsContainer = FORUM_COMMENTS_CONTAINER_SELECTORS.map((s) => root.querySelector(s)).find(
+        (el) => el instanceof import_node_html_parser.HTMLElement
+      );
+      if (commentsContainer) {
+        const clonedComments = commentsContainer.clone();
+        if (clonedComments instanceof import_node_html_parser.HTMLElement) {
+          FORUM_OBVIOUS_NON_CONTENT_SELECTORS.forEach((selector) => {
+            try {
+              clonedComments.querySelectorAll(selector).forEach((el) => el.remove());
+            } catch {
+            }
+          });
+          tempContainer.appendChild(clonedComments);
+        }
+      }
+    } catch (e) {
+      console.warn("Error finding forum comments container:", e);
+    }
+    if (tempContainer.childNodes.length > 0) {
+      return tempContainer;
+    }
+    const body = root.querySelector("body");
+    if (body) {
+      const clonedBody = body.clone();
+      if (clonedBody instanceof import_node_html_parser.HTMLElement) {
+        FORUM_OBVIOUS_NON_CONTENT_SELECTORS.forEach((selector) => {
+          try {
+            clonedBody.querySelectorAll(selector).forEach((el) => el.remove());
+          } catch {
+          }
+        });
+        this.removeHighLinkDensityElements(clonedBody, DEFAULT_LINK_DENSITY_THRESHOLD);
+        return clonedBody;
+      }
+    }
+    return root;
+  }
+  // Helper function to check link density within an element
+  hasHighLinkDensity(element, threshold) {
+    const textContent = element.textContent || "";
+    if (textContent.length < MIN_LINK_DENSITY_TEXT_LENGTH) return false;
+    const links = element.querySelectorAll("a");
+    if (links.length < 3) return false;
+    const textLength = textContent.length;
+    let linkTextLength = 0;
+    element.querySelectorAll("a").forEach((link) => {
+      if (link.closest("a") === link) {
+        linkTextLength += link.textContent?.length || 0;
+      }
+    });
+    if (textLength === 0) return false;
+    return linkTextLength / textLength > threshold;
+  }
+  // --- Markdown Postprocessing ---
+  postprocessMarkdown(markdown, options) {
+    let processed = markdown;
+    processed = processed.replace(/^(\s*\n)?(#{1,6}\s.*)$/gm, "\n\n$2\n\n");
+    processed = processed.replace(/^(\s*\n)?(([\*\-+>]|\d+\.)\s)/gm, (_match, _p1, p2) => `
+
+${p2}`);
+    processed = processed.replace(
+      /(\n([\*\-+]|\d+\.)\s(?:(?!\n\n|\n {2,}|\n\t)[\s\S])*?)\n(?=([\*\-+]|\d+\.)\s)/g,
+      "$1"
+    );
+    processed = processed.replace(/\[\]\([^)]*\)/g, "");
+    processed = processed.replace(/!\[\]\([^)]*\)/g, "");
+    processed = processed.replace(/(!?\[[^\]]*\]\()(\/\/)/g, "$1https://");
+    const maxNewlines = "\n".repeat(POSTPROCESSING_MAX_CONSECUTIVE_NEWLINES + 1);
+    const newlineRegex = new RegExp(`${maxNewlines}+`, "g");
+    processed = processed.replace(newlineRegex, "\n".repeat(POSTPROCESSING_MAX_CONSECUTIVE_NEWLINES));
+    processed = processed.replace(/^[ \t]+|[ \t]+$/gm, "");
+    processed = processed.replace(/^(\s*\n)?(```(.*)\n[\s\S]*?\n```)(\s*\n)?/gm, "\n\n$2\n\n");
+    processed = processed.replace(/^(.{30,})$(\n\1)+/gm, "$1");
+    processed = processed.replace(/(\n---\n)(\S)/g, "$1\n$2");
+    if (options.maxContentLength && processed.length > options.maxContentLength) {
+      const truncatedPoint = processed.lastIndexOf(".", options.maxContentLength - 15);
+      const sliceEnd = truncatedPoint > options.maxContentLength / 2 ? truncatedPoint + 1 : options.maxContentLength;
+      processed = processed.slice(0, sliceEnd) + "... (truncated)";
+    }
+    return processed.trim();
+  }
+};
+
+// src/errors.ts
+var FetchError = class _FetchError extends Error {
+  /** A specific error code (e.g., ERR_NAVIGATION_TIMEOUT, ERR_HTTP_ERROR). */
+  code;
+  /** The original error object, if available. */
+  originalError;
+  /** HTTP status code, if relevant. */
+  statusCode;
+  /**
+   * Creates an instance of FetchError.
+   * @param message The error message.
+   * @param code Optional error code string.
+   * @param originalError Optional original error.
+   * @param statusCode Optional HTTP status code.
+   */
+  constructor(message, code, originalError, statusCode) {
+    super(message);
+    this.name = "FetchError";
+    this.code = code;
+    this.originalError = originalError;
+    this.statusCode = statusCode;
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, _FetchError);
+    }
+  }
+};
+
+// src/FetchEngine.ts
+var FetchEngineHttpError = class extends FetchError {
+  constructor(message, statusCode) {
+    super(message, "ERR_HTTP_ERROR", void 0, statusCode);
+    this.statusCode = statusCode;
+    this.name = "FetchEngineHttpError";
+  }
+};
+var FetchEngine = class _FetchEngine {
+  options;
+  static DEFAULT_OPTIONS = {
+    markdown: false
+  };
+  /**
+   * Creates an instance of FetchEngine.
+   * @param options Configuration options for the FetchEngine.
+   */
+  constructor(options = {}) {
+    this.options = { ..._FetchEngine.DEFAULT_OPTIONS, ...options };
+  }
+  /**
+   * Fetches HTML or converts to Markdown from the specified URL.
+   *
+   * @param url The URL to fetch.
+   * @returns A Promise resolving to an HTMLFetchResult object.
+   * @throws {FetchEngineHttpError} If the HTTP response status is not ok (e.g., 404, 500).
+   * @throws {Error} If the content type is not HTML or for other network errors.
+   */
+  async fetchHTML(url, options) {
+    const effectiveOptions = { ...this.options, ...options };
+    let response;
+    try {
+      response = await fetch(url, {
+        redirect: "follow",
+        headers: {
+          // Standard browser-like headers
+          "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+          Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+          "Accept-Language": "en-US,en;q=0.9"
+        }
+      });
+      if (!response.ok) {
+        throw new FetchEngineHttpError(`HTTP error! status: ${response.status}`, response.status);
+      }
+      const contentTypeHeader = response.headers.get("content-type");
+      if (!contentTypeHeader || !contentTypeHeader.includes("text/html")) {
+        throw new FetchError("Content-Type is not text/html", "ERR_NON_HTML_CONTENT");
+      }
+      const html = await response.text();
+      const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
+      const title = titleMatch ? titleMatch[1].trim() : null;
+      let finalContent = html;
+      let finalContentType = "html";
+      if (effectiveOptions.markdown) {
+        try {
+          const converter = new MarkdownConverter();
+          finalContent = converter.convert(html);
+          finalContentType = "markdown";
+        } catch (conversionError) {
+          console.error(`Markdown conversion failed for ${url} (FetchEngine):`, conversionError);
+        }
+      }
+      return {
+        content: finalContent,
+        contentType: finalContentType,
+        title,
+        url: response.url,
+        // Use the final URL after redirects
+        isFromCache: false,
+        statusCode: response.status,
+        error: void 0
+      };
+    } catch (error) {
+      if (error instanceof FetchEngineHttpError || error instanceof FetchError && error.code === "ERR_NON_HTML_CONTENT") {
+        throw error;
+      }
+      const message = error instanceof Error ? error.message : "Unknown fetch error";
+      throw new FetchError(`Fetch failed: ${message}`, "ERR_FETCH_FAILED", error instanceof Error ? error : void 0);
+    }
+  }
+  /**
+   * Cleans up resources used by the engine.
+   * For FetchEngine, this is a no-op as it doesn't manage persistent resources.
+   * @returns A Promise that resolves when cleanup is complete.
+   */
+  async cleanup() {
+    return Promise.resolve();
+  }
+  /**
+   * Retrieves metrics for the engine.
+   * FetchEngine does not manage browsers, so it returns an empty array.
+   * @returns An empty array.
+   */
+  getMetrics() {
+    return [];
+  }
+};
+
+// src/browser/PlaywrightBrowserPool.ts
+var import_playwright = require("playwright");
+var import_user_agents = __toESM(require("user-agents"), 1);
+var import_uuid = require("uuid");
+var import_p_queue = __toESM(require("p-queue"), 1);
+var import_playwright_extra = require("playwright-extra");
+var chromiumWithExtras;
+var StealthPluginInstance;
+async function loadDependencies() {
+  if (!chromiumWithExtras) {
+    chromiumWithExtras = (0, import_playwright_extra.addExtra)(import_playwright.chromium);
+    const StealthPluginModule = await import("puppeteer-extra-plugin-stealth");
+    const stealthPluginFactory = typeof StealthPluginModule.default === "function" ? StealthPluginModule.default : StealthPluginModule;
+    if (typeof stealthPluginFactory !== "function") {
+      throw new Error("puppeteer-extra-plugin-stealth export is not a function or module structure is unexpected.");
+    }
+    StealthPluginInstance = stealthPluginFactory();
+    chromiumWithExtras.use(StealthPluginInstance);
+  }
+}
+var PlaywrightBrowserPool = class _PlaywrightBrowserPool {
+  pool = /* @__PURE__ */ new Set();
+  maxBrowsers;
+  maxPagesPerContext;
+  maxBrowserAge;
+  healthCheckInterval;
+  healthCheckTimer = null;
+  maxIdleTime;
+  isCleaningUp = false;
+  useHeadedMode;
+  blockedDomains;
+  blockedResourceTypes;
+  proxyConfig;
+  static DEFAULT_BLOCKED_DOMAINS = [
+    "doubleclick.net",
+    "google-analytics.com",
+    "googletagmanager.com",
+    "googlesyndication.com",
+    "googleadservices.com",
+    "adservice.google.com",
+    "facebook.net",
+    "fbcdn.net",
+    "connect.facebook.net",
+    "ads-twitter.com",
+    "platform.twitter.com",
+    "analytics.tiktok.com",
+    "ads.tiktok.com",
+    "amazon-adsystem.com",
+    "adnxs.com",
+    "criteo.com",
+    "scorecardresearch.com",
+    "quantserve.com",
+    "rubiconproject.com",
+    "pubmatic.com",
+    "taboola.com",
+    "outbrain.com"
+  ];
+  static DEFAULT_BLOCKED_RESOURCE_TYPES = ["image", "font", "media", "websocket"];
+  acquireQueue = new import_p_queue.default({ concurrency: 1 });
+  constructor(config = {}) {
+    this.maxBrowsers = config.maxBrowsers ?? 2;
+    this.maxPagesPerContext = config.maxPagesPerContext ?? 6;
+    this.maxBrowserAge = config.maxBrowserAge ?? 20 * 60 * 1e3;
+    this.healthCheckInterval = config.healthCheckInterval ?? 60 * 1e3;
+    this.useHeadedMode = config.useHeadedMode ?? false;
+    this.maxIdleTime = config.maxIdleTime ?? 5 * 60 * 1e3;
+    this.blockedDomains = config.blockedDomains && config.blockedDomains.length > 0 ? config.blockedDomains : _PlaywrightBrowserPool.DEFAULT_BLOCKED_DOMAINS;
+    this.blockedResourceTypes = config.blockedResourceTypes && config.blockedResourceTypes.length > 0 ? config.blockedResourceTypes : _PlaywrightBrowserPool.DEFAULT_BLOCKED_RESOURCE_TYPES;
+    this.proxyConfig = config.proxy;
+  }
+  async initialize() {
+    await loadDependencies();
+    if (this.isCleaningUp) return;
+    await this.ensureMinimumInstances();
+    this.scheduleHealthCheck();
+  }
+  scheduleHealthCheck() {
+    if (this.isCleaningUp) return;
+    if (this.healthCheckTimer) {
+      clearTimeout(this.healthCheckTimer);
+    }
+    if (this.healthCheckInterval > 0) {
+      this.healthCheckTimer = setTimeout(() => {
+        this.healthCheck().catch((_err) => {
+        });
+      }, this.healthCheckInterval);
+    }
+  }
+  async ensureMinimumInstances() {
+    if (this.isCleaningUp) return;
+    while (this.pool.size < this.maxBrowsers) {
+      try {
+        await this.createBrowserInstance();
+      } catch (error) {
+        break;
+      }
+    }
+  }
+  async createBrowserInstance() {
+    await loadDependencies();
+    const id = (0, import_uuid.v4)();
+    const launchOptions = {
+      headless: !this.useHeadedMode,
+      args: [
+        "--no-sandbox",
+        "--disable-setuid-sandbox",
+        "--disable-dev-shm-usage",
+        "--disable-accelerated-2d-canvas",
+        "--no-first-run",
+        "--no-zygote",
+        "--disable-gpu",
+        "--mute-audio",
+        "--disable-background-networking"
+      ],
+      proxy: this.proxyConfig
+    };
+    const browser = await chromiumWithExtras.launch(launchOptions);
+    const context = await browser.newContext({
+      userAgent: new import_user_agents.default().toString(),
+      viewport: {
+        width: 1280 + Math.floor(Math.random() * 120),
+        height: 720 + Math.floor(Math.random() * 80)
+      },
+      javaScriptEnabled: true,
+      ignoreHTTPSErrors: true
+    });
+    await context.route("**/*", async (route) => {
+      const request = route.request();
+      const url = request.url();
+      const resourceType = request.resourceType();
+      try {
+        const hostname = new URL(url).hostname.toLowerCase();
+        if (this.blockedDomains.some((domain) => hostname.includes(domain)) || this.blockedResourceTypes.includes(resourceType)) {
+          await route.abort("aborted");
+        } else {
+          await route.continue();
+        }
+      } catch (_e) {
+        await route.continue();
+      }
+    });
+    const now = /* @__PURE__ */ new Date();
+    const metrics = {
+      id,
+      pagesCreated: 0,
+      activePages: 0,
+      lastUsed: now,
+      errors: 0,
+      createdAt: now,
+      isHealthy: true
+    };
+    const instance = {
+      id,
+      browser,
+      context,
+      pages: /* @__PURE__ */ new Set(),
+      metrics,
+      isHealthy: true,
+      disconnectedHandler: () => {
+      }
+    };
+    instance.disconnectedHandler = () => {
+      if (instance.isHealthy) {
+        instance.isHealthy = false;
+        instance.metrics.isHealthy = false;
+        this.healthCheck().catch((_err) => {
+        });
+      }
+    };
+    browser.on("disconnected", instance.disconnectedHandler);
+    this.pool.add(instance);
+    return instance;
+  }
+  acquirePage() {
+    return this.acquireQueue.add(async () => {
+      if (this.isCleaningUp) {
+        throw new Error("Pool is shutting down.");
+      }
+      let bestInstance = null;
+      for (const instance of this.pool) {
+        if (instance.isHealthy && instance.pages.size < this.maxPagesPerContext) {
+          if (!bestInstance || instance.pages.size < bestInstance.pages.size) {
+            bestInstance = instance;
+          }
+        }
+      }
+      if (!bestInstance && this.pool.size < this.maxBrowsers) {
+        try {
+          bestInstance = await this.createBrowserInstance();
+        } catch (error) {
+          throw new Error(`Failed to create new browser instance for acquisition: ${error.message}`);
+        }
+      }
+      if (!bestInstance) {
+        await this.ensureMinimumInstances();
+        for (const instance of this.pool) {
+          if (instance.isHealthy && instance.pages.size < this.maxPagesPerContext) {
+            if (!bestInstance || instance.pages.size < bestInstance.pages.size) {
+              bestInstance = instance;
+            }
+          }
+        }
+        if (!bestInstance) {
+          throw new Error("Failed to acquire Playwright page: No available or creatable browser instance.");
+        }
+      }
+      try {
+        const page = await bestInstance.context.newPage();
+        bestInstance.pages.add(page);
+        bestInstance.metrics.pagesCreated++;
+        bestInstance.metrics.activePages = bestInstance.pages.size;
+        bestInstance.metrics.lastUsed = /* @__PURE__ */ new Date();
+        page.on("close", () => {
+          bestInstance.pages.delete(page);
+          bestInstance.metrics.activePages = bestInstance.pages.size;
+          bestInstance.metrics.lastUsed = /* @__PURE__ */ new Date();
+        });
+        page.on("crash", () => {
+          bestInstance.metrics.errors++;
+          bestInstance.pages.delete(page);
+          bestInstance.isHealthy = false;
+          bestInstance.metrics.isHealthy = false;
+          this.healthCheck().catch((_err) => {
+          });
+        });
+        return page;
+      } catch (error) {
+        bestInstance.metrics.errors++;
+        bestInstance.isHealthy = false;
+        bestInstance.metrics.isHealthy = false;
+        this.healthCheck().catch((_err) => {
+        });
+        throw new Error(`Failed to create new page: ${error.message}`);
+      }
+    });
+  }
+  async healthCheck() {
+    if (this.isCleaningUp) return;
+    const now = /* @__PURE__ */ new Date();
+    const checks = [];
+    for (const instance of this.pool) {
+      checks.push(
+        (async () => {
+          if (!instance.isHealthy) {
+            return;
+          }
+          let shouldRemove = false;
+          let reason = "unknown";
+          if (!instance.browser.isConnected()) {
+            shouldRemove = true;
+            reason = "browser disconnected";
+          }
+          if (!shouldRemove && this.maxBrowserAge > 0 && now.getTime() - instance.metrics.createdAt.getTime() > this.maxBrowserAge) {
+            shouldRemove = true;
+            reason = "max age reached";
+          }
+          if (!shouldRemove && this.pool.size > 1 && // Only remove idle if pool has more than 1
+          instance.pages.size === 0 && this.maxIdleTime > 0 && now.getTime() - instance.metrics.lastUsed.getTime() > this.maxIdleTime) {
+            shouldRemove = true;
+            reason = "idle timeout";
+          }
+          if (shouldRemove) {
+            instance.isHealthy = false;
+            instance.metrics.isHealthy = false;
+            await this.closeAndRemoveInstance(instance, reason);
+          } else {
+            instance.isHealthy = true;
+            instance.metrics.isHealthy = true;
+          }
+        })().catch((_err) => {
+        })
+      );
+    }
+    try {
+      await Promise.allSettled(checks);
+    } finally {
+      await this.ensureMinimumInstances();
+      this.scheduleHealthCheck();
+    }
+  }
+  async closeAndRemoveInstance(instance, _reason) {
+    const removed = this.pool.delete(instance);
+    if (!removed) return;
+    instance.browser.off("disconnected", instance.disconnectedHandler);
+    try {
+      await instance.context.close();
+    } catch (_error) {
+    }
+    try {
+      await instance.browser.close();
+    } catch (_error) {
+    }
+  }
+  async releasePage(page) {
+    if (!page || page.isClosed()) return;
+    let ownerInstance;
+    for (const instance of this.pool) {
+      if (instance.pages.has(page)) {
+        ownerInstance = instance;
+        break;
+      }
+    }
+    try {
+      await page.close();
+      if (ownerInstance) {
+        ownerInstance.pages.delete(page);
+        ownerInstance.metrics.activePages = ownerInstance.pages.size;
+        ownerInstance.metrics.lastUsed = /* @__PURE__ */ new Date();
+      }
+    } catch (error) {
+      if (ownerInstance) {
+        ownerInstance.isHealthy = false;
+        ownerInstance.metrics.isHealthy = false;
+        ownerInstance.metrics.errors++;
+        ownerInstance.pages.delete(page);
+        ownerInstance.metrics.activePages = ownerInstance.pages.size;
+      }
+    }
+  }
+  async cleanup() {
+    if (this.isCleaningUp) return;
+    this.isCleaningUp = true;
+    if (this.healthCheckTimer) {
+      clearTimeout(this.healthCheckTimer);
+      this.healthCheckTimer = null;
+    }
+    this.acquireQueue.clear();
+    await this.acquireQueue.onIdle();
+    const closePromises = [...this.pool].map((instance) => this.closeAndRemoveInstance(instance, "cleanup"));
+    this.pool.clear();
+    await Promise.allSettled(closePromises);
+    this.isCleaningUp = false;
+  }
+  getMetrics() {
+    return [...this.pool].map((instance) => ({
+      ...instance.metrics,
+      activePages: instance.pages.size,
+      isHealthy: instance.isHealthy
+    }));
+  }
+};
+
+// src/PlaywrightEngine.ts
+var import_p_queue2 = __toESM(require("p-queue"), 1);
+var import_axios = __toESM(require("axios"), 1);
+function delay(time) {
+  return new Promise((resolve) => setTimeout(resolve, time));
+}
+var PlaywrightEngine = class _PlaywrightEngine {
+  browserPool = null;
+  queue;
+  cache = /* @__PURE__ */ new Map();
+  config;
+  // Browser pooling safety flags
+  initializingBrowserPool = false;
+  isUsingHeadedMode = false;
+  // Tracks current pool mode
+  headedFallbackSites = /* @__PURE__ */ new Set();
+  // Stores domains marked for headed mode
+  // Default configuration - Ensure all required fields are present
+  static DEFAULT_CONFIG = {
+    concurrentPages: 3,
+    maxRetries: 3,
+    retryDelay: 5e3,
+    cacheTTL: 15 * 60 * 1e3,
+    useHttpFallback: true,
+    useHeadedModeFallback: false,
+    defaultFastMode: true,
+    simulateHumanBehavior: true,
+    maxBrowsers: 2,
+    maxPagesPerContext: 6,
+    maxBrowserAge: 20 * 60 * 1e3,
+    healthCheckInterval: 60 * 1e3,
+    poolBlockedDomains: [],
+    poolBlockedResourceTypes: [],
+    proxy: void 0,
+    useHeadedMode: false,
+    // ADDED default
+    markdown: true
+  };
+  /**
+   * Creates an instance of PlaywrightEngine.
+   *
+   * @param config Configuration options for the engine and its browser pool.
+   *               See `PlaywrightEngineConfig` for details.
+   */
+  constructor(config = {}) {
+    this.config = { ..._PlaywrightEngine.DEFAULT_CONFIG, ...config };
+    this.queue = new import_p_queue2.default({ concurrency: this.config.concurrentPages });
+  }
+  /**
+   * Initialize the browser pool with improved error handling and mode switching.
+   */
+  async initializeBrowserPool(useHeadedMode = false) {
+    if (this.browserPool && this.isUsingHeadedMode === useHeadedMode) {
+      return;
+    }
+    if (this.initializingBrowserPool) {
+      while (this.initializingBrowserPool) {
+        await delay(100);
+      }
+      if (this.browserPool && this.isUsingHeadedMode === useHeadedMode) {
+        return;
+      }
+    }
+    this.initializingBrowserPool = true;
+    try {
+      if (this.browserPool && this.isUsingHeadedMode !== useHeadedMode) {
+        await this.browserPool.cleanup();
+        this.browserPool = null;
+      }
+      this.isUsingHeadedMode = useHeadedMode;
+      this.browserPool = new PlaywrightBrowserPool({
+        maxBrowsers: this.config.maxBrowsers,
+        maxPagesPerContext: this.config.maxPagesPerContext,
+        maxBrowserAge: this.config.maxBrowserAge,
+        healthCheckInterval: this.config.healthCheckInterval,
+        useHeadedMode,
+        blockedDomains: this.config.poolBlockedDomains,
+        blockedResourceTypes: this.config.poolBlockedResourceTypes,
+        proxy: this.config.proxy
+      });
+      await this.browserPool.initialize();
+    } catch (error) {
+      this.browserPool = null;
+      this.isUsingHeadedMode = false;
+      throw error;
+    } finally {
+      this.initializingBrowserPool = false;
+    }
+  }
+  /**
+   * Fallback method using simple HTTP requests via Axios.
+   * Ensures return type matches HTMLFetchResult.
+   */
+  async fetchHTMLWithHttpFallback(url) {
+    try {
+      const response = await import_axios.default.get(url, {
+        headers: {
+          // Use more standard browser-like headers
+          "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+          Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+          "Accept-Language": "en-US,en;q=0.9",
+          "Accept-Encoding": "gzip, deflate, br",
+          // Allow compression
+          Referer: "https://www.google.com/",
+          // Common referer
+          "Upgrade-Insecure-Requests": "1",
+          "Sec-Ch-Ua": '"Not A(Brand";v="99", "Google Chrome";v="121", "Chromium";v="121"',
+          "Sec-Ch-Ua-Mobile": "?0",
+          "Sec-Ch-Ua-Platform": '"Windows"',
+          "Sec-Fetch-Dest": "document",
+          "Sec-Fetch-Mode": "navigate",
+          "Sec-Fetch-Site": "cross-site",
+          "Sec-Fetch-User": "?1",
+          Connection: "keep-alive"
+          // Keep connection open
+          // Avoid Cache-Control/Pragma unless specifically needed
+        },
+        maxRedirects: 5,
+        timeout: 3e4,
+        responseType: "text",
+        // Decompress response automatically
+        decompress: true
+      });
+      const titleMatch = response.data.match(/<title[^>]*>([^<]+)<\/title>/i);
+      let title = titleMatch ? titleMatch[1].trim() : "";
+      if (!title && /<html>([^<]+)<\/html>/.test(response.data)) {
+        title = response.data.replace(/<\/?html>/g, "").trim();
+      }
+      const lowerHtml = response.data.toLowerCase();
+      const isChallengeOrBot = /cloudflare|checking your browser|please wait|verification|captcha|attention required/i.test(lowerHtml);
+      if (isChallengeOrBot) {
+        throw new FetchError("Received challenge page via HTTP fallback", "ERR_CHALLENGE_PAGE");
+      }
+      const originalHtml = response.data;
+      let finalContent = originalHtml;
+      let finalContentType = "html";
+      if (this.config.markdown) {
+        try {
+          const converter = new MarkdownConverter();
+          finalContent = converter.convert(originalHtml);
+          finalContentType = "markdown";
+        } catch (conversionError) {
+          console.error(`Markdown conversion failed for ${url} (HTTP fallback):`, conversionError);
+        }
+      }
+      return {
+        content: finalContent,
+        contentType: finalContentType,
+        title,
+        // title is extracted from original HTML
+        url: response.request?.res?.responseUrl || response.config.url || url,
+        isFromCache: false,
+        statusCode: response.status,
+        error: void 0
+      };
+    } catch (error) {
+      if (!(error instanceof FetchError)) {
+        throw new FetchError(`HTTP fallback failed: ${error.message}`, "ERR_HTTP_FALLBACK_FAILED", error);
+      }
+      throw error;
+    }
+  }
+  checkCache(url) {
+    const cached = this.cache.get(url);
+    if (cached && Date.now() - cached.timestamp < this.config.cacheTTL) {
+      return cached.result;
+    }
+    if (cached) {
+      this.cache.delete(url);
+    }
+    return null;
+  }
+  /**
+   * Safely check if a page is still usable and connected.
+   */
+  async isPageValid(page) {
+    if (!page || page.isClosed()) return false;
+    try {
+      if (!page.context().browser()?.isConnected()) return false;
+      await page.evaluate("1 + 1", { timeout: 1e3 });
+      return true;
+    } catch (error) {
+      return false;
+    }
+  }
+  /**
+   * Simulate human-like interactions on the page.
+   */
+  async simulateHumanBehavior(page) {
+    if (!await this.isPageValid(page)) return;
+    try {
+      const viewport = page.viewportSize();
+      if (!viewport) return;
+      await page.mouse.move(Math.random() * viewport.width, Math.random() * viewport.height / 3, { steps: 5 });
+      await delay(150 + Math.random() * 200);
+      await page.mouse.move(
+        Math.random() * viewport.width,
+        viewport.height / 2 + Math.random() * viewport.height / 2,
+        { steps: 10 }
+      );
+      await delay(200 + Math.random() * 300);
+      await page.evaluate(() => {
+        window.scrollBy({
+          top: window.innerHeight * (0.3 + Math.random() * 0.4),
+          behavior: "smooth"
+        });
+      });
+      await delay(400 + Math.random() * 600);
+      await page.evaluate(() => {
+        window.scrollBy({
+          top: window.innerHeight * (0.2 + Math.random() * 0.3),
+          behavior: "smooth"
+        });
+      });
+      await delay(300 + Math.random() * 400);
+    } catch (_error) {
+    }
+  }
+  /**
+   * Adds a result to the in-memory cache.
+   */
+  addToCache(url, result) {
+    if (this.config.cacheTTL <= 0) return;
+    const entry = {
+      result: { ...result, isFromCache: true },
+      // Mark as cached
+      timestamp: Date.now()
+    };
+    this.cache.set(url, entry);
+  }
+  /**
+   * Public method to fetch HTML. Delegates to the internal recursive fetch method.
+   *
+   * @param url The URL to fetch.
+   * @param options Optional settings for this specific fetch operation.
+   * @param options.fastMode Overrides the engine's `defaultFastMode` configuration for this request.
+   * @returns A Promise resolving to an HTMLFetchResult object.
+   * @throws {FetchError} If the fetch fails after all retries or encounters critical errors.
+   */
+  async fetchHTML(url, options = {}) {
+    const fetchConfig = {
+      ...this.config,
+      markdown: options.markdown === void 0 ? this.config.markdown : options.markdown,
+      fastMode: options.fastMode === void 0 ? this.config.defaultFastMode : options.fastMode
+    };
+    return this._fetchRecursive(url, fetchConfig, 0, 0);
+  }
+  /**
+   * Internal recursive method to handle fetching with retries.
+   *
+   * @param url URL to fetch
+   * @param currentConfig The merged configuration including markdown option
+   * @param retryAttempt Current retry attempt number (starts at 0)
+   * @param parentRetryCount Tracks retries related to pool initialization errors (starts at 0)
+   * @returns Promise resolving to HTMLFetchResult
+   */
+  async _fetchRecursive(url, currentConfig, retryAttempt, parentRetryCount) {
+    const useFastMode = currentConfig.fastMode;
+    if (retryAttempt === 0 && parentRetryCount === 0) {
+      const cachedResult = this.checkCache(url);
+      if (cachedResult) {
+        if (currentConfig.markdown && !cachedResult.content.startsWith("#") && !cachedResult.content.includes("\n\n---\n\n")) {
+          try {
+            const converter = new MarkdownConverter();
+            cachedResult.content = converter.convert(cachedResult.content);
+          } catch (e) {
+            console.error("Failed to convert cached result to markdown", e);
+          }
+        } else if (!currentConfig.markdown && (cachedResult.content.startsWith("#") || cachedResult.content.includes("\n\n---\n\n"))) {
+          console.warn("Cached result is Markdown, but HTML was requested. Re-fetching.");
+          this.cache.delete(url);
+          return this._fetchRecursive(url, currentConfig, 0, 0);
+        }
+        return cachedResult;
+      }
+    }
+    try {
+      if (currentConfig.useHttpFallback && retryAttempt === 0 && parentRetryCount === 0) {
+        try {
+          const httpResult = await this.fetchHTMLWithHttpFallback(url);
+          if (this.config.cacheTTL > 0) {
+            this.addToCache(url, httpResult);
+          }
+          return httpResult;
+        } catch (httpError) {
+          if (httpError instanceof FetchError && httpError.code === "ERR_CHALLENGE_PAGE") {
+          } else {
+          }
+        }
+      }
+      const useHeadedMode = currentConfig.useHeadedModeFallback && (retryAttempt >= 2 || this.shouldUseHeadedMode(url)) || currentConfig.useHeadedMode;
+      try {
+        if (!this.browserPool || this.isUsingHeadedMode !== useHeadedMode) {
+          await this.initializeBrowserPool(useHeadedMode);
+        }
+      } catch (initError) {
+        if (parentRetryCount < 1) {
+          await delay(currentConfig.retryDelay);
+          return this._fetchRecursive(url, currentConfig, retryAttempt, parentRetryCount + 1);
+        }
+        throw new FetchError(
+          `Pool init failed: ${initError.message}`,
+          "ERR_POOL_INIT_FAILED",
+          initError
+        );
+      }
+      if (!this.browserPool) {
+        throw new FetchError("Browser pool unavailable.", "ERR_POOL_UNAVAILABLE");
+      }
+      const result = await this.queue.add(
+        () => this.fetchWithPlaywright(url, this.browserPool, useFastMode, currentConfig.markdown)
+      );
+      if (result && this.config.cacheTTL > 0) {
+        this.addToCache(url, result);
+      }
+      if (!result) {
+        throw new FetchError("Playwright fetch queued but no result.", "ERR_QUEUE_NO_RESULT");
+      }
+      return result;
+    } catch (error) {
+      if (useFastMode && retryAttempt === 0 && parentRetryCount === 0) {
+        return this._fetchRecursive(url, { ...currentConfig, fastMode: false }, 0, parentRetryCount);
+      }
+      if (retryAttempt < currentConfig.maxRetries) {
+        await delay(currentConfig.retryDelay);
+        return this._fetchRecursive(url, currentConfig, retryAttempt + 1, parentRetryCount);
+      }
+      const finalError = error instanceof FetchError ? error : new FetchError(`Fetch failed: ${error.message}`, "ERR_FETCH_FAILED", error);
+      throw new FetchError(
+        `Fetch failed after ${currentConfig.maxRetries} retries: ${finalError.message}`,
+        finalError.code,
+        finalError.originalError || error
+      );
+    }
+  }
+  /**
+   * Performs the actual page fetch using a Playwright page from the pool.
+   * Ensures return type matches HTMLFetchResult.
+   */
+  async fetchWithPlaywright(url, pool, fastMode, convertToMarkdown) {
+    let page = null;
+    try {
+      page = await pool.acquirePage();
+      await this.applyBlockingRules(page, fastMode);
+      let response = null;
+      try {
+        response = await page.goto(url, {
+          waitUntil: "domcontentloaded",
+          timeout: 6e4
+        });
+      } catch (navigationError) {
+        throw new FetchError(
+          `Playwright navigation failed: ${navigationError.message}`,
+          "ERR_NAVIGATION",
+          navigationError
+        );
+      }
+      if (!response) {
+        throw new FetchError("Playwright navigation did not return a response.", "ERR_NO_RESPONSE");
+      }
+      if (!response.ok()) {
+        throw new FetchError(
+          `HTTP error status received: ${response.status()}`,
+          "ERR_HTTP_ERROR",
+          void 0,
+          response.status()
+        );
+      }
+      const contentType = response.headers()["content-type"] || "";
+      if (!contentType.includes("html")) {
+        throw new FetchError(`Invalid content type received: ${contentType}`, "ERR_NON_HTML_CONTENT");
+      }
+      if (!fastMode && this.config.simulateHumanBehavior) {
+        await this.simulateHumanBehavior(page);
+      }
+      const html = await page.content();
+      const title = await page.title();
+      const finalUrl = page.url();
+      const status = response?.status();
+      let finalContent = html;
+      let finalContentType = "html";
+      if (convertToMarkdown) {
+        try {
+          const converter = new MarkdownConverter();
+          finalContent = converter.convert(html);
+          finalContentType = "markdown";
+        } catch (conversionError) {
+          console.error(`Markdown conversion failed for ${url} (Playwright):`, conversionError);
+        }
+      }
+      return {
+        content: finalContent,
+        contentType: finalContentType,
+        title: title || null,
+        url: finalUrl,
+        isFromCache: false,
+        statusCode: status,
+        error: void 0
+      };
+    } finally {
+      if (page) {
+        await pool.releasePage(page);
+      }
+    }
+  }
+  async applyBlockingRules(page, fastMode) {
+    const blockedResources = fastMode ? this.config.poolBlockedResourceTypes.concat(["image", "font", "stylesheet", "media"]) : this.config.poolBlockedResourceTypes;
+    const blockedDomains = this.config.poolBlockedDomains;
+    if (blockedResources.length > 0 || blockedDomains.length > 0) {
+      try {
+        await page.route("**/*", (route) => {
+          const resourceType = route.request().resourceType();
+          const requestUrl = route.request().url();
+          if (blockedResources.includes(resourceType)) {
+            return route.abort();
+          }
+          if (blockedDomains.some(
+            (pattern) => new RegExp(pattern.replace(/\./g, "\\.").replace(/\*/g, ".*")).test(requestUrl)
+          )) {
+            return route.abort();
+          }
+          return route.continue();
+        });
+      } catch (_error) {
+      }
+    }
+  }
+  /**
+   * Cleans up resources used by the engine, primarily closing browser instances in the pool.
+   *
+   * It is crucial to call this method when finished with the engine instance to release resources.
+   * @returns A Promise that resolves when cleanup is complete.
+   */
+  async cleanup() {
+    try {
+      await this.queue.onIdle();
+      this.queue.clear();
+      if (this.browserPool) {
+        await this.browserPool.cleanup();
+        this.browserPool = null;
+      }
+      this.isUsingHeadedMode = false;
+    } catch (_error) {
+    }
+  }
+  /**
+   * Retrieves metrics from the underlying browser pool.
+   * @returns An array of BrowserMetrics objects, one for each active browser instance, or an empty array if the pool is not initialized.
+   */
+  getMetrics() {
+    if (this.browserPool) {
+      return this.browserPool.getMetrics();
+    }
+    return [];
+  }
+  // Helper to check if a specific domain is marked for headed mode
+  shouldUseHeadedMode(url) {
+    if (!this.config.useHeadedModeFallback) return false;
+    try {
+      const domain = new URL(url).hostname;
+      return this.headedFallbackSites.has(domain);
+    } catch {
+      return false;
+    }
+  }
+};
+
+// src/HybridEngine.ts
+var HybridEngine = class {
+  fetchEngine;
+  playwrightEngine;
+  config;
+  // Store config for potential per-request PW overrides
+  constructor(config = {}) {
+    this.fetchEngine = new FetchEngine({ markdown: config.markdown });
+    this.playwrightEngine = new PlaywrightEngine(config);
+    this.config = config;
+  }
+  async fetchHTML(url, options = {}) {
+    try {
+      const fetchResult = await this.fetchEngine.fetchHTML(url);
+      return fetchResult;
+    } catch (fetchError) {
+      console.warn(`FetchEngine failed for ${url}: ${fetchError.message}. Falling back to PlaywrightEngine.`);
+      const playwrightOptions = {
+        ...this.config,
+        // Start with base config given to HybridEngine
+        ...options
+        // Override with per-request options
+      };
+      try {
+        const playwrightResult = await this.playwrightEngine.fetchHTML(url, playwrightOptions);
+        return playwrightResult;
+      } catch (playwrightError) {
+        console.error(`PlaywrightEngine fallback failed for ${url}: ${playwrightError.message}`);
+        throw playwrightError;
+      }
+    }
+  }
+  /**
+   * Delegates getMetrics to the PlaywrightEngine.
+   */
+  getMetrics() {
+    return this.playwrightEngine.getMetrics();
+  }
+  /**
+   * Calls cleanup on both underlying engines.
+   */
+  async cleanup() {
+    await Promise.allSettled([
+      this.fetchEngine.cleanup(),
+      // Although a no-op, call for consistency
+      this.playwrightEngine.cleanup()
+    ]);
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  FetchEngine,
+  HybridEngine,
+  PlaywrightEngine
+});
+//# sourceMappingURL=index.cjs.map