@vakra-dev/reader 0.0.2 → 0.1.0

package/dist/cli/index.js

@@ -17,132 +17,6 @@ import { ConnectionToHeroCore } from "@ulixee/hero";
 // src/scraper.ts
 import pLimit from "p-limit";
 
-// src/cloudflare/detector.ts
-var CHALLENGE_DOM_SELECTORS = [
-  "#challenge-running",
-  "#challenge-stage",
-  "#challenge-form",
-  ".cf-browser-verification"
-];
-var CHALLENGE_TEXT_PATTERNS = [
-  "verifying you are human",
-  "checking if the site connection is secure",
-  "this process is automatic. your browser will redirect"
-];
-var BLOCKED_SIGNALS = [
-  "you have been blocked",
-  "access to this page has been denied",
-  "sorry, you have been blocked",
-  "access denied",
-  "403 forbidden"
-];
-async function detectChallenge(hero) {
-  const signals = [];
-  let type = "none";
-  try {
-    if (!hero.document) {
-      return {
-        isChallenge: false,
-        type: "none",
-        confidence: 0,
-        signals: ["No document available"]
-      };
-    }
-    const html = await hero.document.documentElement.outerHTML;
-    const htmlLower = html.toLowerCase();
-    for (const selector of CHALLENGE_DOM_SELECTORS) {
-      if (htmlLower.includes(selector.toLowerCase())) {
-        signals.push(`Challenge element: ${selector}`);
-        type = "js_challenge";
-      }
-    }
-    for (const pattern of CHALLENGE_TEXT_PATTERNS) {
-      if (htmlLower.includes(pattern)) {
-        signals.push(`Challenge text: "${pattern}"`);
-        type = type === "none" ? "js_challenge" : type;
-      }
-    }
-    if (htmlLower.includes("waiting for") && htmlLower.includes("to respond")) {
-      signals.push('Challenge text: "waiting for...to respond"');
-      type = type === "none" ? "js_challenge" : type;
-    }
-    for (const pattern of BLOCKED_SIGNALS) {
-      if (htmlLower.includes(pattern)) {
-        signals.push(`Blocked: "${pattern}"`);
-        type = "blocked";
-        break;
-      }
-    }
-    const isChallenge = signals.length > 0;
-    const confidence = isChallenge ? 100 : 0;
-    return {
-      isChallenge,
-      type: isChallenge ? type : "none",
-      confidence,
-      signals
-    };
-  } catch (error) {
-    return {
-      isChallenge: false,
-      type: "none",
-      confidence: 0,
-      signals: [`Error during detection: ${error.message}`]
-    };
-  }
-}
-
-// src/cloudflare/handler.ts
-async function waitForChallengeResolution(hero, options) {
-  const { maxWaitMs = 45e3, pollIntervalMs = 500, verbose = false, initialUrl } = options;
-  const startTime = Date.now();
-  const log = (msg) => verbose && console.log(`   ${msg}`);
-  while (Date.now() - startTime < maxWaitMs) {
-    const elapsed = Date.now() - startTime;
-    try {
-      const currentUrl = await hero.url;
-      if (currentUrl !== initialUrl) {
-        log(`\u2713 URL changed: ${initialUrl} \u2192 ${currentUrl}`);
-        log(`  Waiting for new page to load...`);
-        try {
-          await hero.waitForLoad("DomContentLoaded", { timeoutMs: 3e4 });
-          log(`  DOMContentLoaded`);
-        } catch {
-          log(`  DOMContentLoaded timeout, continuing...`);
-        }
-        await hero.waitForPaintingStable().catch(() => {
-        });
-        log(`  Page stabilized`);
-        return { resolved: true, method: "url_redirect", waitedMs: elapsed };
-      }
-    } catch {
-    }
-    const detection = await detectChallenge(hero);
-    if (!detection.isChallenge) {
-      log(`\u2713 Challenge signals cleared (confidence dropped to ${detection.confidence})`);
-      log(`  Waiting for page to load...`);
-      try {
-        await hero.waitForLoad("DomContentLoaded", { timeoutMs: 3e4 });
-        log(`  DOMContentLoaded`);
-      } catch {
-        log(`  DOMContentLoaded timeout, continuing...`);
-      }
-      await hero.waitForPaintingStable().catch(() => {
-      });
-      log(`  Page stabilized`);
-      return { resolved: true, method: "signals_cleared", waitedMs: elapsed };
-    }
-    log(
-      `\u23F3 ${(elapsed / 1e3).toFixed(1)}s - Still challenge (confidence: ${detection.confidence})`
-    );
-    await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
-  }
-  return {
-    resolved: false,
-    method: "timeout",
-    waitedMs: Date.now() - startTime
-  };
-}
-
 // src/formatters/markdown.ts
 import TurndownService from "turndown";
 var turndownService = new TurndownService({
@@ -156,84 +30,6 @@ var turndownService = new TurndownService({
   linkStyle: "inlined",
   linkReferenceStyle: "full"
 });
-function formatToMarkdown(pages, baseUrl, scrapedAt, duration, website, includeMetadata = true) {
-  const sections = [];
-  if (includeMetadata) {
-    sections.push(createMarkdownHeader(baseUrl, scrapedAt, duration, website, pages.length));
-  }
-  if (pages.length > 1) {
-    sections.push(createMarkdownTOC(pages));
-  }
-  sections.push(...pages.map((page, index) => createMarkdownPage(page, index + 1)));
-  return sections.join("\n\n");
-}
-function createMarkdownHeader(baseUrl, scrapedAt, duration, website, totalPages) {
-  const title = website.title || extractDomainFromUrl(baseUrl);
-  const description = website.description || "";
-  let header = `# Website Scrape: ${title}
-
-`;
-  header += `**Base URL:** ${baseUrl}
-`;
-  header += `**Scraped at:** ${new Date(scrapedAt).toLocaleString()}
-`;
-  header += `**Duration:** ${duration}ms
-`;
-  header += `**Total pages:** ${totalPages}
-`;
-  if (description) {
-    header += `**Description:** ${description}
-`;
-  }
-  if (website.author) {
-    header += `**Author:** ${website.author}
-`;
-  }
-  if (website.language) {
-    header += `**Language:** ${website.language}
-`;
-  }
-  return header;
-}
-function createMarkdownTOC(pages) {
-  let toc = "## Table of Contents\n\n";
-  pages.forEach((page, index) => {
-    const depth = "  ".repeat(page.depth);
-    const pageNumber = index + 1;
-    const title = page.title || `Page ${pageNumber}`;
-    const cleanTitle = title.replace(/[#[\]/\\:*?"<>|]/g, "").trim();
-    const anchor = cleanTitle.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
-    toc += `${depth}${pageNumber}. [${title}](#page-${pageNumber}-${anchor})
-`;
-  });
-  return toc;
-}
-function createMarkdownPage(page, pageNumber) {
-  const title = page.title || `Page ${pageNumber}`;
-  const cleanTitle = title.replace(/[#[\]/\\:*?"<>|]/g, "").trim();
-  const anchor = cleanTitle.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
-  let pageContent = `---
-
-`;
-  pageContent += `## Page ${pageNumber}: ${title} {#page-${pageNumber}-${anchor}}
-
-`;
-  pageContent += `**URL:** ${page.url}
-`;
-  pageContent += `**Title:** ${page.title}
-`;
-  pageContent += `**Depth:** ${page.depth}
-`;
-  pageContent += `**Fetched at:** ${new Date(page.fetchedAt).toLocaleString()}
-
-`;
-  pageContent += `---
-
-`;
-  const markdown = htmlToMarkdown(page.html);
-  pageContent += markdown;
-  return pageContent;
-}
 function htmlToMarkdown(html) {
   try {
     return turndownService.turndown(html);
@@ -242,574 +38,339 @@ function htmlToMarkdown(html) {
     return html.replace(/<[^>]*>/g, "").trim();
   }
 }
-function extractDomainFromUrl(url) {
-  try {
-    return new URL(url).hostname;
-  } catch {
-    return "Unknown";
-  }
-}
-
-// src/formatters/html.ts
-function formatToHTML(pages, baseUrl, scrapedAt, duration, website) {
-  const html = `<!DOCTYPE html>
-<html lang="${website.language || "en"}">
-<head>
-    <meta charset="${website.charset || "UTF-8"}">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Scrape: ${website.title || extractDomainFromUrl2(baseUrl)}</title>
-    ${generateMetaTags(website)}
-    <style>
-        ${generateCSS()}
-    </style>
-</head>
-<body>
-    <header class="header">
-        <h1>Website Scrape: ${escapeHtml(website.title || extractDomainFromUrl2(baseUrl))}</h1>
-        <div class="meta-info">
-            <p><strong>Base URL:</strong> <a href="${escapeHtml(
-    baseUrl
-  )}" target="_blank">${escapeHtml(baseUrl)}</a></p>
-            <p><strong>Scraped at:</strong> ${new Date(scrapedAt).toLocaleString()}</p>
-            <p><strong>Duration:</strong> ${duration}ms</p>
-            <p><strong>Total pages:</strong> ${pages.length}</p>
-            ${website.description ? `<p><strong>Description:</strong> ${escapeHtml(website.description)}</p>` : ""}
-            ${website.author ? `<p><strong>Author:</strong> ${escapeHtml(website.author)}</p>` : ""}
-            ${website.language ? `<p><strong>Language:</strong> ${escapeHtml(website.language)}</p>` : ""}
-        </div>
-    </header>
-
-    ${pages.length > 1 ? generateTOC(pages) : ""}
-
-    <main class="content">
-        ${pages.map((page, index) => generatePageHTML(page, index + 1)).join("\n")}
-    </main>
-
-    <footer class="footer">
-        <p>Generated by Reader JS/TS SDK</p>
-    </footer>
-
-    <script>
-        ${generateJavaScript()}
-    </script>
-</body>
-</html>`;
-  return html;
-}
-function generateMetaTags(website) {
-  const tags = [];
-  if (website.description) {
-    tags.push(`<meta name="description" content="${escapeHtml(website.description)}">`);
-  }
-  if (website.author) {
-    tags.push(`<meta name="author" content="${escapeHtml(website.author)}">`);
-  }
-  if (website.keywords) {
-    tags.push(`<meta name="keywords" content="${escapeHtml(website.keywords.join(", "))}">`);
-  }
-  if (website.robots) {
-    tags.push(`<meta name="robots" content="${escapeHtml(website.robots)}">`);
-  }
-  if (website.themeColor) {
-    tags.push(`<meta name="theme-color" content="${escapeHtml(website.themeColor)}">`);
-  }
-  if (website.favicon) {
-    tags.push(`<link rel="icon" href="${escapeHtml(website.favicon)}">`);
-  }
-  if (website.canonical) {
-    tags.push(`<link rel="canonical" href="${escapeHtml(website.canonical)}">`);
-  }
-  if (website.openGraph) {
-    const og = website.openGraph;
-    if (og.title) tags.push(`<meta property="og:title" content="${escapeHtml(og.title)}">`);
-    if (og.description)
-      tags.push(`<meta property="og:description" content="${escapeHtml(og.description)}">`);
-    if (og.type) tags.push(`<meta property="og:type" content="${escapeHtml(og.type)}">`);
-    if (og.url) tags.push(`<meta property="og:url" content="${escapeHtml(og.url)}">`);
-    if (og.image) tags.push(`<meta property="og:image" content="${escapeHtml(og.image)}">`);
-    if (og.siteName)
-      tags.push(`<meta property="og:site_name" content="${escapeHtml(og.siteName)}">`);
-    if (og.locale) tags.push(`<meta property="og:locale" content="${escapeHtml(og.locale)}">`);
-  }
-  if (website.twitter) {
-    const twitter = website.twitter;
-    if (twitter.card) tags.push(`<meta name="twitter:card" content="${escapeHtml(twitter.card)}">`);
-    if (twitter.site) tags.push(`<meta name="twitter:site" content="${escapeHtml(twitter.site)}">`);
-    if (twitter.creator)
-      tags.push(`<meta name="twitter:creator" content="${escapeHtml(twitter.creator)}">`);
-    if (twitter.title)
-      tags.push(`<meta name="twitter:title" content="${escapeHtml(twitter.title)}">`);
-    if (twitter.description)
-      tags.push(`<meta name="twitter:description" content="${escapeHtml(twitter.description)}">`);
-    if (twitter.image)
-      tags.push(`<meta name="twitter:image" content="${escapeHtml(twitter.image)}">`);
-  }
-  return tags.join("\n    ");
-}
-function generateCSS() {
-  return `
-        * {
-            margin: 0;
-            padding: 0;
-            box-sizing: border-box;
-        }
-
-        body {
-            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
-            line-height: 1.6;
-            color: #333;
-            background-color: #f8f9fa;
-        }
-
-        .header {
-            background: white;
-            padding: 2rem;
-            border-bottom: 1px solid #e9ecef;
-            margin-bottom: 2rem;
-        }
-
-        .header h1 {
-            color: #2c3e50;
-            margin-bottom: 1rem;
-            font-size: 2rem;
-        }
-
-        .meta-info {
-            display: grid;
-            grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
-            gap: 0.5rem;
-        }
-
-        .meta-info p {
-            margin: 0.25rem 0;
-            font-size: 0.9rem;
-            color: #6c757d;
-        }
-
-        .toc {
-            background: white;
-            padding: 1.5rem;
-            margin: 2rem 0;
-            border-radius: 8px;
-            border: 1px solid #e9ecef;
-        }
-
-        .toc h2 {
-            color: #2c3e50;
-            margin-bottom: 1rem;
-            font-size: 1.25rem;
-        }
-
-        .toc ul {
-            list-style: none;
-        }
-
-        .toc li {
-            margin: 0.5rem 0;
-        }
-
-        .toc a {
-            color: #007bff;
-            text-decoration: none;
-            transition: color 0.2s;
-        }
-
-        .toc a:hover {
-            color: #0056b3;
-            text-decoration: underline;
-        }
-
-        .content {
-            max-width: 800px;
-            margin: 0 auto;
-            padding: 0 1rem;
-        }
-
-        .page {
-            background: white;
-            margin: 2rem 0;
-            padding: 2rem;
-            border-radius: 8px;
-            border: 1px solid #e9ecef;
-            box-shadow: 0 2px 4px rgba(0,0,0,0.05);
-        }
-
-        .page-header {
-            border-bottom: 2px solid #e9ecef;
-            padding-bottom: 1rem;
-            margin-bottom: 2rem;
-        }
-
-        .page-header h2 {
-            color: #2c3e50;
-            margin-bottom: 0.5rem;
-            font-size: 1.5rem;
-        }
-
-        .page-meta {
-            display: flex;
-            flex-wrap: wrap;
-            gap: 1rem;
-            font-size: 0.9rem;
-            color: #6c757d;
-        }
-
-        .page-content {
-            line-height: 1.8;
-        }
-
-        .page-content h1, .page-content h2, .page-content h3,
-        .page-content h4, .page-content h5, .page-content h6 {
-            color: #2c3e50;
-            margin: 1.5rem 0 0.5rem 0;
-        }
-
-        .page-content p {
-            margin: 1rem 0;
-        }
-
-        .page-content a {
-            color: #007bff;
-            text-decoration: none;
-        }
-
-        .page-content a:hover {
-            text-decoration: underline;
-        }
-
-        .page-content code {
-            background: #f8f9fa;
-            padding: 0.2rem 0.4rem;
-            border-radius: 4px;
-            font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
-            font-size: 0.9em;
-        }
-
-        .page-content pre {
-            background: #f8f9fa;
-            padding: 1rem;
-            border-radius: 4px;
-            overflow-x: auto;
-            margin: 1rem 0;
-        }
-
-        .page-content blockquote {
-            border-left: 4px solid #007bff;
-            padding-left: 1rem;
-            margin: 1rem 0;
-            color: #6c757d;
-        }
-
-        .footer {
-            text-align: center;
-            padding: 2rem;
-            margin-top: 3rem;
-            border-top: 1px solid #e9ecef;
-            color: #6c757d;
-            font-size: 0.9rem;
-        }
-
-        @media (max-width: 768px) {
-            .header {
-                padding: 1rem;
-            }
-
-            .header h1 {
-                font-size: 1.5rem;
-            }
-
-            .page {
-                padding: 1rem;
-            }
-
-            .page-meta {
-                flex-direction: column;
-                gap: 0.5rem;
-            }
-        }
-    `.trim();
-}
-function generateTOC(pages) {
-  const tocItems = pages.map((page, index) => {
-    const pageNumber = index + 1;
-    const title = page.title || `Page ${pageNumber}`;
-    const id = `page-${pageNumber}`;
-    return `<li><a href="#${id}">${pageNumber}. ${escapeHtml(title)}</a></li>`;
-  }).join("\n");
-  return `
-    <nav class="toc">
-        <h2>Table of Contents</h2>
-        <ul>
-            ${tocItems}
-        </ul>
-    </nav>`;
-}
-function generatePageHTML(page, pageNumber) {
-  const id = `page-${pageNumber}`;
-  const title = page.title || `Page ${pageNumber}`;
-  return `
-    <article class="page" id="${id}">
-        <div class="page-header">
-            <h2>${pageNumber}. ${escapeHtml(title)}</h2>
-            <div class="page-meta">
-                <span><strong>URL:</strong> <a href="${escapeHtml(
-    page.url
-  )}" target="_blank">${escapeHtml(page.url)}</a></span>
-                <span><strong>Depth:</strong> ${page.depth}</span>
-                <span><strong>Fetched:</strong> ${new Date(page.fetchedAt).toLocaleString()}</span>
-            </div>
-        </div>
-        <div class="page-content">
-            ${page.html}
-        </div>
-    </article>`;
-}
-function generateJavaScript() {
-  return `
-        // Smooth scrolling for TOC links
-        document.querySelectorAll('a[href^="#"]').forEach(anchor => {
-            anchor.addEventListener('click', function (e) {
-                e.preventDefault();
-                const target = document.querySelector(this.getAttribute('href'));
-                if (target) {
-                    target.scrollIntoView({
-                        behavior: 'smooth',
-                        block: 'start'
-                    });
-                }
-            });
-        });
-
-        // Highlight current section in TOC
-        window.addEventListener('scroll', function() {
-            const pages = document.querySelectorAll('.page');
-            const tocLinks = document.querySelectorAll('.toc a');
-            
-            let currentPage = null;
-            pages.forEach(page => {
-                const rect = page.getBoundingClientRect();
-                if (rect.top <= 100) {
-                    currentPage = page;
-                }
-            });
-
-            tocLinks.forEach(link => {
-                link.style.fontWeight = 'normal';
-                const target = document.querySelector(link.getAttribute('href'));
-                if (target === currentPage) {
-                    link.style.fontWeight = 'bold';
-                }
-            });
-        });
-    `;
-}
-function escapeHtml(text) {
-  return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, "&quot;").replace(/'/g, "&#039;").replace(/\//g, "&#x2F;");
-}
-function extractDomainFromUrl2(url) {
-  try {
-    return new URL(url).hostname;
-  } catch {
-    return "Unknown";
-  }
-}
-
-// src/formatters/json.ts
-function formatToJson(pages, baseUrl, scrapedAt, duration, website) {
-  const jsonResult = {
-    metadata: {
-      baseUrl,
-      totalPages: pages.length,
-      scrapedAt,
-      duration,
-      website
-    },
-    pages: pages.map((page, index) => ({
-      index: index + 1,
-      url: page.url,
-      title: page.title,
-      markdown: page.markdown,
-      html: page.html,
-      fetchedAt: page.fetchedAt,
-      depth: page.depth,
-      wordCount: countWords(page.markdown),
-      readingTime: estimateReadingTime(page.markdown)
-    }))
-  };
-  return JSON.stringify(jsonResult, null, 2);
-}
-function countWords(markdown) {
-  const plainText = markdown.replace(/#{1,6}\s+/g, "").replace(/\*\*(.*?)\*\*/g, "$1").replace(/\*(.*?)\*/g, "$1").replace(/`(.*?)`/g, "$1").replace(/```[\s\S]*?```/g, "").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replace(/^\s*[-*+]\s+/gm, "").replace(/^\s*\d+\.\s+/gm, "").replace(/^\s*>\s+/gm, "").replace(/\n{3,}/g, "\n\n").trim();
-  return plainText.split(/\s+/).filter((word) => word.length > 0).length;
-}
-function estimateReadingTime(markdown) {
-  const wordCount = countWords(markdown);
-  return Math.ceil(wordCount / 200);
-}
-
-// src/formatters/text.ts
-import { parseHTML } from "linkedom";
-function formatToText(pages, baseUrl, scrapedAt, duration, website, includeMetadata = true) {
-  const sections = [];
-  if (includeMetadata) {
-    sections.push(createTextHeader(baseUrl, scrapedAt, duration, website, pages.length));
-  }
-  sections.push(...pages.map((page, index) => createTextPage(page, index + 1, pages.length > 1)));
-  return sections.join("\n\n");
-}
-function createTextHeader(baseUrl, scrapedAt, duration, website, totalPages) {
-  const title = website.title || extractDomainFromUrl3(baseUrl);
-  const lines = [];
-  lines.push(`=== ${title} ===`);
-  lines.push("");
-  lines.push(`URL: ${baseUrl}`);
-  lines.push(`Scraped: ${new Date(scrapedAt).toLocaleString()}`);
-  lines.push(`Duration: ${duration}ms`);
-  lines.push(`Pages: ${totalPages}`);
-  if (website.description) {
-    lines.push(`Description: ${website.description}`);
-  }
-  if (website.author) {
-    lines.push(`Author: ${website.author}`);
-  }
-  if (website.language) {
-    lines.push(`Language: ${website.language}`);
-  }
-  return lines.join("\n");
-}
-function createTextPage(page, pageNumber, showSeparator) {
-  const lines = [];
-  if (showSeparator) {
-    lines.push("\u2500".repeat(60));
-    lines.push(`Page ${pageNumber}: ${page.title || "Untitled"}`);
-    lines.push(`URL: ${page.url}`);
-    lines.push("\u2500".repeat(60));
-  }
-  const plainText = htmlToPlainText(page.html);
-  lines.push(plainText);
-  return lines.join("\n");
-}
-function htmlToPlainText(html) {
-  const { document } = parseHTML(html);
-  const elementsToRemove = ["script", "style", "noscript", "svg", "canvas", "template"];
-  elementsToRemove.forEach((tag) => {
-    document.querySelectorAll(tag).forEach((el) => el.remove());
-  });
-  let text = document.body?.textContent || document.documentElement?.textContent || "";
-  text = text.replace(/[ \t]+/g, " ");
-  text = text.replace(/\n[ \t]+/g, "\n");
-  text = text.replace(/[ \t]+\n/g, "\n");
-  text = text.replace(/\n{3,}/g, "\n\n");
-  text = text.trim();
-  return text;
-}
-function extractDomainFromUrl3(url) {
-  try {
-    return new URL(url).hostname;
-  } catch {
-    return "Unknown";
-  }
-}
 
 // src/utils/content-cleaner.ts
-import { parseHTML as parseHTML2 } from "linkedom";
+import { parseHTML } from "linkedom";
 var ALWAYS_REMOVE_SELECTORS = [
-  // Navigation and menus
-  "nav",
-  "header nav",
-  "footer nav",
-  ".nav",
-  ".navigation",
-  ".menu",
-  ".navbar",
-  ".sidebar",
-  ".aside",
-  // Header and footer elements
-  "header",
-  "footer",
-  ".site-header",
-  ".page-header",
-  ".site-footer",
-  ".page-footer",
-  // Social media and sharing
-  ".social",
-  ".share",
-  ".sharing",
-  ".twitter",
-  ".facebook",
-  ".linkedin",
-  ".instagram",
-  // Comments and discussions
-  ".comments",
-  ".comment",
-  ".discussion",
-  ".disqus",
-  // Forms and interactive elements
-  "form",
-  "input",
-  "button:not([type='submit'])",
-  "select",
-  "textarea",
   // Scripts and styles
   "script",
   "style",
   "noscript",
+  "link[rel='stylesheet']",
   // Hidden elements
   "[hidden]",
+  "[aria-hidden='true']",
   "[style*='display: none']",
   "[style*='display:none']",
-  // Common utility classes
-  ".cookie",
-  ".cookie-banner",
-  ".popup",
+  "[style*='visibility: hidden']",
+  "[style*='visibility:hidden']",
+  // SVG icons and decorative elements
+  "svg[aria-hidden='true']",
+  "svg.icon",
+  "svg[class*='icon']",
+  // Template and metadata
+  "template",
+  "meta",
+  // Embeds that don't convert to text
+  "iframe",
+  "canvas",
+  "object",
+  "embed",
+  // Forms (usually not main content)
+  "form",
+  "input",
+  "select",
+  "textarea",
+  "button"
+];
+var OVERLAY_SELECTORS = [
+  "[class*='modal']",
+  "[class*='popup']",
+  "[class*='overlay']",
+  "[class*='dialog']",
+  "[role='dialog']",
+  "[role='alertdialog']",
+  "[class*='cookie']",
+  "[class*='consent']",
+  "[class*='gdpr']",
+  "[class*='privacy-banner']",
+  "[class*='notification-bar']",
+  "[id*='cookie']",
+  "[id*='consent']",
+  "[id*='gdpr']",
+  // Fixed/sticky positioned elements
+  "[style*='position: fixed']",
+  "[style*='position:fixed']",
+  "[style*='position: sticky']",
+  "[style*='position:sticky']"
+];
+var NAVIGATION_SELECTORS = [
+  // Semantic elements
+  "header",
+  "footer",
+  "nav",
+  "aside",
+  // Header variations
+  ".header",
+  ".top",
+  ".navbar",
+  "#header",
+  // Footer variations
+  ".footer",
+  ".bottom",
+  "#footer",
+  // Sidebars
+  ".sidebar",
+  ".side",
+  ".aside",
+  "#sidebar",
+  // Modals/popups (backup if not caught by OVERLAY_SELECTORS)
   ".modal",
+  ".popup",
+  "#modal",
   ".overlay",
-  ".notification",
+  // Ads
+  ".ad",
+  ".ads",
+  ".advert",
+  "#ad",
+  // Language selectors
+  ".lang-selector",
+  ".language",
+  "#language-selector",
+  // Social
+  ".social",
+  ".social-media",
+  ".social-links",
+  "#social",
+  // Navigation/menus
+  ".menu",
+  ".navigation",
+  "#nav",
   // Breadcrumbs
-  ".breadcrumb",
   ".breadcrumbs",
-  ".breadcrumb-trail"
+  "#breadcrumbs",
+  // Share buttons
+  ".share",
+  "#share",
+  // Widgets
+  ".widget",
+  "#widget",
+  // Cookie notices (backup)
+  ".cookie",
+  "#cookie"
+];
+var FORCE_INCLUDE_SELECTORS = [
+  // IDs
+  "#main",
+  "#content",
+  "#main-content",
+  "#article",
+  "#post",
+  "#page-content",
+  // Semantic elements
+  "main",
+  "article",
+  "[role='main']",
+  // Classes
+  ".main-content",
+  ".content",
+  ".post-content",
+  ".article-content",
+  ".entry-content",
+  ".page-content",
+  ".article-body",
+  ".post-body",
+  ".story-content",
+  ".blog-content"
 ];
 var AD_SELECTORS = [
-  // Ads and promotions
-  ".ad",
-  ".ads",
-  ".advertisement",
-  ".promotion",
-  ".sponsored",
-  "[class*='ad-']",
-  "[id*='ad-']",
-  "[class*='advert']",
-  "[id*='advert']",
-  "[class*='banner']",
-  "[id*='banner']",
+  // Google ads
+  "ins.adsbygoogle",
   ".google-ad",
   ".adsense",
+  // Generic ad containers
   "[data-ad]",
   "[data-ads]",
-  "ins.adsbygoogle",
-  // Tracking
-  "[class*='tracking']",
-  "[id*='tracking']",
-  "[class*='analytics']",
-  "[id*='analytics']"
+  "[data-ad-slot]",
+  "[data-ad-client]",
+  // Common ad class patterns
+  ".ad-container",
+  ".ad-wrapper",
+  ".advertisement",
+  ".sponsored-content",
+  // Tracking pixels
+  "img[width='1'][height='1']",
+  "img[src*='pixel']",
+  "img[src*='tracking']",
+  "img[src*='analytics']"
 ];
-function cleanHtml(html, baseUrl, options = {}) {
-  const { removeAds = true, removeBase64Images = true } = options;
-  const { document } = parseHTML2(html);
-  for (const selector of ALWAYS_REMOVE_SELECTORS) {
+function getLinkDensity(element) {
+  const text = element.textContent || "";
+  const textLength = text.trim().length;
+  if (textLength === 0) return 1;
+  let linkLength = 0;
+  element.querySelectorAll("a").forEach((link) => {
+    linkLength += (link.textContent || "").trim().length;
+  });
+  return linkLength / textLength;
+}
+function getContentScore(element) {
+  let score = 0;
+  const text = element.textContent || "";
+  const textLength = text.trim().length;
+  score += Math.min(textLength / 100, 50);
+  score += element.querySelectorAll("p").length * 3;
+  score += element.querySelectorAll("h1, h2, h3, h4, h5, h6").length * 2;
+  score += element.querySelectorAll("img").length * 1;
+  score -= element.querySelectorAll("a").length * 0.5;
+  score -= element.querySelectorAll("li").length * 0.2;
+  const linkDensity = getLinkDensity(element);
+  if (linkDensity > 0.5) score -= 30;
+  else if (linkDensity > 0.3) score -= 15;
+  const classAndId = (element.className || "") + " " + (element.id || "");
+  if (/article|content|post|body|main|entry/i.test(classAndId)) score += 25;
+  if (/comment|sidebar|footer|nav|menu|header|widget|ad/i.test(classAndId)) score -= 25;
+  return score;
+}
+function looksLikeNavigation(element) {
+  const linkDensity = getLinkDensity(element);
+  if (linkDensity > 0.5) return true;
+  const listItems = element.querySelectorAll("li");
+  const links = element.querySelectorAll("a");
+  if (listItems.length > 5 && links.length > listItems.length * 0.8) return true;
+  return false;
+}
+function removeElements(document, selectors) {
+  for (const selector of selectors) {
     try {
       document.querySelectorAll(selector).forEach((el) => el.remove());
     } catch {
     }
   }
+}
+function removeWithProtection(document, selectorsToRemove, protectedSelectors) {
+  for (const selector of selectorsToRemove) {
+    try {
+      document.querySelectorAll(selector).forEach((element) => {
+        const isProtected = protectedSelectors.some((ps) => {
+          try {
+            return element.matches(ps);
+          } catch {
+            return false;
+          }
+        });
+        if (isProtected) return;
+        const containsProtected = protectedSelectors.some((ps) => {
+          try {
+            return element.querySelector(ps) !== null;
+          } catch {
+            return false;
+          }
+        });
+        if (containsProtected) return;
+        element.remove();
+      });
+    } catch {
+    }
+  }
+}
+function findMainContent(document) {
+  const isValidContent = (el) => {
+    if (!el) return false;
+    const text = el.textContent || "";
+    if (text.trim().length < 100) return false;
+    if (looksLikeNavigation(el)) return false;
+    return true;
+  };
+  const main = document.querySelector("main");
+  if (isValidContent(main) && getLinkDensity(main) < 0.4) {
+    return main;
+  }
+  const roleMain = document.querySelector('[role="main"]');
+  if (isValidContent(roleMain) && getLinkDensity(roleMain) < 0.4) {
+    return roleMain;
+  }
+  const articles = document.querySelectorAll("article");
+  if (articles.length === 1 && isValidContent(articles[0])) {
+    return articles[0];
+  }
+  const contentSelectors = [
+    "#content",
+    "#main-content",
+    "#main",
+    ".content",
+    ".main-content",
+    ".post-content",
+    ".article-content",
+    ".entry-content",
+    ".page-content",
+    ".article-body",
+    ".post-body",
+    ".story-content",
+    ".blog-content"
+  ];
+  for (const selector of contentSelectors) {
+    try {
+      const el = document.querySelector(selector);
+      if (isValidContent(el) && getLinkDensity(el) < 0.4) {
+        return el;
+      }
+    } catch {
+    }
+  }
+  const candidates = [];
+  const containers = document.querySelectorAll("div, section, article");
+  containers.forEach((el) => {
+    const text = el.textContent || "";
+    if (text.trim().length < 200) return;
+    const score = getContentScore(el);
+    if (score > 0) {
+      candidates.push({ el, score });
+    }
+  });
+  candidates.sort((a, b) => b.score - a.score);
+  if (candidates.length > 0 && candidates[0].score > 20) {
+    return candidates[0].el;
+  }
+  return null;
+}
+function cleanHtml(html, baseUrl, options = {}) {
+  const {
+    removeAds = true,
+    removeBase64Images = true,
+    onlyMainContent = true,
+    includeTags,
+    excludeTags
+  } = options;
+  const { document } = parseHTML(html);
+  removeElements(document, ALWAYS_REMOVE_SELECTORS);
+  removeElements(document, OVERLAY_SELECTORS);
   if (removeAds) {
-    for (const selector of AD_SELECTORS) {
+    removeElements(document, AD_SELECTORS);
+  }
+  if (excludeTags && excludeTags.length > 0) {
+    removeElements(document, excludeTags);
+  }
+  if (onlyMainContent) {
+    removeWithProtection(document, NAVIGATION_SELECTORS, FORCE_INCLUDE_SELECTORS);
+    const mainContent = findMainContent(document);
+    if (mainContent) {
+      const body = document.body;
+      if (body) {
+        const clone = mainContent.cloneNode(true);
+        body.innerHTML = "";
+        body.appendChild(clone);
+      }
+    }
+  }
+  if (includeTags && includeTags.length > 0) {
+    const matchedElements = [];
+    for (const selector of includeTags) {
       try {
-        document.querySelectorAll(selector).forEach((el) => el.remove());
+        document.querySelectorAll(selector).forEach((el) => {
+          matchedElements.push(el.cloneNode(true));
+        });
       } catch {
       }
     }
+    if (matchedElements.length > 0) {
+      const body = document.body;
+      if (body) {
+        body.innerHTML = "";
+        matchedElements.forEach((el) => body.appendChild(el));
+      }
+    }
   }
   if (removeBase64Images) {
     removeBase64ImagesFromDocument(document);
@@ -834,7 +395,10 @@ function removeBase64ImagesFromDocument(document) {
   document.querySelectorAll("[style*='data:image']").forEach((el) => {
     const style = el.getAttribute("style");
     if (style) {
-      const cleanedStyle = style.replace(/background(-image)?:\s*url\([^)]*data:image[^)]*\)[^;]*;?/gi, "");
+      const cleanedStyle = style.replace(
+        /background(-image)?:\s*url\([^)]*data:image[^)]*\)[^;]*;?/gi,
+        ""
+      );
       if (cleanedStyle.trim()) {
         el.setAttribute("style", cleanedStyle);
       } else {
@@ -871,7 +435,7 @@ function cleanContent(html, baseUrl, options = {}) {
 }
 
 // src/utils/metadata-extractor.ts
-import { parseHTML as parseHTML3 } from "linkedom";
+import { parseHTML as parseHTML2 } from "linkedom";
 
 // src/utils/url-helpers.ts
 import { URL as URL2 } from "url";
@@ -944,8 +508,26 @@ function isSameDomain(url, baseUrl) {
 function getUrlKey(url) {
   try {
     const parsedUrl = new URL2(url);
+    parsedUrl.hash = "";
     parsedUrl.search = "";
-    return parsedUrl.toString().toLowerCase();
+    if (parsedUrl.hostname.startsWith("www.")) {
+      parsedUrl.hostname = parsedUrl.hostname.slice(4);
+    }
+    if (parsedUrl.protocol === "http:" && parsedUrl.port === "80" || parsedUrl.protocol === "https:" && parsedUrl.port === "443") {
+      parsedUrl.port = "";
+    }
+    const indexFiles = ["index.html", "index.htm", "default.html", "default.htm", "index.php"];
+    for (const indexFile of indexFiles) {
+      if (parsedUrl.pathname.endsWith(`/${indexFile}`)) {
+        parsedUrl.pathname = parsedUrl.pathname.slice(0, -indexFile.length);
+        break;
+      }
+    }
+    let normalized = parsedUrl.toString().toLowerCase();
+    if (normalized.endsWith("/") && parsedUrl.pathname !== "/") {
+      normalized = normalized.slice(0, -1);
+    }
+    return normalized;
   } catch {
     return url.toLowerCase();
   }
@@ -1012,7 +594,7 @@ function extractMetadata(html, baseUrl) {
   return extractWebsiteMetadata(html, baseUrl);
 }
 function extractWebsiteMetadata(html, baseUrl) {
-  const { document } = parseHTML3(html);
+  const { document } = parseHTML2(html);
   const metadata = {
     title: null,
     description: null,
@@ -1228,96 +810,934 @@ function parseRobotsTxt(content, userAgent = "*") {
       }
     }
   }
-  return rules;
-}
-function isPathAllowed(path, rules) {
-  const normalizedPath = path.startsWith("/") ? path : "/" + path;
-  for (const allowedPath of rules.allowedPaths) {
-    if (pathMatches(normalizedPath, allowedPath)) {
-      return true;
-    }
+  return rules;
+}
+function isPathAllowed(path, rules) {
+  const normalizedPath = path.startsWith("/") ? path : "/" + path;
+  for (const allowedPath of rules.allowedPaths) {
+    if (pathMatches(normalizedPath, allowedPath)) {
+      return true;
+    }
+  }
+  for (const disallowedPath of rules.disallowedPaths) {
+    if (pathMatches(normalizedPath, disallowedPath)) {
+      return false;
+    }
+  }
+  return true;
+}
+function pathMatches(path, pattern) {
+  if (!pattern) {
+    return false;
+  }
+  let regexPattern = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+  if (regexPattern.endsWith("\\$")) {
+    regexPattern = regexPattern.slice(0, -2) + "$";
+  } else {
+    regexPattern = "^" + regexPattern;
+  }
+  try {
+    const regex = new RegExp(regexPattern);
+    return regex.test(path);
+  } catch {
+    return path.startsWith(pattern);
+  }
+}
+async function fetchRobotsTxt(baseUrl) {
+  try {
+    const url = new URL("/robots.txt", baseUrl);
+    const response = await fetch(url.toString(), {
+      headers: {
+        "User-Agent": "ReaderEngine/1.0"
+      }
+    });
+    if (!response.ok) {
+      return null;
+    }
+    const content = await response.text();
+    return parseRobotsTxt(content, "ReaderEngine");
+  } catch {
+    return null;
+  }
+}
+function isUrlAllowed(url, rules) {
+  if (!rules) {
+    return true;
+  }
+  try {
+    const parsedUrl = new URL(url);
+    return isPathAllowed(parsedUrl.pathname + parsedUrl.search, rules);
+  } catch {
+    return true;
+  }
+}
+
+// src/types.ts
+var DEFAULT_OPTIONS = {
+  urls: [],
+  formats: ["markdown"],
+  timeoutMs: 3e4,
+  includePatterns: [],
+  excludePatterns: [],
+  // Content cleaning defaults
+  removeAds: true,
+  removeBase64Images: true,
+  onlyMainContent: true,
+  includeTags: [],
+  excludeTags: [],
+  skipTLSVerification: true,
+  // Batch defaults
+  batchConcurrency: 1,
+  batchTimeoutMs: 3e5,
+  maxRetries: 2,
+  onProgress: () => {
+  },
+  // Default no-op progress callback
+  // Hero-specific defaults
+  verbose: false,
+  showChrome: false
+};
+
+// src/engines/types.ts
+var ENGINE_CONFIGS = {
+  http: {
+    name: "http",
+    timeout: 3e3,
+    maxTimeout: 1e4,
+    quality: 100,
+    features: {
+      javascript: false,
+      cloudflare: false,
+      tlsFingerprint: false,
+      waitFor: false,
+      screenshots: false
+    }
+  },
+  tlsclient: {
+    name: "tlsclient",
+    timeout: 5e3,
+    maxTimeout: 15e3,
+    quality: 80,
+    features: {
+      javascript: false,
+      cloudflare: false,
+      tlsFingerprint: true,
+      waitFor: false,
+      screenshots: false
+    }
+  },
+  hero: {
+    name: "hero",
+    timeout: 3e4,
+    maxTimeout: 6e4,
+    quality: 50,
+    features: {
+      javascript: true,
+      cloudflare: true,
+      tlsFingerprint: true,
+      waitFor: true,
+      screenshots: true
+    }
+  }
+};
+var DEFAULT_ENGINE_ORDER = ["http", "tlsclient", "hero"];
+
+// src/engines/errors.ts
+var EngineError = class extends Error {
+  engine;
+  retryable;
+  constructor(engine, message, options) {
+    super(`[${engine}] ${message}`);
+    this.name = "EngineError";
+    this.engine = engine;
+    this.retryable = options?.retryable ?? true;
+    this.cause = options?.cause;
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, this.constructor);
+    }
+  }
+};
+var ChallengeDetectedError = class extends EngineError {
+  challengeType;
+  constructor(engine, challengeType) {
+    super(engine, `Challenge detected: ${challengeType || "unknown"}`, { retryable: true });
+    this.name = "ChallengeDetectedError";
+    this.challengeType = challengeType || "unknown";
+  }
+};
+var InsufficientContentError = class extends EngineError {
+  contentLength;
+  threshold;
+  constructor(engine, contentLength, threshold = 100) {
+    super(engine, `Insufficient content: ${contentLength} chars (threshold: ${threshold})`, { retryable: true });
+    this.name = "InsufficientContentError";
+    this.contentLength = contentLength;
+    this.threshold = threshold;
+  }
+};
+var HttpError = class extends EngineError {
+  statusCode;
+  constructor(engine, statusCode, statusText) {
+    const retryable = statusCode >= 500 || statusCode === 429;
+    super(engine, `HTTP ${statusCode}${statusText ? `: ${statusText}` : ""}`, { retryable });
+    this.name = "HttpError";
+    this.statusCode = statusCode;
+  }
+};
+var EngineTimeoutError = class extends EngineError {
+  timeoutMs;
+  constructor(engine, timeoutMs) {
+    super(engine, `Timeout after ${timeoutMs}ms`, { retryable: true });
+    this.name = "EngineTimeoutError";
+    this.timeoutMs = timeoutMs;
+  }
+};
+var EngineUnavailableError = class extends EngineError {
+  constructor(engine, reason) {
+    super(engine, reason || "Engine not available", { retryable: false });
+    this.name = "EngineUnavailableError";
+  }
+};
+var AllEnginesFailedError = class extends Error {
+  attemptedEngines;
+  errors;
+  constructor(attemptedEngines, errors) {
+    const summary = attemptedEngines.map((e) => `${e}: ${errors.get(e)?.message || "unknown"}`).join("; ");
+    super(`All engines failed: ${summary}`);
+    this.name = "AllEnginesFailedError";
+    this.attemptedEngines = attemptedEngines;
+    this.errors = errors;
+  }
+};
+
+// src/engines/http/index.ts
+var DEFAULT_HEADERS = {
+  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
+  "Accept-Language": "en-US,en;q=0.9",
+  "Accept-Encoding": "gzip, deflate, br",
+  "Cache-Control": "no-cache",
+  Pragma: "no-cache",
+  "Sec-Fetch-Dest": "document",
+  "Sec-Fetch-Mode": "navigate",
+  "Sec-Fetch-Site": "none",
+  "Sec-Fetch-User": "?1",
+  "Upgrade-Insecure-Requests": "1"
+};
+var CHALLENGE_PATTERNS = [
+  // Cloudflare
+  "cf-browser-verification",
+  "cf_chl_opt",
+  "challenge-platform",
+  "cf-spinner",
+  "Just a moment",
+  "Checking your browser",
+  "checking if the site connection is secure",
+  "Enable JavaScript and cookies",
+  "Attention Required",
+  "_cf_chl_tk",
+  "Verifying you are human",
+  "cf-turnstile",
+  "/cdn-cgi/challenge-platform/",
+  // Generic bot detection
+  "Please Wait...",
+  "DDoS protection by",
+  "Access denied",
+  "bot detection",
+  "are you a robot",
+  "complete the security check"
+];
+var CLOUDFLARE_INFRA_PATTERNS = ["/cdn-cgi/", "cloudflare", "__cf_bm", "cf-ray"];
+var MIN_CONTENT_LENGTH = 100;
+var HttpEngine = class {
+  config = ENGINE_CONFIGS.http;
+  async scrape(meta) {
+    const startTime = Date.now();
+    const { url, options, logger: logger4, abortSignal } = meta;
+    try {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), this.config.maxTimeout);
+      if (abortSignal) {
+        abortSignal.addEventListener("abort", () => controller.abort(), { once: true });
+      }
+      logger4?.debug(`[http] Fetching ${url}`);
+      const response = await fetch(url, {
+        method: "GET",
+        headers: {
+          ...DEFAULT_HEADERS,
+          ...options.headers || {}
+        },
+        redirect: "follow",
+        signal: controller.signal
+      });
+      clearTimeout(timeoutId);
+      const duration = Date.now() - startTime;
+      const html = await response.text();
+      logger4?.debug(`[http] Got response: ${response.status} (${html.length} chars) in ${duration}ms`);
+      if (response.status >= 400) {
+        throw new HttpError("http", response.status, response.statusText);
+      }
+      const challengeType = this.detectChallenge(html);
+      if (challengeType) {
+        logger4?.debug(`[http] Challenge detected: ${challengeType}`);
+        throw new ChallengeDetectedError("http", challengeType);
+      }
+      const textContent = this.extractText(html);
+      if (textContent.length < MIN_CONTENT_LENGTH) {
+        logger4?.debug(`[http] Insufficient content: ${textContent.length} chars`);
+        throw new InsufficientContentError("http", textContent.length, MIN_CONTENT_LENGTH);
+      }
+      return {
+        html,
+        url: response.url,
+        statusCode: response.status,
+        contentType: response.headers.get("content-type") || void 0,
+        headers: this.headersToRecord(response.headers),
+        engine: "http",
+        duration
+      };
+    } catch (error) {
+      if (error instanceof ChallengeDetectedError || error instanceof InsufficientContentError || error instanceof HttpError) {
+        throw error;
+      }
+      if (error instanceof Error) {
+        if (error.name === "AbortError") {
+          throw new EngineTimeoutError("http", this.config.maxTimeout);
+        }
+        throw new EngineError("http", error.message, { cause: error });
+      }
+      throw new EngineError("http", String(error));
+    }
+  }
+  /**
+   * Detect challenge patterns in HTML
+   * @returns Challenge type or null if no challenge detected
+   */
+  detectChallenge(html) {
+    const htmlLower = html.toLowerCase();
+    const hasCloudflare = CLOUDFLARE_INFRA_PATTERNS.some((p) => htmlLower.includes(p.toLowerCase()));
+    for (const pattern of CHALLENGE_PATTERNS) {
+      if (htmlLower.includes(pattern.toLowerCase())) {
+        if (hasCloudflare || pattern.includes("cf")) {
+          return "cloudflare";
+        }
+        return "bot-detection";
+      }
+    }
+    return null;
+  }
+  /**
+   * Convert Headers to Record<string, string>
+   */
+  headersToRecord(headers) {
+    const record = {};
+    headers.forEach((value, key) => {
+      record[key] = value;
+    });
+    return record;
+  }
+  /**
+   * Extract visible text from HTML (rough extraction)
+   */
+  extractText(html) {
+    return html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+  }
+  isAvailable() {
+    return true;
+  }
+};
+var httpEngine = new HttpEngine();
+
+// src/engines/tlsclient/index.ts
+import { gotScraping } from "got-scraping";
+var JS_REQUIRED_PATTERNS = [
+  // Cloudflare JS challenge
+  "cf-browser-verification",
+  "challenge-platform",
+  "_cf_chl_tk",
+  "/cdn-cgi/challenge-platform/",
+  // Generic JS requirements
+  "Enable JavaScript",
+  "JavaScript is required",
+  "Please enable JavaScript",
+  "requires JavaScript",
+  "noscript"
+];
+var BLOCKED_PATTERNS = [
+  "Access denied",
+  "Sorry, you have been blocked",
+  "bot detected",
+  "suspicious activity",
+  "too many requests"
+];
+var MIN_CONTENT_LENGTH2 = 100;
+var TlsClientEngine = class {
+  config = ENGINE_CONFIGS.tlsclient;
+  available = true;
+  constructor() {
+    try {
+      if (!gotScraping) {
+        this.available = false;
+      }
+    } catch {
+      this.available = false;
+    }
+  }
+  async scrape(meta) {
+    if (!this.available) {
+      throw new EngineUnavailableError("tlsclient", "got-scraping not available");
+    }
+    const startTime = Date.now();
+    const { url, options, logger: logger4, abortSignal } = meta;
+    try {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), this.config.maxTimeout);
+      if (abortSignal) {
+        abortSignal.addEventListener("abort", () => controller.abort(), { once: true });
+      }
+      logger4?.debug(`[tlsclient] Fetching ${url}`);
+      const response = await gotScraping({
+        url,
+        timeout: {
+          request: this.config.maxTimeout
+        },
+        headers: options.headers,
+        followRedirect: true
+        // got-scraping handles browser fingerprinting automatically
+        // It uses header generators and proper TLS settings
+      });
+      clearTimeout(timeoutId);
+      const duration = Date.now() - startTime;
+      const html = response.body;
+      logger4?.debug(`[tlsclient] Got response: ${response.statusCode} (${html.length} chars) in ${duration}ms`);
+      if (response.statusCode >= 400) {
+        throw new HttpError("tlsclient", response.statusCode, response.statusMessage);
+      }
+      const challengeType = this.detectJsRequired(html);
+      if (challengeType) {
+        logger4?.debug(`[tlsclient] JS required: ${challengeType}`);
+        throw new ChallengeDetectedError("tlsclient", challengeType);
+      }
+      const blockedReason = this.detectBlocked(html);
+      if (blockedReason) {
+        logger4?.debug(`[tlsclient] Blocked: ${blockedReason}`);
+        throw new ChallengeDetectedError("tlsclient", `blocked: ${blockedReason}`);
+      }
+      const textContent = this.extractText(html);
+      if (textContent.length < MIN_CONTENT_LENGTH2) {
+        logger4?.debug(`[tlsclient] Insufficient content: ${textContent.length} chars`);
+        throw new InsufficientContentError("tlsclient", textContent.length, MIN_CONTENT_LENGTH2);
+      }
+      return {
+        html,
+        url: response.url,
+        statusCode: response.statusCode,
+        contentType: response.headers["content-type"],
+        headers: response.headers,
+        engine: "tlsclient",
+        duration
+      };
+    } catch (error) {
+      if (error instanceof ChallengeDetectedError || error instanceof InsufficientContentError || error instanceof HttpError || error instanceof EngineUnavailableError) {
+        throw error;
+      }
+      if (error instanceof Error) {
+        if (error.name === "TimeoutError" || error.message.includes("timeout")) {
+          throw new EngineTimeoutError("tlsclient", this.config.maxTimeout);
+        }
+        if (error.name === "AbortError") {
+          throw new EngineTimeoutError("tlsclient", this.config.maxTimeout);
+        }
+        throw new EngineError("tlsclient", error.message, { cause: error });
+      }
+      throw new EngineError("tlsclient", String(error));
+    }
+  }
+  /**
+   * Detect patterns that require JS execution
+   */
+  detectJsRequired(html) {
+    const htmlLower = html.toLowerCase();
+    for (const pattern of JS_REQUIRED_PATTERNS) {
+      if (htmlLower.includes(pattern.toLowerCase())) {
+        if (pattern.includes("cf") || pattern.includes("cloudflare")) {
+          return "cloudflare-js";
+        }
+        return "js-required";
+      }
+    }
+    return null;
+  }
+  /**
+   * Detect blocked/denied patterns
+   */
+  detectBlocked(html) {
+    const htmlLower = html.toLowerCase();
+    for (const pattern of BLOCKED_PATTERNS) {
+      if (htmlLower.includes(pattern.toLowerCase())) {
+        return pattern;
+      }
+    }
+    return null;
+  }
+  /**
+   * Extract visible text from HTML
+   */
+  extractText(html) {
+    return html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+  }
+  isAvailable() {
+    return this.available;
+  }
+};
+var tlsClientEngine = new TlsClientEngine();
+
+// src/cloudflare/detector.ts
+var CLOUDFLARE_CHALLENGE_SELECTORS = [
+  "#challenge-running",
+  "#challenge-stage",
+  "#challenge-form",
+  ".cf-browser-verification",
+  "#cf-wrapper",
+  "#cf-hcaptcha-container",
+  "#turnstile-wrapper"
+];
+var CLOUDFLARE_TEXT_PATTERNS = [
+  "checking if the site connection is secure",
+  "this process is automatic. your browser will redirect",
+  "ray id:",
+  "performance & security by cloudflare"
+];
+var CLOUDFLARE_INFRA_PATTERNS2 = [
+  "/cdn-cgi/",
+  "cloudflare",
+  "__cf_bm",
+  "cf-ray"
+];
+var CLOUDFLARE_BLOCKED_PATTERNS = [
+  "sorry, you have been blocked",
+  "ray id:"
+];
+async function detectChallenge(hero) {
+  const signals = [];
+  let type = "none";
+  let hasCloudflareInfra = false;
+  let hasChallengeIndicator = false;
+  try {
+    if (!hero.document) {
+      return {
+        isChallenge: false,
+        type: "none",
+        confidence: 0,
+        signals: ["No document available"]
+      };
+    }
+    const html = await hero.document.documentElement.outerHTML;
+    const htmlLower = html.toLowerCase();
+    for (const pattern of CLOUDFLARE_INFRA_PATTERNS2) {
+      if (htmlLower.includes(pattern)) {
+        hasCloudflareInfra = true;
+        signals.push(`Cloudflare infra: "${pattern}"`);
+        break;
+      }
+    }
+    if (!hasCloudflareInfra) {
+      return {
+        isChallenge: false,
+        type: "none",
+        confidence: 0,
+        signals: ["No Cloudflare infrastructure detected"]
+      };
+    }
+    for (const selector of CLOUDFLARE_CHALLENGE_SELECTORS) {
+      try {
+        const element = await hero.document.querySelector(selector);
+        if (element) {
+          hasChallengeIndicator = true;
+          signals.push(`Challenge element: ${selector}`);
+          type = "js_challenge";
+        }
+      } catch {
+      }
+    }
+    for (const pattern of CLOUDFLARE_TEXT_PATTERNS) {
+      if (htmlLower.includes(pattern)) {
+        hasChallengeIndicator = true;
+        signals.push(`Challenge text: "${pattern}"`);
+        type = type === "none" ? "js_challenge" : type;
+      }
+    }
+    if (htmlLower.includes("waiting for") && htmlLower.includes("to respond")) {
+      hasChallengeIndicator = true;
+      signals.push('Challenge text: "waiting for...to respond"');
+      type = type === "none" ? "js_challenge" : type;
+    }
+    const hasBlocked = CLOUDFLARE_BLOCKED_PATTERNS.every((p) => htmlLower.includes(p));
+    if (hasBlocked) {
+      hasChallengeIndicator = true;
+      signals.push("Cloudflare block page detected");
+      type = "blocked";
+    }
+    const isChallenge = hasCloudflareInfra && hasChallengeIndicator;
+    const confidence = isChallenge ? 100 : 0;
+    return {
+      isChallenge,
+      type: isChallenge ? type : "none",
+      confidence,
+      signals
+    };
+  } catch (error) {
+    return {
+      isChallenge: false,
+      type: "none",
+      confidence: 0,
+      signals: [`Error during detection: ${error.message}`]
+    };
+  }
+}
+
+// src/cloudflare/handler.ts
+async function waitForChallengeResolution(hero, options) {
+  const { maxWaitMs = 45e3, pollIntervalMs = 500, verbose = false, initialUrl } = options;
+  const startTime = Date.now();
+  const log = (msg) => verbose && console.log(`   ${msg}`);
+  while (Date.now() - startTime < maxWaitMs) {
+    const elapsed = Date.now() - startTime;
+    try {
+      const currentUrl = await hero.url;
+      if (currentUrl !== initialUrl) {
+        log(`\u2713 URL changed: ${initialUrl} \u2192 ${currentUrl}`);
+        log(`  Waiting for new page to load...`);
+        try {
+          await hero.waitForLoad("DomContentLoaded", { timeoutMs: 3e4 });
+          log(`  DOMContentLoaded`);
+        } catch {
+          log(`  DOMContentLoaded timeout, continuing...`);
+        }
+        await hero.waitForPaintingStable().catch(() => {
+        });
+        log(`  Page stabilized`);
+        return { resolved: true, method: "url_redirect", waitedMs: elapsed };
+      }
+    } catch {
+    }
+    const detection = await detectChallenge(hero);
+    if (!detection.isChallenge) {
+      log(`\u2713 Challenge signals cleared (confidence dropped to ${detection.confidence})`);
+      log(`  Waiting for page to load...`);
+      try {
+        await hero.waitForLoad("DomContentLoaded", { timeoutMs: 3e4 });
+        log(`  DOMContentLoaded`);
+      } catch {
+        log(`  DOMContentLoaded timeout, continuing...`);
+      }
+      await hero.waitForPaintingStable().catch(() => {
+      });
+      log(`  Page stabilized`);
+      return { resolved: true, method: "signals_cleared", waitedMs: elapsed };
+    }
+    log(
+      `\u23F3 ${(elapsed / 1e3).toFixed(1)}s - Still challenge (confidence: ${detection.confidence})`
+    );
+    await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
+  }
+  return {
+    resolved: false,
+    method: "timeout",
+    waitedMs: Date.now() - startTime
+  };
+}
+
+// src/engines/hero/index.ts
+var MIN_CONTENT_LENGTH3 = 100;
+var HeroEngine = class {
+  config = ENGINE_CONFIGS.hero;
+  async scrape(meta) {
+    const startTime = Date.now();
+    const { url, options, logger: logger4, abortSignal } = meta;
+    const pool = options.pool;
+    if (!pool) {
+      throw new EngineUnavailableError("hero", "Browser pool not available");
+    }
+    if (abortSignal?.aborted) {
+      throw new EngineTimeoutError("hero", 0);
+    }
+    logger4?.debug(`[hero] Starting browser scrape of ${url}`);
+    try {
+      const result = await pool.withBrowser(async (hero) => {
+        let aborted = false;
+        if (abortSignal) {
+          abortSignal.addEventListener("abort", () => {
+            aborted = true;
+          }, { once: true });
+        }
+        const timeoutMs = options.timeoutMs || this.config.maxTimeout;
+        await hero.goto(url, { timeoutMs });
+        if (aborted) {
+          throw new EngineTimeoutError("hero", Date.now() - startTime);
+        }
+        try {
+          await hero.waitForLoad("DomContentLoaded", { timeoutMs });
+        } catch {
+        }
+        await hero.waitForPaintingStable();
+        if (aborted) {
+          throw new EngineTimeoutError("hero", Date.now() - startTime);
+        }
+        const initialUrl = await hero.url;
+        const detection = await detectChallenge(hero);
+        if (detection.isChallenge) {
+          logger4?.debug(`[hero] Challenge detected: ${detection.type}`);
+          if (detection.type === "blocked") {
+            throw new ChallengeDetectedError("hero", "blocked");
+          }
+          const resolution = await waitForChallengeResolution(hero, {
+            maxWaitMs: 45e3,
+            pollIntervalMs: 500,
+            verbose: options.verbose,
+            initialUrl
+          });
+          if (!resolution.resolved) {
+            throw new ChallengeDetectedError("hero", `unresolved: ${detection.type}`);
+          }
+          logger4?.debug(`[hero] Challenge resolved via ${resolution.method} in ${resolution.waitedMs}ms`);
+        }
+        if (aborted) {
+          throw new EngineTimeoutError("hero", Date.now() - startTime);
+        }
+        await this.waitForFinalPage(hero, url, logger4);
+        if (aborted) {
+          throw new EngineTimeoutError("hero", Date.now() - startTime);
+        }
+        if (options.waitForSelector) {
+          try {
+            await hero.waitForElement(hero.document.querySelector(options.waitForSelector), {
+              timeoutMs
+            });
+          } catch {
+            logger4?.debug(`[hero] Selector not found: ${options.waitForSelector}`);
+          }
+        }
+        const html = await hero.document.documentElement.outerHTML;
+        const finalUrl = await hero.url;
+        const textContent = this.extractText(html);
+        if (textContent.length < MIN_CONTENT_LENGTH3) {
+          logger4?.debug(`[hero] Insufficient content: ${textContent.length} chars`);
+          throw new InsufficientContentError("hero", textContent.length, MIN_CONTENT_LENGTH3);
+        }
+        const duration = Date.now() - startTime;
+        logger4?.debug(`[hero] Success: ${html.length} chars in ${duration}ms`);
+        return {
+          html,
+          url: finalUrl,
+          statusCode: 200,
+          // Hero doesn't expose status code directly
+          engine: "hero",
+          duration
+        };
+      });
+      return result;
+    } catch (error) {
+      if (error instanceof ChallengeDetectedError || error instanceof InsufficientContentError || error instanceof EngineTimeoutError || error instanceof EngineUnavailableError) {
+        throw error;
+      }
+      if (error instanceof Error) {
+        if (error.name === "TimeoutError" || error.message.includes("timeout")) {
+          throw new EngineTimeoutError("hero", this.config.maxTimeout);
+        }
+        if (error.message.includes("Navigation") || error.message.includes("ERR_")) {
+          throw new EngineError("hero", `Navigation failed: ${error.message}`, { cause: error });
+        }
+        throw new EngineError("hero", error.message, { cause: error });
+      }
+      throw new EngineError("hero", String(error));
+    }
+  }
+  /**
+   * Wait for the final page to load after any Cloudflare redirects
+   */
+  async waitForFinalPage(hero, originalUrl, logger4) {
+    const maxWaitMs = 15e3;
+    const startTime = Date.now();
+    try {
+      await hero.waitForLoad("AllContentLoaded", { timeoutMs: maxWaitMs });
+    } catch {
+    }
+    let currentUrl = await hero.url;
+    const normalizeUrl2 = (url) => url.replace(/\/+$/, "");
+    const urlChanged = normalizeUrl2(currentUrl) !== normalizeUrl2(originalUrl);
+    if (urlChanged || currentUrl.includes("__cf_chl")) {
+      logger4?.debug(`[hero] Cloudflare redirect detected: ${originalUrl} \u2192 ${currentUrl}`);
+      let lastUrl = currentUrl;
+      let stableCount = 0;
+      while (Date.now() - startTime < maxWaitMs) {
+        await new Promise((resolve) => setTimeout(resolve, 500));
+        try {
+          currentUrl = await hero.url;
+          if (currentUrl === lastUrl) {
+            stableCount++;
+            if (stableCount >= 2) {
+              break;
+            }
+          } else {
+            stableCount = 0;
+            lastUrl = currentUrl;
+            logger4?.debug(`[hero] URL changed to: ${currentUrl}`);
+          }
+        } catch {
+        }
+      }
+      try {
+        await hero.waitForLoad("AllContentLoaded", { timeoutMs: 1e4 });
+      } catch {
+      }
+    }
+    await hero.waitForPaintingStable();
+    await new Promise((resolve) => setTimeout(resolve, 2e3));
+  }
+  /**
+   * Extract visible text from HTML
+   */
+  extractText(html) {
+    return html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
   }
-  for (const disallowedPath of rules.disallowedPaths) {
-    if (pathMatches(normalizedPath, disallowedPath)) {
-      return false;
-    }
+  isAvailable() {
+    return true;
   }
-  return true;
-}
-function pathMatches(path, pattern) {
-  if (!pattern) {
-    return false;
+};
+var heroEngine = new HeroEngine();
+
+// src/engines/orchestrator.ts
+var ENGINE_REGISTRY = {
+  http: httpEngine,
+  tlsclient: tlsClientEngine,
+  hero: heroEngine
+};
+var EngineOrchestrator = class {
+  options;
+  engines;
+  engineOrder;
+  constructor(options = {}) {
+    this.options = options;
+    this.engineOrder = this.resolveEngineOrder();
+    this.engines = this.engineOrder.map((name) => ENGINE_REGISTRY[name]).filter((engine) => engine.isAvailable());
   }
-  let regexPattern = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
-  if (regexPattern.endsWith("\\$")) {
-    regexPattern = regexPattern.slice(0, -2) + "$";
-  } else {
-    regexPattern = "^" + regexPattern;
+  /**
+   * Resolve the engine order based on options
+   */
+  resolveEngineOrder() {
+    if (this.options.forceEngine) {
+      return [this.options.forceEngine];
+    }
+    let order = this.options.engines || [...DEFAULT_ENGINE_ORDER];
+    if (this.options.skipEngines) {
+      order = order.filter((e) => !this.options.skipEngines.includes(e));
+    }
+    return order;
   }
-  try {
-    const regex = new RegExp(regexPattern);
-    return regex.test(path);
-  } catch {
-    return path.startsWith(pattern);
+  /**
+   * Get available engines
+   */
+  getAvailableEngines() {
+    return this.engines.map((e) => e.config.name);
   }
-}
-async function fetchRobotsTxt(baseUrl) {
-  try {
-    const url = new URL("/robots.txt", baseUrl);
-    const response = await fetch(url.toString(), {
-      headers: {
-        "User-Agent": "ReaderEngine/1.0"
+  /**
+   * Scrape a URL using the engine cascade
+   *
+   * @param meta - Engine metadata (url, options, logger, abortSignal)
+   * @returns Scrape result with engine metadata
+   * @throws AllEnginesFailedError if all engines fail
+   */
+  async scrape(meta) {
+    const attemptedEngines = [];
+    const engineErrors = /* @__PURE__ */ new Map();
+    const logger4 = meta.logger || this.options.logger;
+    const verbose = this.options.verbose || meta.options.verbose;
+    if (this.engines.length === 0) {
+      throw new AllEnginesFailedError([], engineErrors);
+    }
+    const log = (msg) => {
+      if (verbose) {
+        logger4?.info(msg);
+      } else {
+        logger4?.debug(msg);
+      }
+    };
+    log(`[orchestrator] Starting scrape of ${meta.url} with engines: ${this.engineOrder.join(" \u2192 ")}`);
+    for (const engine of this.engines) {
+      const engineName = engine.config.name;
+      attemptedEngines.push(engineName);
+      try {
+        log(`[orchestrator] Trying ${engineName} engine...`);
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), engine.config.maxTimeout);
+        if (meta.abortSignal) {
+          meta.abortSignal.addEventListener("abort", () => controller.abort(), { once: true });
+        }
+        try {
+          const result = await engine.scrape({
+            ...meta,
+            abortSignal: controller.signal
+          });
+          clearTimeout(timeoutId);
+          log(`[orchestrator] \u2713 ${engineName} succeeded in ${result.duration}ms`);
+          return {
+            ...result,
+            attemptedEngines,
+            engineErrors
+          };
+        } finally {
+          clearTimeout(timeoutId);
+        }
+      } catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        engineErrors.set(engineName, err);
+        if (error instanceof ChallengeDetectedError) {
+          log(`[orchestrator] ${engineName} detected challenge: ${error.challengeType}`);
+        } else if (error instanceof InsufficientContentError) {
+          log(`[orchestrator] ${engineName} insufficient content: ${error.contentLength} chars`);
+        } else if (error instanceof HttpError) {
+          log(`[orchestrator] ${engineName} HTTP error: ${error.statusCode}`);
+        } else if (error instanceof EngineTimeoutError) {
+          log(`[orchestrator] ${engineName} timed out after ${error.timeoutMs}ms`);
+        } else if (error instanceof EngineUnavailableError) {
+          log(`[orchestrator] ${engineName} unavailable: ${err.message}`);
+        } else {
+          log(`[orchestrator] ${engineName} failed: ${err.message}`);
+        }
+        if (!this.shouldRetry(error)) {
+          log(`[orchestrator] Non-retryable error, stopping cascade`);
+          break;
+        }
+        log(`[orchestrator] Falling back to next engine...`);
       }
-    });
-    if (!response.ok) {
-      return null;
     }
-    const content = await response.text();
-    return parseRobotsTxt(content, "ReaderEngine");
-  } catch {
-    return null;
-  }
-}
-function isUrlAllowed(url, rules) {
-  if (!rules) {
-    return true;
+    log(`[orchestrator] All engines failed for ${meta.url}`);
+    throw new AllEnginesFailedError(attemptedEngines, engineErrors);
   }
-  try {
-    const parsedUrl = new URL(url);
-    return isPathAllowed(parsedUrl.pathname + parsedUrl.search, rules);
-  } catch {
+  /**
+   * Determine if we should retry with next engine
+   */
+  shouldRetry(error) {
+    if (error instanceof ChallengeDetectedError || error instanceof InsufficientContentError || error instanceof EngineTimeoutError) {
+      return true;
+    }
+    if (error instanceof HttpError) {
+      return error.statusCode === 403 || error.statusCode === 429 || error.statusCode >= 500;
+    }
+    if (error instanceof EngineUnavailableError) {
+      return true;
+    }
+    if (error instanceof EngineError) {
+      return error.retryable;
+    }
     return true;
   }
-}
-
-// src/types.ts
-var DEFAULT_OPTIONS = {
-  urls: [],
-  formats: ["markdown"],
-  includeMetadata: true,
-  timeoutMs: 3e4,
-  includePatterns: [],
-  excludePatterns: [],
-  // Content cleaning defaults
-  removeAds: true,
-  removeBase64Images: true,
-  skipTLSVerification: true,
-  // Batch defaults
-  batchConcurrency: 1,
-  batchTimeoutMs: 3e5,
-  maxRetries: 2,
-  onProgress: () => {
-  },
-  // Default no-op progress callback
-  // Hero-specific defaults
-  verbose: false,
-  showChrome: false
 };
 
 // src/scraper.ts
 var Scraper = class {
   options;
-  pool;
   logger = createLogger("scraper");
   robotsCache = /* @__PURE__ */ new Map();
   constructor(options) {
@@ -1325,10 +1745,6 @@ var Scraper = class {
       ...DEFAULT_OPTIONS,
       ...options
     };
-    if (!options.pool) {
-      throw new Error("Browser pool must be provided. Use ReaderClient for automatic pool management.");
-    }
-    this.pool = options.pool;
   }
   /**
    * Get robots.txt rules for a URL, cached per domain
@@ -1396,52 +1812,7 @@ var Scraper = class {
     return { result: null, error: lastError };
   }
   /**
-   * Wait for the final page to load after any Cloudflare redirects
-   * Cloudflare often does silent redirects even when bypassed, we need to ensure
-   * we're on the actual content page before scraping.
-   */
-  async waitForFinalPage(hero, originalUrl, verbose) {
-    const maxWaitMs = 15e3;
-    const startTime = Date.now();
-    const log = (msg) => verbose && this.logger.info(msg);
-    try {
-      await hero.waitForLoad("AllContentLoaded", { timeoutMs: maxWaitMs });
-    } catch {
-    }
-    let currentUrl = await hero.url;
-    const normalizeUrl2 = (url) => url.replace(/\/+$/, "");
-    const urlChanged = normalizeUrl2(currentUrl) !== normalizeUrl2(originalUrl);
-    if (urlChanged || currentUrl.includes("__cf_chl")) {
-      log(`Cloudflare redirect detected: ${originalUrl} \u2192 ${currentUrl}`);
-      let lastUrl = currentUrl;
-      let stableCount = 0;
-      while (Date.now() - startTime < maxWaitMs) {
-        await new Promise((resolve) => setTimeout(resolve, 500));
-        try {
-          currentUrl = await hero.url;
-          if (currentUrl === lastUrl) {
-            stableCount++;
-            if (stableCount >= 2) {
-              break;
-            }
-          } else {
-            stableCount = 0;
-            lastUrl = currentUrl;
-            log(`URL changed to: ${currentUrl}`);
-          }
-        } catch {
-        }
-      }
-      try {
-        await hero.waitForLoad("AllContentLoaded", { timeoutMs: 1e4 });
-      } catch {
-      }
-    }
-    await hero.waitForPaintingStable();
-    await new Promise((resolve) => setTimeout(resolve, 2e3));
-  }
-  /**
-   * Scrape a single URL
+   * Scrape a single URL using the engine orchestrator
    */
   async scrapeSingleUrl(url, index) {
     const startTime = Date.now();
@@ -1450,133 +1821,84 @@ var Scraper = class {
       throw new Error(`URL blocked by robots.txt: ${url}`);
     }
     try {
-      return await this.pool.withBrowser(async (hero) => {
-        await hero.goto(url, { timeoutMs: this.options.timeoutMs });
-        try {
-          await hero.waitForLoad("DomContentLoaded", { timeoutMs: this.options.timeoutMs });
-        } catch {
-        }
-        await hero.waitForPaintingStable();
-        let hadChallenge = false;
-        let challengeType = "none";
-        let waitTimeMs = 0;
-        const initialUrl = await hero.url;
-        const detection = await detectChallenge(hero);
-        if (detection.isChallenge) {
-          hadChallenge = true;
-          challengeType = detection.type;
-          if (this.options.verbose) {
-            this.logger.info(`Challenge detected on ${url}: ${detection.type}`);
-          }
-          const result2 = await waitForChallengeResolution(hero, {
-            maxWaitMs: 45e3,
-            pollIntervalMs: 500,
-            verbose: this.options.verbose,
-            initialUrl
-          });
-          waitTimeMs = result2.waitedMs;
-          if (!result2.resolved) {
-            throw new Error(`Challenge not resolved: ${detection.type}`);
-          }
-          if (this.options.verbose) {
-            this.logger.info(`Challenge resolved via ${result2.method} in ${waitTimeMs}ms`);
-          }
-        }
-        await this.waitForFinalPage(hero, url, this.options.verbose);
-        if (this.options.waitForSelector) {
-          try {
-            await hero.waitForElement(hero.document.querySelector(this.options.waitForSelector), {
-              timeoutMs: this.options.timeoutMs
-            });
-          } catch (error) {
-            this.logger.warn(`Selector not found: ${this.options.waitForSelector}`);
-          }
-        }
-        const pageTitle = await hero.document.title;
-        const html = await hero.document.documentElement.outerHTML;
-        const cleanedHtml = cleanContent(html, url, {
-          removeAds: this.options.removeAds,
-          removeBase64Images: this.options.removeBase64Images
+      const orchestrator = new EngineOrchestrator({
+        engines: this.options.engines,
+        skipEngines: this.options.skipEngines,
+        forceEngine: this.options.forceEngine,
+        logger: this.logger,
+        verbose: this.options.verbose
+      });
+      const engineResult = await orchestrator.scrape({
+        url,
+        options: this.options,
+        logger: this.logger
+      });
+      if (this.options.verbose) {
+        this.logger.info(
+          `[scraper] ${url} scraped with ${engineResult.engine} engine in ${engineResult.duration}ms (attempted: ${engineResult.attemptedEngines.join(" \u2192 ")})`
+        );
+      }
+      const cleanedHtml = cleanContent(engineResult.html, engineResult.url, {
+        removeAds: this.options.removeAds,
+        removeBase64Images: this.options.removeBase64Images,
+        onlyMainContent: this.options.onlyMainContent,
+        includeTags: this.options.includeTags,
+        excludeTags: this.options.excludeTags
+      });
+      const websiteMetadata = extractMetadata(cleanedHtml, engineResult.url);
+      const duration = Date.now() - startTime;
+      const markdown = this.options.formats.includes("markdown") ? htmlToMarkdown(cleanedHtml) : void 0;
+      const htmlOutput = this.options.formats.includes("html") ? cleanedHtml : void 0;
+      if (this.options.onProgress) {
+        this.options.onProgress({
+          completed: index + 1,
+          total: this.options.urls.length,
+          currentUrl: url
         });
-        const websiteMetadata = extractMetadata(cleanedHtml, url);
-        const duration = Date.now() - startTime;
-        const scrapedAt = (/* @__PURE__ */ new Date()).toISOString();
-        const page = {
-          url,
-          title: pageTitle,
-          markdown: "",
-          // Will be set by formatter
-          html: cleanedHtml,
-          fetchedAt: scrapedAt,
-          depth: 0,
-          hadChallenge,
-          challengeType,
-          waitTimeMs
-        };
-        const markdown = this.options.formats.includes("markdown") ? formatToMarkdown(
-          [page],
-          url,
-          scrapedAt,
-          duration,
-          websiteMetadata,
-          this.options.includeMetadata
-        ) : void 0;
-        const htmlOutput = this.options.formats.includes("html") ? formatToHTML([page], url, scrapedAt, duration, websiteMetadata) : void 0;
-        const json = this.options.formats.includes("json") ? formatToJson([page], url, scrapedAt, duration, websiteMetadata) : void 0;
-        const text = this.options.formats.includes("text") ? formatToText(
-          [page],
-          url,
-          scrapedAt,
-          duration,
-          websiteMetadata,
-          this.options.includeMetadata
-        ) : void 0;
-        if (this.options.onProgress) {
-          this.options.onProgress({
-            completed: index + 1,
-            total: this.options.urls.length,
-            currentUrl: url
-          });
-        }
-        let proxyMetadata;
-        if (this.options.proxy) {
-          const proxy = this.options.proxy;
-          if (proxy.url) {
-            try {
-              const proxyUrl = new URL(proxy.url);
-              proxyMetadata = {
-                host: proxyUrl.hostname,
-                port: parseInt(proxyUrl.port, 10) || 80,
-                country: proxy.country
-              };
-            } catch {
-            }
-          } else if (proxy.host && proxy.port) {
+      }
+      let proxyMetadata;
+      if (this.options.proxy) {
+        const proxy = this.options.proxy;
+        if (proxy.url) {
+          try {
+            const proxyUrl = new URL(proxy.url);
             proxyMetadata = {
-              host: proxy.host,
-              port: proxy.port,
+              host: proxyUrl.hostname,
+              port: parseInt(proxyUrl.port, 10) || 80,
               country: proxy.country
             };
+          } catch {
           }
+        } else if (proxy.host && proxy.port) {
+          proxyMetadata = {
+            host: proxy.host,
+            port: proxy.port,
+            country: proxy.country
+          };
         }
-        const result = {
-          markdown,
-          html: htmlOutput,
-          json,
-          text,
-          metadata: {
-            baseUrl: url,
-            totalPages: 1,
-            scrapedAt: (/* @__PURE__ */ new Date()).toISOString(),
-            duration,
-            website: websiteMetadata,
-            proxy: proxyMetadata
-          }
-        };
-        return result;
-      });
+      }
+      const result = {
+        markdown,
+        html: htmlOutput,
+        metadata: {
+          baseUrl: url,
+          totalPages: 1,
+          scrapedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          duration,
+          website: websiteMetadata,
+          proxy: proxyMetadata
+        }
+      };
+      return result;
     } catch (error) {
-      this.logger.error(`Failed to scrape ${url}: ${error.message}`);
+      if (error instanceof AllEnginesFailedError) {
+        const engineSummary = error.attemptedEngines.map((e) => `${e}: ${error.errors.get(e)?.message || "unknown"}`).join("; ");
+        this.logger.error(`Failed to scrape ${url}: All engines failed - ${engineSummary}`);
+      } else if (error instanceof Error) {
+        this.logger.error(`Failed to scrape ${url}: ${error.message}`);
+      } else {
+        this.logger.error(`Failed to scrape ${url}: ${String(error)}`);
+      }
       if (this.options.onProgress) {
         this.options.onProgress({
           completed: index + 1,
@@ -1618,7 +1940,7 @@ async function scrape(options) {
 }
 
 // src/crawler.ts
-import { parseHTML as parseHTML4 } from "linkedom";
+import { parseHTML as parseHTML3 } from "linkedom";
 
 // src/utils/rate-limiter.ts
 import pLimit2 from "p-limit";
@@ -1767,12 +2089,26 @@ var Crawler = class {
    */
   extractLinks(html, baseUrl, depth) {
     const links = [];
-    const { document } = parseHTML4(html);
+    const { document } = parseHTML3(html);
     document.querySelectorAll("a[href]").forEach((anchor) => {
-      const href = anchor.getAttribute("href");
+      const rawHref = anchor.getAttribute("href");
+      if (!rawHref) return;
+      const href = rawHref.trim();
       if (!href) return;
-      const resolved = resolveUrl(href, baseUrl);
+      if (href.startsWith("#")) return;
+      const lowerHref = href.toLowerCase();
+      if (lowerHref.startsWith("javascript:") || lowerHref.startsWith("mailto:") || lowerHref.startsWith("tel:") || lowerHref.startsWith("data:") || lowerHref.startsWith("blob:") || lowerHref.startsWith("ftp:")) {
+        return;
+      }
+      let resolved = resolveUrl(href, baseUrl);
       if (!resolved || !isValidUrl(resolved)) return;
+      try {
+        const parsed = new URL(resolved);
+        parsed.hash = "";
+        resolved = parsed.toString();
+      } catch {
+        return;
+      }
       if (!isSameDomain(resolved, this.options.url)) return;
       if (!isContentUrl(resolved)) return;
       if (!shouldIncludeUrl(resolved, this.options.includePatterns, this.options.excludePatterns)) return;
@@ -2883,9 +3219,9 @@ program.command("status").description("Check daemon status").option("-p, --port
 });
 program.command("scrape <urls...>").description("Scrape one or more URLs").option(
   "-f, --format <formats>",
-  "Output formats (comma-separated: markdown,html,json,text)",
+  "Content formats to include (comma-separated: markdown,html)",
   "markdown"
-).option("-o, --output <file>", "Output file (stdout if omitted)").option("-c, --concurrency <n>", "Parallel requests", "1").option("-t, --timeout <ms>", "Request timeout in milliseconds", "30000").option("--proxy <url>", "Proxy URL (e.g., http://user:pass@host:port)").option("--user-agent <string>", "Custom user agent string").option("--batch-timeout <ms>", "Total timeout for entire batch operation", "300000").option("--show-chrome", "Show browser window for debugging").option("--no-metadata", "Exclude metadata from output").option("--standalone", "Force standalone mode (bypass daemon)").option("-p, --port <n>", `Daemon port (default: ${DEFAULT_DAEMON_PORT})`, String(DEFAULT_DAEMON_PORT)).option("-v, --verbose", "Enable verbose logging").action(async (urls, options) => {
+).option("-o, --output <file>", "Output file (stdout if omitted)").option("-c, --concurrency <n>", "Parallel requests", "1").option("-t, --timeout <ms>", "Request timeout in milliseconds", "30000").option("--proxy <url>", "Proxy URL (e.g., http://user:pass@host:port)").option("--user-agent <string>", "Custom user agent string").option("--batch-timeout <ms>", "Total timeout for entire batch operation", "300000").option("--show-chrome", "Show browser window for debugging").option("--standalone", "Force standalone mode (bypass daemon)").option("-p, --port <n>", `Daemon port (default: ${DEFAULT_DAEMON_PORT})`, String(DEFAULT_DAEMON_PORT)).option("-v, --verbose", "Enable verbose logging").option("--no-main-content", "Disable main content extraction (include full page)").option("--include-tags <selectors>", "CSS selectors for elements to include (comma-separated)").option("--exclude-tags <selectors>", "CSS selectors for elements to exclude (comma-separated)").option("--engine <name>", "Force a specific engine (http, tlsclient, hero)").option("--skip-engine <names>", "Skip specific engines (comma-separated: http,tlsclient,hero)").action(async (urls, options) => {
   const port = parseInt(options.port, 10);
   const useStandalone = options.standalone || false;
   let useDaemon = false;
@@ -2902,7 +3238,7 @@ program.command("scrape <urls...>").description("Scrape one or more URLs").optio
   }) : null;
   try {
     const formats = options.format.split(",").map((f) => f.trim());
-    const validFormats = ["markdown", "html", "json", "text"];
+    const validFormats = ["markdown", "html"];
     for (const format of formats) {
       if (!validFormats.includes(format)) {
         console.error(
@@ -2915,6 +3251,9 @@ program.command("scrape <urls...>").description("Scrape one or more URLs").optio
       console.error(`Scraping ${urls.length} URL(s)...`);
       console.error(`Formats: ${formats.join(", ")}`);
     }
+    const includeTags = options.includeTags ? options.includeTags.split(",").map((s) => s.trim()) : void 0;
+    const excludeTags = options.excludeTags ? options.excludeTags.split(",").map((s) => s.trim()) : void 0;
+    const skipEngines = options.skipEngine ? options.skipEngine.split(",").map((s) => s.trim()) : void 0;
     const scrapeOptions = {
       urls,
       formats,
@@ -2923,33 +3262,29 @@ program.command("scrape <urls...>").description("Scrape one or more URLs").optio
       batchTimeoutMs: parseInt(options.batchTimeout, 10),
       proxy: options.proxy ? { url: options.proxy } : void 0,
       userAgent: options.userAgent,
-      includeMetadata: options.metadata !== false,
       verbose: options.verbose || false,
       showChrome: options.showChrome || false,
+      // Content cleaning options
+      onlyMainContent: options.mainContent !== false,
+      // --no-main-content sets this to false
+      includeTags,
+      excludeTags,
+      // Engine options
+      forceEngine: options.engine,
+      skipEngines,
       onProgress: options.verbose ? ({ completed, total, currentUrl }) => {
         console.error(`[${completed}/${total}] ${currentUrl}`);
       } : void 0
     };
     const result = useDaemon ? await daemonClient.scrape(scrapeOptions) : await standaloneClient.scrape(scrapeOptions);
-    let output = "";
-    for (const site of result.data) {
-      if (formats.includes("markdown") && site.markdown) {
-        output += site.markdown + "\n\n";
-      } else if (formats.includes("text") && site.text) {
-        output += site.text + "\n\n";
-      } else if (formats.includes("html") && site.html) {
-        output += site.html + "\n\n";
-      } else if (formats.includes("json") && site.json) {
-        output += site.json + "\n\n";
-      }
-    }
+    const output = JSON.stringify(result, null, 2);
     if (options.output) {
-      writeFileSync(options.output, output.trim());
+      writeFileSync(options.output, output);
       if (options.verbose) {
         console.error(`Output written to ${options.output}`);
       }
     } else {
-      console.log(output.trim());
+      console.log(output);
     }
     if (options.verbose) {
       console.error(`
@@ -2972,7 +3307,7 @@ Summary:`);
     }
   }
 });
-program.command("crawl <url>").description("Crawl a website to discover and optionally scrape pages").option("-d, --depth <n>", "Maximum crawl depth", "1").option("-m, --max-pages <n>", "Maximum pages to discover", "20").option("-s, --scrape", "Also scrape content of discovered pages").option("-f, --format <formats>", "Output formats when scraping (comma-separated)", "markdown").option("-o, --output <file>", "Output file (stdout if omitted)").option("--delay <ms>", "Delay between requests in milliseconds", "1000").option("-t, --timeout <ms>", "Total timeout for crawl operation in milliseconds").option("--include <patterns>", "URL patterns to include (comma-separated regex)").option("--exclude <patterns>", "URL patterns to exclude (comma-separated regex)").option("--proxy <url>", "Proxy URL (e.g., http://user:pass@host:port)").option("--user-agent <string>", "Custom user agent string").option("--show-chrome", "Show browser window for debugging").option("--standalone", "Force standalone mode (bypass daemon)").option("-p, --port <n>", `Daemon port (default: ${DEFAULT_DAEMON_PORT})`, String(DEFAULT_DAEMON_PORT)).option("-v, --verbose", "Enable verbose logging").action(async (url, options) => {
+program.command("crawl <url>").description("Crawl a website to discover and optionally scrape pages").option("-d, --depth <n>", "Maximum crawl depth", "1").option("-m, --max-pages <n>", "Maximum pages to discover", "20").option("-s, --scrape", "Also scrape content of discovered pages").option("-f, --format <formats>", "Content formats when scraping (comma-separated: markdown,html)", "markdown").option("-o, --output <file>", "Output file (stdout if omitted)").option("--delay <ms>", "Delay between requests in milliseconds", "1000").option("-t, --timeout <ms>", "Total timeout for crawl operation in milliseconds").option("--include <patterns>", "URL patterns to include (comma-separated regex)").option("--exclude <patterns>", "URL patterns to exclude (comma-separated regex)").option("--proxy <url>", "Proxy URL (e.g., http://user:pass@host:port)").option("--user-agent <string>", "Custom user agent string").option("--show-chrome", "Show browser window for debugging").option("--standalone", "Force standalone mode (bypass daemon)").option("-p, --port <n>", `Daemon port (default: ${DEFAULT_DAEMON_PORT})`, String(DEFAULT_DAEMON_PORT)).option("-v, --verbose", "Enable verbose logging").action(async (url, options) => {
   const port = parseInt(options.port, 10);
   const useStandalone = options.standalone || false;
   let useDaemon = false;
@@ -3008,38 +3343,20 @@ program.command("crawl <url>").description("Crawl a website to discover and opti
       verbose: options.verbose || false,
       showChrome: options.showChrome || false
     };
-    const result = useDaemon ? await daemonClient.crawl(crawlOptions) : await standaloneClient.crawl(crawlOptions);
-    let output = "";
-    if (options.scrape && result.scraped) {
-      const formats = options.format.split(",").map((f) => f.trim());
-      for (const site of result.scraped.data) {
-        if (formats.includes("markdown") && site.markdown) {
-          output += site.markdown + "\n\n";
-        } else if (formats.includes("text") && site.text) {
-          output += site.text + "\n\n";
-        } else if (formats.includes("html") && site.html) {
-          output += site.html + "\n\n";
-        } else if (formats.includes("json") && site.json) {
-          output += site.json + "\n\n";
-        }
-      }
-    } else {
-      output = JSON.stringify(
-        {
-          urls: result.urls,
-          metadata: result.metadata
-        },
-        null,
-        2
-      );
-    }
+    const formats = options.format.split(",").map((f) => f.trim());
+    const crawlOptionsWithFormats = {
+      ...crawlOptions,
+      formats
+    };
+    const result = useDaemon ? await daemonClient.crawl(crawlOptionsWithFormats) : await standaloneClient.crawl(crawlOptionsWithFormats);
+    const output = JSON.stringify(result, null, 2);
     if (options.output) {
-      writeFileSync(options.output, output.trim());
+      writeFileSync(options.output, output);
       if (options.verbose) {
         console.error(`Output written to ${options.output}`);
       }
     } else {
-      console.log(output.trim());
+      console.log(output);
     }
     if (options.verbose) {
       console.error(`