@vakra-dev/reader 0.0.1

package/dist/index.js

@@ -0,0 +1,3073 @@
+// src/client.ts
+import HeroCore from "@ulixee/hero-core";
+import { TransportBridge } from "@ulixee/net";
+import { ConnectionToHeroCore } from "@ulixee/hero";
+
+// src/scraper.ts
+import pLimit from "p-limit";
+
+// src/cloudflare/detector.ts
+var CHALLENGE_DOM_SELECTORS = [
+  "#challenge-running",
+  "#challenge-stage",
+  "#challenge-form",
+  ".cf-browser-verification"
+];
+var CHALLENGE_TEXT_PATTERNS = [
+  "verifying you are human",
+  "checking if the site connection is secure",
+  "this process is automatic. your browser will redirect"
+];
+var BLOCKED_SIGNALS = [
+  "you have been blocked",
+  "access to this page has been denied",
+  "sorry, you have been blocked",
+  "access denied",
+  "403 forbidden"
+];
+async function detectChallenge(hero) {
+  const signals = [];
+  let type = "none";
+  try {
+    if (!hero.document) {
+      return {
+        isChallenge: false,
+        type: "none",
+        confidence: 0,
+        signals: ["No document available"]
+      };
+    }
+    const html = await hero.document.documentElement.outerHTML;
+    const htmlLower = html.toLowerCase();
+    for (const selector of CHALLENGE_DOM_SELECTORS) {
+      if (htmlLower.includes(selector.toLowerCase())) {
+        signals.push(`Challenge element: ${selector}`);
+        type = "js_challenge";
+      }
+    }
+    for (const pattern of CHALLENGE_TEXT_PATTERNS) {
+      if (htmlLower.includes(pattern)) {
+        signals.push(`Challenge text: "${pattern}"`);
+        type = type === "none" ? "js_challenge" : type;
+      }
+    }
+    if (htmlLower.includes("waiting for") && htmlLower.includes("to respond")) {
+      signals.push('Challenge text: "waiting for...to respond"');
+      type = type === "none" ? "js_challenge" : type;
+    }
+    for (const pattern of BLOCKED_SIGNALS) {
+      if (htmlLower.includes(pattern)) {
+        signals.push(`Blocked: "${pattern}"`);
+        type = "blocked";
+        break;
+      }
+    }
+    const isChallenge = signals.length > 0;
+    const confidence = isChallenge ? 100 : 0;
+    return {
+      isChallenge,
+      type: isChallenge ? type : "none",
+      confidence,
+      signals
+    };
+  } catch (error) {
+    return {
+      isChallenge: false,
+      type: "none",
+      confidence: 0,
+      signals: [`Error during detection: ${error.message}`]
+    };
+  }
+}
+async function isChallengePage(hero) {
+  const detection = await detectChallenge(hero);
+  return detection.isChallenge;
+}
+
+// src/cloudflare/handler.ts
+async function waitForChallengeResolution(hero, options) {
+  const { maxWaitMs = 45e3, pollIntervalMs = 500, verbose = false, initialUrl } = options;
+  const startTime = Date.now();
+  const log = (msg) => verbose && console.log(`   ${msg}`);
+  while (Date.now() - startTime < maxWaitMs) {
+    const elapsed = Date.now() - startTime;
+    try {
+      const currentUrl = await hero.url;
+      if (currentUrl !== initialUrl) {
+        log(`\u2713 URL changed: ${initialUrl} \u2192 ${currentUrl}`);
+        log(`  Waiting for new page to load...`);
+        try {
+          await hero.waitForLoad("DomContentLoaded", { timeoutMs: 3e4 });
+          log(`  DOMContentLoaded`);
+        } catch {
+          log(`  DOMContentLoaded timeout, continuing...`);
+        }
+        await hero.waitForPaintingStable().catch(() => {
+        });
+        log(`  Page stabilized`);
+        return { resolved: true, method: "url_redirect", waitedMs: elapsed };
+      }
+    } catch {
+    }
+    const detection = await detectChallenge(hero);
+    if (!detection.isChallenge) {
+      log(`\u2713 Challenge signals cleared (confidence dropped to ${detection.confidence})`);
+      log(`  Waiting for page to load...`);
+      try {
+        await hero.waitForLoad("DomContentLoaded", { timeoutMs: 3e4 });
+        log(`  DOMContentLoaded`);
+      } catch {
+        log(`  DOMContentLoaded timeout, continuing...`);
+      }
+      await hero.waitForPaintingStable().catch(() => {
+      });
+      log(`  Page stabilized`);
+      return { resolved: true, method: "signals_cleared", waitedMs: elapsed };
+    }
+    log(
+      `\u23F3 ${(elapsed / 1e3).toFixed(1)}s - Still challenge (confidence: ${detection.confidence})`
+    );
+    await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
+  }
+  return {
+    resolved: false,
+    method: "timeout",
+    waitedMs: Date.now() - startTime
+  };
+}
+async function waitForSelector(hero, selector, maxWaitMs, verbose = false) {
+  const startTime = Date.now();
+  const log = (msg) => verbose && console.log(`   ${msg}`);
+  log(`Waiting for selector: "${selector}"`);
+  while (Date.now() - startTime < maxWaitMs) {
+    try {
+      const element = await hero.document.querySelector(selector);
+      if (element) {
+        const elapsed = Date.now() - startTime;
+        log(`\u2713 Selector found after ${(elapsed / 1e3).toFixed(1)}s`);
+        return { found: true, waitedMs: elapsed };
+      }
+    } catch {
+    }
+    await new Promise((resolve) => setTimeout(resolve, 300));
+  }
+  log(`\u2717 Selector not found within timeout`);
+  return { found: false, waitedMs: Date.now() - startTime };
+}
+async function handleChallenge(hero, options = {}) {
+  const initialUrl = await hero.url;
+  const detection = await detectChallenge(hero);
+  if (!detection.isChallenge) {
+    return { resolved: true, method: "signals_cleared", waitedMs: 0 };
+  }
+  return waitForChallengeResolution(hero, {
+    ...options,
+    initialUrl
+  });
+}
+
+// src/formatters/markdown.ts
+import TurndownService from "turndown";
+var turndownService = new TurndownService({
+  headingStyle: "atx",
+  hr: "---",
+  bulletListMarker: "-",
+  codeBlockStyle: "fenced",
+  fence: "```",
+  emDelimiter: "*",
+  strongDelimiter: "**",
+  linkStyle: "inlined",
+  linkReferenceStyle: "full"
+});
+function formatToMarkdown(pages, baseUrl, scrapedAt, duration, website, includeMetadata = true) {
+  const sections = [];
+  if (includeMetadata) {
+    sections.push(createMarkdownHeader(baseUrl, scrapedAt, duration, website, pages.length));
+  }
+  if (pages.length > 1) {
+    sections.push(createMarkdownTOC(pages));
+  }
+  sections.push(...pages.map((page, index) => createMarkdownPage(page, index + 1)));
+  return sections.join("\n\n");
+}
+function createMarkdownHeader(baseUrl, scrapedAt, duration, website, totalPages) {
+  const title = website.title || extractDomainFromUrl(baseUrl);
+  const description = website.description || "";
+  let header = `# Website Scrape: ${title}
+
+`;
+  header += `**Base URL:** ${baseUrl}
+`;
+  header += `**Scraped at:** ${new Date(scrapedAt).toLocaleString()}
+`;
+  header += `**Duration:** ${duration}ms
+`;
+  header += `**Total pages:** ${totalPages}
+`;
+  if (description) {
+    header += `**Description:** ${description}
+`;
+  }
+  if (website.author) {
+    header += `**Author:** ${website.author}
+`;
+  }
+  if (website.language) {
+    header += `**Language:** ${website.language}
+`;
+  }
+  return header;
+}
+function createMarkdownTOC(pages) {
+  let toc = "## Table of Contents\n\n";
+  pages.forEach((page, index) => {
+    const depth = "  ".repeat(page.depth);
+    const pageNumber = index + 1;
+    const title = page.title || `Page ${pageNumber}`;
+    const cleanTitle = title.replace(/[#[\]/\\:*?"<>|]/g, "").trim();
+    const anchor = cleanTitle.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
+    toc += `${depth}${pageNumber}. [${title}](#page-${pageNumber}-${anchor})
+`;
+  });
+  return toc;
+}
+function createMarkdownPage(page, pageNumber) {
+  const title = page.title || `Page ${pageNumber}`;
+  const cleanTitle = title.replace(/[#[\]/\\:*?"<>|]/g, "").trim();
+  const anchor = cleanTitle.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
+  let pageContent = `---
+
+`;
+  pageContent += `## Page ${pageNumber}: ${title} {#page-${pageNumber}-${anchor}}
+
+`;
+  pageContent += `**URL:** ${page.url}
+`;
+  pageContent += `**Title:** ${page.title}
+`;
+  pageContent += `**Depth:** ${page.depth}
+`;
+  pageContent += `**Fetched at:** ${new Date(page.fetchedAt).toLocaleString()}
+
+`;
+  pageContent += `---
+
+`;
+  const markdown = htmlToMarkdown(page.html);
+  pageContent += markdown;
+  return pageContent;
+}
+function htmlToMarkdown(html) {
+  try {
+    return turndownService.turndown(html);
+  } catch (error) {
+    console.warn("Error converting HTML to Markdown:", error);
+    return html.replace(/<[^>]*>/g, "").trim();
+  }
+}
+function extractDomainFromUrl(url) {
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return "Unknown";
+  }
+}
+
+// src/formatters/html.ts
+function formatToHTML(pages, baseUrl, scrapedAt, duration, website) {
+  const html = `<!DOCTYPE html>
+<html lang="${website.language || "en"}">
+<head>
+    <meta charset="${website.charset || "UTF-8"}">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Scrape: ${website.title || extractDomainFromUrl2(baseUrl)}</title>
+    ${generateMetaTags(website)}
+    <style>
+        ${generateCSS()}
+    </style>
+</head>
+<body>
+    <header class="header">
+        <h1>Website Scrape: ${escapeHtml(website.title || extractDomainFromUrl2(baseUrl))}</h1>
+        <div class="meta-info">
+            <p><strong>Base URL:</strong> <a href="${escapeHtml(
+    baseUrl
+  )}" target="_blank">${escapeHtml(baseUrl)}</a></p>
+            <p><strong>Scraped at:</strong> ${new Date(scrapedAt).toLocaleString()}</p>
+            <p><strong>Duration:</strong> ${duration}ms</p>
+            <p><strong>Total pages:</strong> ${pages.length}</p>
+            ${website.description ? `<p><strong>Description:</strong> ${escapeHtml(website.description)}</p>` : ""}
+            ${website.author ? `<p><strong>Author:</strong> ${escapeHtml(website.author)}</p>` : ""}
+            ${website.language ? `<p><strong>Language:</strong> ${escapeHtml(website.language)}</p>` : ""}
+        </div>
+    </header>
+
+    ${pages.length > 1 ? generateTOC(pages) : ""}
+
+    <main class="content">
+        ${pages.map((page, index) => generatePageHTML(page, index + 1)).join("\n")}
+    </main>
+
+    <footer class="footer">
+        <p>Generated by Reader JS/TS SDK</p>
+    </footer>
+
+    <script>
+        ${generateJavaScript()}
+    </script>
+</body>
+</html>`;
+  return html;
+}
+function generateMetaTags(website) {
+  const tags = [];
+  if (website.description) {
+    tags.push(`<meta name="description" content="${escapeHtml(website.description)}">`);
+  }
+  if (website.author) {
+    tags.push(`<meta name="author" content="${escapeHtml(website.author)}">`);
+  }
+  if (website.keywords) {
+    tags.push(`<meta name="keywords" content="${escapeHtml(website.keywords.join(", "))}">`);
+  }
+  if (website.robots) {
+    tags.push(`<meta name="robots" content="${escapeHtml(website.robots)}">`);
+  }
+  if (website.themeColor) {
+    tags.push(`<meta name="theme-color" content="${escapeHtml(website.themeColor)}">`);
+  }
+  if (website.favicon) {
+    tags.push(`<link rel="icon" href="${escapeHtml(website.favicon)}">`);
+  }
+  if (website.canonical) {
+    tags.push(`<link rel="canonical" href="${escapeHtml(website.canonical)}">`);
+  }
+  if (website.openGraph) {
+    const og = website.openGraph;
+    if (og.title) tags.push(`<meta property="og:title" content="${escapeHtml(og.title)}">`);
+    if (og.description)
+      tags.push(`<meta property="og:description" content="${escapeHtml(og.description)}">`);
+    if (og.type) tags.push(`<meta property="og:type" content="${escapeHtml(og.type)}">`);
+    if (og.url) tags.push(`<meta property="og:url" content="${escapeHtml(og.url)}">`);
+    if (og.image) tags.push(`<meta property="og:image" content="${escapeHtml(og.image)}">`);
+    if (og.siteName)
+      tags.push(`<meta property="og:site_name" content="${escapeHtml(og.siteName)}">`);
+    if (og.locale) tags.push(`<meta property="og:locale" content="${escapeHtml(og.locale)}">`);
+  }
+  if (website.twitter) {
+    const twitter = website.twitter;
+    if (twitter.card) tags.push(`<meta name="twitter:card" content="${escapeHtml(twitter.card)}">`);
+    if (twitter.site) tags.push(`<meta name="twitter:site" content="${escapeHtml(twitter.site)}">`);
+    if (twitter.creator)
+      tags.push(`<meta name="twitter:creator" content="${escapeHtml(twitter.creator)}">`);
+    if (twitter.title)
+      tags.push(`<meta name="twitter:title" content="${escapeHtml(twitter.title)}">`);
+    if (twitter.description)
+      tags.push(`<meta name="twitter:description" content="${escapeHtml(twitter.description)}">`);
+    if (twitter.image)
+      tags.push(`<meta name="twitter:image" content="${escapeHtml(twitter.image)}">`);
+  }
+  return tags.join("\n    ");
+}
+function generateCSS() {
+  return `
+        * {
+            margin: 0;
+            padding: 0;
+            box-sizing: border-box;
+        }
+
+        body {
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+            line-height: 1.6;
+            color: #333;
+            background-color: #f8f9fa;
+        }
+
+        .header {
+            background: white;
+            padding: 2rem;
+            border-bottom: 1px solid #e9ecef;
+            margin-bottom: 2rem;
+        }
+
+        .header h1 {
+            color: #2c3e50;
+            margin-bottom: 1rem;
+            font-size: 2rem;
+        }
+
+        .meta-info {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+            gap: 0.5rem;
+        }
+
+        .meta-info p {
+            margin: 0.25rem 0;
+            font-size: 0.9rem;
+            color: #6c757d;
+        }
+
+        .toc {
+            background: white;
+            padding: 1.5rem;
+            margin: 2rem 0;
+            border-radius: 8px;
+            border: 1px solid #e9ecef;
+        }
+
+        .toc h2 {
+            color: #2c3e50;
+            margin-bottom: 1rem;
+            font-size: 1.25rem;
+        }
+
+        .toc ul {
+            list-style: none;
+        }
+
+        .toc li {
+            margin: 0.5rem 0;
+        }
+
+        .toc a {
+            color: #007bff;
+            text-decoration: none;
+            transition: color 0.2s;
+        }
+
+        .toc a:hover {
+            color: #0056b3;
+            text-decoration: underline;
+        }
+
+        .content {
+            max-width: 800px;
+            margin: 0 auto;
+            padding: 0 1rem;
+        }
+
+        .page {
+            background: white;
+            margin: 2rem 0;
+            padding: 2rem;
+            border-radius: 8px;
+            border: 1px solid #e9ecef;
+            box-shadow: 0 2px 4px rgba(0,0,0,0.05);
+        }
+
+        .page-header {
+            border-bottom: 2px solid #e9ecef;
+            padding-bottom: 1rem;
+            margin-bottom: 2rem;
+        }
+
+        .page-header h2 {
+            color: #2c3e50;
+            margin-bottom: 0.5rem;
+            font-size: 1.5rem;
+        }
+
+        .page-meta {
+            display: flex;
+            flex-wrap: wrap;
+            gap: 1rem;
+            font-size: 0.9rem;
+            color: #6c757d;
+        }
+
+        .page-content {
+            line-height: 1.8;
+        }
+
+        .page-content h1, .page-content h2, .page-content h3,
+        .page-content h4, .page-content h5, .page-content h6 {
+            color: #2c3e50;
+            margin: 1.5rem 0 0.5rem 0;
+        }
+
+        .page-content p {
+            margin: 1rem 0;
+        }
+
+        .page-content a {
+            color: #007bff;
+            text-decoration: none;
+        }
+
+        .page-content a:hover {
+            text-decoration: underline;
+        }
+
+        .page-content code {
+            background: #f8f9fa;
+            padding: 0.2rem 0.4rem;
+            border-radius: 4px;
+            font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
+            font-size: 0.9em;
+        }
+
+        .page-content pre {
+            background: #f8f9fa;
+            padding: 1rem;
+            border-radius: 4px;
+            overflow-x: auto;
+            margin: 1rem 0;
+        }
+
+        .page-content blockquote {
+            border-left: 4px solid #007bff;
+            padding-left: 1rem;
+            margin: 1rem 0;
+            color: #6c757d;
+        }
+
+        .footer {
+            text-align: center;
+            padding: 2rem;
+            margin-top: 3rem;
+            border-top: 1px solid #e9ecef;
+            color: #6c757d;
+            font-size: 0.9rem;
+        }
+
+        @media (max-width: 768px) {
+            .header {
+                padding: 1rem;
+            }
+
+            .header h1 {
+                font-size: 1.5rem;
+            }
+
+            .page {
+                padding: 1rem;
+            }
+
+            .page-meta {
+                flex-direction: column;
+                gap: 0.5rem;
+            }
+        }
+    `.trim();
+}
+function generateTOC(pages) {
+  const tocItems = pages.map((page, index) => {
+    const pageNumber = index + 1;
+    const title = page.title || `Page ${pageNumber}`;
+    const id = `page-${pageNumber}`;
+    return `<li><a href="#${id}">${pageNumber}. ${escapeHtml(title)}</a></li>`;
+  }).join("\n");
+  return `
+    <nav class="toc">
+        <h2>Table of Contents</h2>
+        <ul>
+            ${tocItems}
+        </ul>
+    </nav>`;
+}
+function generatePageHTML(page, pageNumber) {
+  const id = `page-${pageNumber}`;
+  const title = page.title || `Page ${pageNumber}`;
+  return `
+    <article class="page" id="${id}">
+        <div class="page-header">
+            <h2>${pageNumber}. ${escapeHtml(title)}</h2>
+            <div class="page-meta">
+                <span><strong>URL:</strong> <a href="${escapeHtml(
+    page.url
+  )}" target="_blank">${escapeHtml(page.url)}</a></span>
+                <span><strong>Depth:</strong> ${page.depth}</span>
+                <span><strong>Fetched:</strong> ${new Date(page.fetchedAt).toLocaleString()}</span>
+            </div>
+        </div>
+        <div class="page-content">
+            ${page.html}
+        </div>
+    </article>`;
+}
+function generateJavaScript() {
+  return `
+        // Smooth scrolling for TOC links
+        document.querySelectorAll('a[href^="#"]').forEach(anchor => {
+            anchor.addEventListener('click', function (e) {
+                e.preventDefault();
+                const target = document.querySelector(this.getAttribute('href'));
+                if (target) {
+                    target.scrollIntoView({
+                        behavior: 'smooth',
+                        block: 'start'
+                    });
+                }
+            });
+        });
+
+        // Highlight current section in TOC
+        window.addEventListener('scroll', function() {
+            const pages = document.querySelectorAll('.page');
+            const tocLinks = document.querySelectorAll('.toc a');
+            
+            let currentPage = null;
+            pages.forEach(page => {
+                const rect = page.getBoundingClientRect();
+                if (rect.top <= 100) {
+                    currentPage = page;
+                }
+            });
+
+            tocLinks.forEach(link => {
+                link.style.fontWeight = 'normal';
+                const target = document.querySelector(link.getAttribute('href'));
+                if (target === currentPage) {
+                    link.style.fontWeight = 'bold';
+                }
+            });
+        });
+    `;
+}
+function escapeHtml(text) {
+  return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, "&quot;").replace(/'/g, "&#039;").replace(/\//g, "&#x2F;");
+}
+function extractDomainFromUrl2(url) {
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return "Unknown";
+  }
+}
+
+// src/formatters/json.ts
+function formatToJson(pages, baseUrl, scrapedAt, duration, website) {
+  const jsonResult = {
+    metadata: {
+      baseUrl,
+      totalPages: pages.length,
+      scrapedAt,
+      duration,
+      website
+    },
+    pages: pages.map((page, index) => ({
+      index: index + 1,
+      url: page.url,
+      title: page.title,
+      markdown: page.markdown,
+      html: page.html,
+      fetchedAt: page.fetchedAt,
+      depth: page.depth,
+      wordCount: countWords(page.markdown),
+      readingTime: estimateReadingTime(page.markdown)
+    }))
+  };
+  return JSON.stringify(jsonResult, null, 2);
+}
+function formatToJsonLite(pages, baseUrl, scrapedAt, duration, website) {
+  const jsonResult = {
+    metadata: {
+      baseUrl,
+      totalPages: pages.length,
+      scrapedAt,
+      duration,
+      website
+    },
+    pages: pages.map((page, index) => ({
+      index: index + 1,
+      url: page.url,
+      title: page.title,
+      markdown: page.markdown,
+      fetchedAt: page.fetchedAt,
+      depth: page.depth,
+      wordCount: countWords(page.markdown),
+      readingTime: estimateReadingTime(page.markdown)
+    }))
+  };
+  return JSON.stringify(jsonResult, null, 2);
+}
+function countWords(markdown) {
+  const plainText = markdown.replace(/#{1,6}\s+/g, "").replace(/\*\*(.*?)\*\*/g, "$1").replace(/\*(.*?)\*/g, "$1").replace(/`(.*?)`/g, "$1").replace(/```[\s\S]*?```/g, "").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replace(/^\s*[-*+]\s+/gm, "").replace(/^\s*\d+\.\s+/gm, "").replace(/^\s*>\s+/gm, "").replace(/\n{3,}/g, "\n\n").trim();
+  return plainText.split(/\s+/).filter((word) => word.length > 0).length;
+}
+function estimateReadingTime(markdown) {
+  const wordCount = countWords(markdown);
+  return Math.ceil(wordCount / 200);
+}
+
+// src/formatters/text.ts
+import { parseHTML } from "linkedom";
+function formatToText(pages, baseUrl, scrapedAt, duration, website, includeMetadata = true) {
+  const sections = [];
+  if (includeMetadata) {
+    sections.push(createTextHeader(baseUrl, scrapedAt, duration, website, pages.length));
+  }
+  sections.push(...pages.map((page, index) => createTextPage(page, index + 1, pages.length > 1)));
+  return sections.join("\n\n");
+}
+function createTextHeader(baseUrl, scrapedAt, duration, website, totalPages) {
+  const title = website.title || extractDomainFromUrl3(baseUrl);
+  const lines = [];
+  lines.push(`=== ${title} ===`);
+  lines.push("");
+  lines.push(`URL: ${baseUrl}`);
+  lines.push(`Scraped: ${new Date(scrapedAt).toLocaleString()}`);
+  lines.push(`Duration: ${duration}ms`);
+  lines.push(`Pages: ${totalPages}`);
+  if (website.description) {
+    lines.push(`Description: ${website.description}`);
+  }
+  if (website.author) {
+    lines.push(`Author: ${website.author}`);
+  }
+  if (website.language) {
+    lines.push(`Language: ${website.language}`);
+  }
+  return lines.join("\n");
+}
+function createTextPage(page, pageNumber, showSeparator) {
+  const lines = [];
+  if (showSeparator) {
+    lines.push("\u2500".repeat(60));
+    lines.push(`Page ${pageNumber}: ${page.title || "Untitled"}`);
+    lines.push(`URL: ${page.url}`);
+    lines.push("\u2500".repeat(60));
+  }
+  const plainText = htmlToPlainText(page.html);
+  lines.push(plainText);
+  return lines.join("\n");
+}
+function htmlToPlainText(html) {
+  const { document } = parseHTML(html);
+  const elementsToRemove = ["script", "style", "noscript", "svg", "canvas", "template"];
+  elementsToRemove.forEach((tag) => {
+    document.querySelectorAll(tag).forEach((el) => el.remove());
+  });
+  let text = document.body?.textContent || document.documentElement?.textContent || "";
+  text = text.replace(/[ \t]+/g, " ");
+  text = text.replace(/\n[ \t]+/g, "\n");
+  text = text.replace(/[ \t]+\n/g, "\n");
+  text = text.replace(/\n{3,}/g, "\n\n");
+  text = text.trim();
+  return text;
+}
+function extractDomainFromUrl3(url) {
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return "Unknown";
+  }
+}
+
+// src/utils/content-cleaner.ts
+import { parseHTML as parseHTML2 } from "linkedom";
+var ALWAYS_REMOVE_SELECTORS = [
+  // Navigation and menus
+  "nav",
+  "header nav",
+  "footer nav",
+  ".nav",
+  ".navigation",
+  ".menu",
+  ".navbar",
+  ".sidebar",
+  ".aside",
+  // Header and footer elements
+  "header",
+  "footer",
+  ".site-header",
+  ".page-header",
+  ".site-footer",
+  ".page-footer",
+  // Social media and sharing
+  ".social",
+  ".share",
+  ".sharing",
+  ".twitter",
+  ".facebook",
+  ".linkedin",
+  ".instagram",
+  // Comments and discussions
+  ".comments",
+  ".comment",
+  ".discussion",
+  ".disqus",
+  // Forms and interactive elements
+  "form",
+  "input",
+  "button:not([type='submit'])",
+  "select",
+  "textarea",
+  // Scripts and styles
+  "script",
+  "style",
+  "noscript",
+  // Hidden elements
+  "[hidden]",
+  "[style*='display: none']",
+  "[style*='display:none']",
+  // Common utility classes
+  ".cookie",
+  ".cookie-banner",
+  ".popup",
+  ".modal",
+  ".overlay",
+  ".notification",
+  // Breadcrumbs
+  ".breadcrumb",
+  ".breadcrumbs",
+  ".breadcrumb-trail"
+];
+var AD_SELECTORS = [
+  // Ads and promotions
+  ".ad",
+  ".ads",
+  ".advertisement",
+  ".promotion",
+  ".sponsored",
+  "[class*='ad-']",
+  "[id*='ad-']",
+  "[class*='advert']",
+  "[id*='advert']",
+  "[class*='banner']",
+  "[id*='banner']",
+  ".google-ad",
+  ".adsense",
+  "[data-ad]",
+  "[data-ads]",
+  "ins.adsbygoogle",
+  // Tracking
+  "[class*='tracking']",
+  "[id*='tracking']",
+  "[class*='analytics']",
+  "[id*='analytics']"
+];
+function cleanHtml(html, baseUrl, options = {}) {
+  const { removeAds = true, removeBase64Images = true } = options;
+  const { document } = parseHTML2(html);
+  for (const selector of ALWAYS_REMOVE_SELECTORS) {
+    try {
+      document.querySelectorAll(selector).forEach((el) => el.remove());
+    } catch {
+    }
+  }
+  if (removeAds) {
+    for (const selector of AD_SELECTORS) {
+      try {
+        document.querySelectorAll(selector).forEach((el) => el.remove());
+      } catch {
+      }
+    }
+  }
+  if (removeBase64Images) {
+    removeBase64ImagesFromDocument(document);
+  }
+  const walker = document.createTreeWalker(
+    document,
+    128
+    /* NodeFilter.SHOW_COMMENT */
+  );
+  const comments = [];
+  while (walker.nextNode()) {
+    comments.push(walker.currentNode);
+  }
+  comments.forEach((comment) => comment.parentNode?.removeChild(comment));
+  convertRelativeUrls(document, baseUrl);
+  return document.documentElement?.outerHTML || html;
+}
+function removeBase64ImagesFromDocument(document) {
+  document.querySelectorAll("img[src^='data:']").forEach((el) => {
+    el.remove();
+  });
+  document.querySelectorAll("[style*='data:image']").forEach((el) => {
+    const style = el.getAttribute("style");
+    if (style) {
+      const cleanedStyle = style.replace(/background(-image)?:\s*url\([^)]*data:image[^)]*\)[^;]*;?/gi, "");
+      if (cleanedStyle.trim()) {
+        el.setAttribute("style", cleanedStyle);
+      } else {
+        el.removeAttribute("style");
+      }
+    }
+  });
+  document.querySelectorAll("source[src^='data:'], source[srcset*='data:']").forEach((el) => {
+    el.remove();
+  });
+}
+function convertRelativeUrls(document, baseUrl) {
+  document.querySelectorAll("[src]").forEach((el) => {
+    const src = el.getAttribute("src");
+    if (src && !src.startsWith("http") && !src.startsWith("//") && !src.startsWith("data:")) {
+      try {
+        el.setAttribute("src", new URL(src, baseUrl).toString());
+      } catch {
+      }
+    }
+  });
+  document.querySelectorAll("[href]").forEach((el) => {
+    const href = el.getAttribute("href");
+    if (href && !href.startsWith("http") && !href.startsWith("//") && !href.startsWith("#") && !href.startsWith("mailto:") && !href.startsWith("tel:") && !href.startsWith("javascript:")) {
+      try {
+        el.setAttribute("href", new URL(href, baseUrl).toString());
+      } catch {
+      }
+    }
+  });
+}
+function cleanContent(html, baseUrl, options = {}) {
+  return cleanHtml(html, baseUrl, options);
+}
+
+// src/utils/metadata-extractor.ts
+import { parseHTML as parseHTML3 } from "linkedom";
+
+// src/utils/url-helpers.ts
+import { URL as URL2 } from "url";
+import RE2 from "re2";
+function resolveUrl(relative, base) {
+  try {
+    return new URL2(relative, base).toString();
+  } catch {
+    return relative;
+  }
+}
+function isValidUrl(string) {
+  try {
+    new URL2(string);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function normalizeUrl(url, baseUrl) {
+  try {
+    let parsedUrl;
+    if (url.startsWith("http://") || url.startsWith("https://")) {
+      parsedUrl = new URL2(url);
+    } else if (baseUrl) {
+      parsedUrl = new URL2(url, baseUrl);
+    } else {
+      throw new Error("Relative URL requires base URL");
+    }
+    parsedUrl.hash = "";
+    return parsedUrl.toString();
+  } catch {
+    throw new Error(`Invalid URL: ${url}`);
+  }
+}
+function extractBaseDomain(url) {
+  try {
+    const parsedUrl = new URL2(url);
+    return parsedUrl.hostname;
+  } catch {
+    throw new Error(`Invalid URL for domain extraction: ${url}`);
+  }
+}
+function getRootDomain(hostname) {
+  const parts = hostname.split(".");
+  if (parts.length <= 2) {
+    return hostname;
+  }
+  const twoPartTLDs = ["co.uk", "com.au", "co.nz", "com.br", "co.jp", "co.kr", "com.mx", "org.uk"];
+  const lastTwo = parts.slice(-2).join(".");
+  if (twoPartTLDs.includes(lastTwo)) {
+    return parts.slice(-3).join(".");
+  }
+  return parts.slice(-2).join(".");
+}
+function isSameDomain(url, baseUrl) {
+  try {
+    const urlDomain = extractBaseDomain(url);
+    const baseDomain = extractBaseDomain(baseUrl);
+    if (urlDomain === baseDomain) {
+      return true;
+    }
+    const urlRoot = getRootDomain(urlDomain);
+    const baseRoot = getRootDomain(baseDomain);
+    return urlRoot === baseRoot;
+  } catch {
+    return false;
+  }
+}
+function getUrlKey(url) {
+  try {
+    const parsedUrl = new URL2(url);
+    parsedUrl.search = "";
+    return parsedUrl.toString().toLowerCase();
+  } catch {
+    return url.toLowerCase();
+  }
+}
+function validateUrls(urls) {
+  const validUrls = [];
+  const errors = [];
+  if (!urls || urls.length === 0) {
+    return {
+      isValid: false,
+      validUrls: [],
+      errors: [{ url: "", error: "At least one URL is required" }]
+    };
+  }
+  for (const url of urls) {
+    if (!url || typeof url !== "string") {
+      errors.push({
+        url: String(url),
+        error: "URL must be a non-empty string"
+      });
+      continue;
+    }
+    const trimmedUrl = url.trim();
+    if (trimmedUrl === "") {
+      errors.push({ url: String(url), error: "URL cannot be empty" });
+      continue;
+    }
+    if (!isValidUrl(trimmedUrl)) {
+      errors.push({ url: trimmedUrl, error: "Invalid URL format" });
+      continue;
+    }
+    if (!trimmedUrl.startsWith("http://") && !trimmedUrl.startsWith("https://")) {
+      errors.push({
+        url: trimmedUrl,
+        error: "URL must start with http:// or https://"
+      });
+      continue;
+    }
+    validUrls.push(trimmedUrl);
+  }
+  const uniqueValidUrls = Array.from(new Set(validUrls));
+  return {
+    isValid: uniqueValidUrls.length > 0 && errors.length === 0,
+    validUrls: uniqueValidUrls,
+    errors
+  };
+}
+function matchesPatterns(url, patterns) {
+  if (!patterns || patterns.length === 0) {
+    return false;
+  }
+  return patterns.some((pattern) => {
+    try {
+      const regex = new RE2(pattern, "i");
+      return regex.test(url);
+    } catch {
+      return false;
+    }
+  });
+}
+function shouldIncludeUrl(url, includePatterns, excludePatterns) {
+  if (includePatterns && includePatterns.length > 0) {
+    if (!matchesPatterns(url, includePatterns)) {
+      return false;
+    }
+  }
+  if (excludePatterns && excludePatterns.length > 0) {
+    if (matchesPatterns(url, excludePatterns)) {
+      return false;
+    }
+  }
+  return true;
+}
+function isContentUrl(url) {
+  const lowerUrl = url.toLowerCase();
+  const nonContentPatterns = [
+    // Legal and policy pages
+    /\/(privacy|terms|tos|legal|cookie|gdpr|disclaimer|imprint|impressum)\b/i,
+    /\/(privacy-policy|terms-of-service|terms-of-use|terms-and-conditions)\b/i,
+    /\/(cookie-policy|data-protection|acceptable-use|user-agreement)\b/i,
+    /\/(refund|cancellation|shipping|return)-?(policy)?\b/i,
+    // Contact and support pages (usually not main content)
+    /\/(contact|support|help|faq|feedback)\/?$/i,
+    // About pages that are typically boilerplate
+    /\/(about-us|careers|jobs|press|investors|team)\/?$/i,
+    // Authentication and admin areas
+    /\/(admin|login|auth|account|dashboard|profile|settings)\//i,
+    // E-commerce utility pages
+    /\/(cart|checkout|payment|subscription|wishlist)\//i,
+    // File downloads and assets
+    /\/(uploads|assets|files|static|media|resources)\//i,
+    // API endpoints
+    /\/(api|graphql|rest|webhook)\//i
+  ];
+  if (nonContentPatterns.some((pattern) => pattern.test(lowerUrl))) {
+    return false;
+  }
+  const skipExtensions = [".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".exe"];
+  if (skipExtensions.some((ext) => lowerUrl.endsWith(ext))) {
+    return false;
+  }
+  return true;
+}
+function shouldCrawlUrl(url, baseUrl, maxDepth, currentDepth, visited) {
+  if (currentDepth > maxDepth) {
+    return false;
+  }
+  const urlKey = getUrlKey(url);
+  if (visited.has(urlKey)) {
+    return false;
+  }
+  if (!isSameDomain(url, baseUrl)) {
+    return false;
+  }
+  const lowerUrl = url.toLowerCase();
+  const skipExtensions = [
+    ".pdf",
+    ".doc",
+    ".docx",
+    ".xls",
+    ".xlsx",
+    ".ppt",
+    ".pptx",
+    ".zip",
+    ".rar",
+    ".tar",
+    ".gz",
+    ".exe",
+    ".dmg",
+    ".pkg",
+    ".deb",
+    ".rpm",
+    ".apk",
+    ".ipa",
+    // Image files
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".gif",
+    ".bmp",
+    ".svg",
+    ".webp",
+    ".ico",
+    ".favicon",
+    // Video files
+    ".mp4",
+    ".avi",
+    ".mov",
+    ".wmv",
+    ".flv",
+    ".webm",
+    // Audio files
+    ".mp3",
+    ".wav",
+    ".ogg",
+    ".m4a",
+    ".aac",
+    // Font files
+    ".woff",
+    ".woff2",
+    ".ttf",
+    ".otf",
+    ".eot",
+    // Style and script files
+    ".css",
+    ".js",
+    ".mjs",
+    ".ts",
+    ".jsx",
+    ".tsx",
+    // Data and config files
+    ".json",
+    ".xml",
+    ".txt",
+    ".md",
+    ".rss",
+    ".atom",
+    ".sitemap",
+    ".robots",
+    ".webmanifest",
+    // Archive files
+    ".zip",
+    ".tar",
+    ".gz",
+    ".bz2",
+    ".7z"
+  ];
+  if (skipExtensions.some((ext) => lowerUrl.includes(ext))) {
+    return false;
+  }
+  const skipPatterns = [
+    // File downloads and assets
+    /\/(uploads|assets|files|static|media|resources)\//i,
+    // Authentication and admin areas
+    /\/(admin|login|auth|account|dashboard|profile|settings)\//i,
+    // API endpoints
+    /\/(api|graphql|rest|ws:|webhook)\//i,
+    // Common tracking and analytics
+    /\/(analytics|tracking|pixel|beacon|ads)\//i,
+    // Development and testing areas
+    /\/(test|dev|staging|beta|demo)\//i,
+    // Common utility and service pages
+    /\/(search|cart|checkout|payment|subscription)\//i,
+    // Social media and external services
+    /\/(facebook|twitter|instagram|youtube|linkedin|github)\//i,
+    // Legal and policy pages
+    /\/(privacy|terms|tos|legal|cookie|gdpr|disclaimer|imprint|impressum)\b/i,
+    /\/(privacy-policy|terms-of-service|terms-of-use|terms-and-conditions)\b/i,
+    /\/(cookie-policy|data-protection|acceptable-use|user-agreement)\b/i,
+    /\/(refund|cancellation|shipping|return)-?(policy)?\b/i,
+    // Contact and support pages (usually not main content)
+    /\/(contact|support|help|faq|feedback)\/?$/i,
+    // About pages that are typically boilerplate
+    /\/(about-us|careers|jobs|press|investors|team)\/?$/i
+  ];
+  if (skipPatterns.some((pattern) => pattern.test(url))) {
+    return false;
+  }
+  if (url.includes("?") && ["download", "file", "attachment", "export", "print", "share", "email"].some(
+    (param) => url.toLowerCase().includes(param)
+  )) {
+    return false;
+  }
+  if (url.split("/").filter(Boolean).length < 2 && url.split("?")[0].split("/").length <= 2) {
+    return false;
+  }
+  return true;
+}
+
+// src/utils/metadata-extractor.ts
+function extractMetadata(html, baseUrl) {
+  return extractWebsiteMetadata(html, baseUrl);
+}
+function extractWebsiteMetadata(html, baseUrl) {
+  const { document } = parseHTML3(html);
+  const metadata = {
+    title: null,
+    description: null,
+    author: null,
+    language: null,
+    charset: null,
+    favicon: null,
+    canonical: null,
+    image: null,
+    keywords: null,
+    robots: null,
+    themeColor: null,
+    openGraph: null,
+    twitter: null
+  };
+  metadata.title = extractTitle(document);
+  metadata.description = extractMetaContent(document, "description");
+  metadata.author = extractMetaContent(document, "author");
+  metadata.language = extractLanguage(document);
+  metadata.charset = extractCharset(document);
+  metadata.favicon = extractFavicon(document, baseUrl);
+  metadata.canonical = extractCanonical(document, baseUrl);
+  metadata.image = extractMetaContent(document, "og:image") || extractMetaContent(document, "twitter:image");
+  metadata.keywords = extractKeywords(document);
+  metadata.robots = extractMetaContent(document, "robots");
+  metadata.themeColor = extractMetaContent(document, "theme-color");
+  metadata.openGraph = extractOpenGraph(document);
+  metadata.twitter = extractTwitterCard(document);
+  return metadata;
+}
+function extractTitle(document) {
+  const titleElement = document.querySelector("title");
+  if (titleElement?.textContent) {
+    return titleElement.textContent.trim();
+  }
+  return extractMetaContent(document, "og:title");
+}
+function extractMetaContent(document, name) {
+  const byName = document.querySelector(`meta[name="${name}"]`);
+  if (byName) {
+    const content = byName.getAttribute("content");
+    if (content) return content.trim();
+  }
+  const byProperty = document.querySelector(`meta[property="${name}"]`);
+  if (byProperty) {
+    const content = byProperty.getAttribute("content");
+    if (content) return content.trim();
+  }
+  return null;
+}
+function extractLanguage(document) {
+  const lang = document.documentElement?.getAttribute("lang");
+  return lang?.trim() || null;
+}
+function extractCharset(document) {
+  const charsetMeta = document.querySelector("meta[charset]");
+  if (charsetMeta) {
+    const charset = charsetMeta.getAttribute("charset");
+    if (charset) return charset.trim();
+  }
+  const httpEquivMeta = document.querySelector('meta[http-equiv="Content-Type"]');
+  if (httpEquivMeta) {
+    const content = httpEquivMeta.getAttribute("content");
+    if (content) {
+      const charsetMatch = content.match(/charset=([^\s;]+)/i);
+      if (charsetMatch) return charsetMatch[1].trim();
+    }
+  }
+  return null;
+}
+function extractFavicon(document, baseUrl) {
+  const iconSelectors = [
+    'link[rel="icon"]',
+    'link[rel="shortcut icon"]',
+    'link[rel="apple-touch-icon"]',
+    'link[rel*="icon"]'
+  ];
+  for (const selector of iconSelectors) {
+    const iconLink = document.querySelector(selector);
+    if (iconLink) {
+      const href = iconLink.getAttribute("href");
+      if (href) {
+        return normalizeUrl(href, baseUrl);
+      }
+    }
+  }
+  try {
+    return normalizeUrl("/favicon.ico", baseUrl);
+  } catch {
+    return null;
+  }
+}
+function extractCanonical(document, baseUrl) {
+  const canonicalLink = document.querySelector('link[rel="canonical"]');
+  if (canonicalLink) {
+    const href = canonicalLink.getAttribute("href");
+    if (href) {
+      return normalizeUrl(href, baseUrl);
+    }
+  }
+  return null;
+}
+function extractKeywords(document) {
+  const keywordsContent = extractMetaContent(document, "keywords");
+  if (!keywordsContent) {
+    return null;
+  }
+  return keywordsContent.split(",").map((keyword) => keyword.trim()).filter((keyword) => keyword.length > 0);
+}
+function extractOpenGraph(document) {
+  const openGraph = {
+    title: null,
+    description: null,
+    type: null,
+    url: null,
+    image: null,
+    siteName: null,
+    locale: null
+  };
+  openGraph.title = extractMetaContent(document, "og:title");
+  openGraph.description = extractMetaContent(document, "og:description");
+  openGraph.type = extractMetaContent(document, "og:type");
+  openGraph.url = extractMetaContent(document, "og:url");
+  openGraph.image = extractMetaContent(document, "og:image");
+  openGraph.siteName = extractMetaContent(document, "og:site_name");
+  openGraph.locale = extractMetaContent(document, "og:locale");
+  if (Object.values(openGraph).every((value) => !value)) {
+    return null;
+  }
+  return openGraph;
+}
+function extractTwitterCard(document) {
+  const twitter = {
+    card: null,
+    site: null,
+    creator: null,
+    title: null,
+    description: null,
+    image: null
+  };
+  twitter.card = extractMetaContent(document, "twitter:card");
+  twitter.site = extractMetaContent(document, "twitter:site");
+  twitter.creator = extractMetaContent(document, "twitter:creator");
+  twitter.title = extractMetaContent(document, "twitter:title");
+  twitter.description = extractMetaContent(document, "twitter:description");
+  twitter.image = extractMetaContent(document, "twitter:image");
+  if (Object.values(twitter).every((value) => !value)) {
+    return null;
+  }
+  return twitter;
+}
+
+// src/utils/logger.ts
+import pino from "pino";
+function createLogger(name = "reader", level = process.env.LOG_LEVEL || "info") {
+  return pino({
+    name,
+    level,
+    transport: process.env.NODE_ENV !== "production" ? {
+      target: "pino-pretty",
+      options: {
+        colorize: true,
+        translateTime: "SYS:standard",
+        ignore: "pid,hostname"
+      }
+    } : void 0
+  });
+}
+var logger = createLogger();
+
+// src/utils/robots-parser.ts
+function parseRobotsTxt(content, userAgent = "*") {
+  const rules = {
+    disallowedPaths: [],
+    allowedPaths: [],
+    crawlDelay: null
+  };
+  const lines = content.split("\n").map((line) => line.trim());
+  let currentUserAgent = "";
+  let matchesUserAgent = false;
+  for (const line of lines) {
+    if (!line || line.startsWith("#")) {
+      continue;
+    }
+    const colonIndex = line.indexOf(":");
+    if (colonIndex === -1) {
+      continue;
+    }
+    const directive = line.substring(0, colonIndex).trim().toLowerCase();
+    const value = line.substring(colonIndex + 1).trim();
+    if (directive === "user-agent") {
+      currentUserAgent = value.toLowerCase();
+      matchesUserAgent = currentUserAgent === "*" || currentUserAgent === userAgent.toLowerCase();
+    } else if (matchesUserAgent) {
+      if (directive === "disallow" && value) {
+        rules.disallowedPaths.push(value);
+      } else if (directive === "allow" && value) {
+        rules.allowedPaths.push(value);
+      } else if (directive === "crawl-delay") {
+        const delay = parseFloat(value);
+        if (!isNaN(delay)) {
+          rules.crawlDelay = delay * 1e3;
+        }
+      }
+    }
+  }
+  return rules;
+}
+function isPathAllowed(path, rules) {
+  const normalizedPath = path.startsWith("/") ? path : "/" + path;
+  for (const allowedPath of rules.allowedPaths) {
+    if (pathMatches(normalizedPath, allowedPath)) {
+      return true;
+    }
+  }
+  for (const disallowedPath of rules.disallowedPaths) {
+    if (pathMatches(normalizedPath, disallowedPath)) {
+      return false;
+    }
+  }
+  return true;
+}
+function pathMatches(path, pattern) {
+  if (!pattern) {
+    return false;
+  }
+  let regexPattern = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+  if (regexPattern.endsWith("\\$")) {
+    regexPattern = regexPattern.slice(0, -2) + "$";
+  } else {
+    regexPattern = "^" + regexPattern;
+  }
+  try {
+    const regex = new RegExp(regexPattern);
+    return regex.test(path);
+  } catch {
+    return path.startsWith(pattern);
+  }
+}
+async function fetchRobotsTxt(baseUrl) {
+  try {
+    const url = new URL("/robots.txt", baseUrl);
+    const response = await fetch(url.toString(), {
+      headers: {
+        "User-Agent": "ReaderEngine/1.0"
+      }
+    });
+    if (!response.ok) {
+      return null;
+    }
+    const content = await response.text();
+    return parseRobotsTxt(content, "ReaderEngine");
+  } catch {
+    return null;
+  }
+}
+function isUrlAllowed(url, rules) {
+  if (!rules) {
+    return true;
+  }
+  try {
+    const parsedUrl = new URL(url);
+    return isPathAllowed(parsedUrl.pathname + parsedUrl.search, rules);
+  } catch {
+    return true;
+  }
+}
+
+// src/types.ts
+var DEFAULT_OPTIONS = {
+  urls: [],
+  formats: ["markdown"],
+  includeMetadata: true,
+  timeoutMs: 3e4,
+  includePatterns: [],
+  excludePatterns: [],
+  // Content cleaning defaults
+  removeAds: true,
+  removeBase64Images: true,
+  skipTLSVerification: true,
+  // Batch defaults
+  batchConcurrency: 1,
+  batchTimeoutMs: 3e5,
+  maxRetries: 2,
+  onProgress: () => {
+  },
+  // Default no-op progress callback
+  // Hero-specific defaults
+  verbose: false,
+  showChrome: false
+};
+function isValidFormat(format) {
+  return format === "markdown" || format === "html" || format === "json" || format === "text";
+}
+function shouldCrawlUrl2(url, baseDomain) {
+  return url.hostname === baseDomain || url.hostname.endsWith(`.${baseDomain}`);
+}
+
+// src/scraper.ts
+var Scraper = class {
+  options;
+  pool;
+  logger = createLogger("scraper");
+  robotsCache = /* @__PURE__ */ new Map();
+  constructor(options) {
+    this.options = {
+      ...DEFAULT_OPTIONS,
+      ...options
+    };
+    if (!options.pool) {
+      throw new Error("Browser pool must be provided. Use ReaderClient for automatic pool management.");
+    }
+    this.pool = options.pool;
+  }
+  /**
+   * Get robots.txt rules for a URL, cached per domain
+   */
+  async getRobotsRules(url) {
+    const origin = new URL(url).origin;
+    if (!this.robotsCache.has(origin)) {
+      const rules = await fetchRobotsTxt(origin);
+      this.robotsCache.set(origin, rules);
+    }
+    return this.robotsCache.get(origin) ?? null;
+  }
+  /**
+   * Scrape all URLs
+   *
+   * @returns Scrape result with pages and metadata
+   */
+  async scrape() {
+    const startTime = Date.now();
+    const results = await this.scrapeWithConcurrency();
+    return this.buildScrapeResult(results, startTime);
+  }
+  /**
+   * Scrape URLs with concurrency control
+   */
+  async scrapeWithConcurrency() {
+    const limit = pLimit(this.options.batchConcurrency || 1);
+    const tasks = this.options.urls.map(
+      (url, index) => limit(() => this.scrapeSingleUrlWithRetry(url, index))
+    );
+    const batchPromise = Promise.all(tasks);
+    if (this.options.batchTimeoutMs && this.options.batchTimeoutMs > 0) {
+      const timeoutPromise = new Promise((_, reject) => {
+        setTimeout(() => {
+          reject(new Error(`Batch operation timed out after ${this.options.batchTimeoutMs}ms`));
+        }, this.options.batchTimeoutMs);
+      });
+      return Promise.race([batchPromise, timeoutPromise]);
+    }
+    return batchPromise;
+  }
+  /**
+   * Scrape a single URL with retry logic
+   */
+  async scrapeSingleUrlWithRetry(url, index) {
+    const maxRetries = this.options.maxRetries || 2;
+    let lastError;
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      try {
+        const result = await this.scrapeSingleUrl(url, index);
+        if (result) {
+          return { result };
+        }
+        lastError = `Failed to scrape ${url}: No content returned`;
+      } catch (error) {
+        lastError = error.message;
+        if (attempt < maxRetries) {
+          const delay = Math.pow(2, attempt) * 1e3;
+          this.logger.warn(`Retry ${attempt + 1}/${maxRetries} for ${url} in ${delay}ms`);
+          await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+      }
+    }
+    this.logger.error(`Failed to scrape ${url} after ${maxRetries + 1} attempts: ${lastError}`);
+    return { result: null, error: lastError };
+  }
+  /**
+   * Wait for the final page to load after any Cloudflare redirects
+   * Cloudflare often does silent redirects even when bypassed, we need to ensure
+   * we're on the actual content page before scraping.
+   */
+  async waitForFinalPage(hero, originalUrl, verbose) {
+    const maxWaitMs = 15e3;
+    const startTime = Date.now();
+    const log = (msg) => verbose && this.logger.info(msg);
+    try {
+      await hero.waitForLoad("AllContentLoaded", { timeoutMs: maxWaitMs });
+    } catch {
+    }
+    let currentUrl = await hero.url;
+    const normalizeUrl2 = (url) => url.replace(/\/+$/, "");
+    const urlChanged = normalizeUrl2(currentUrl) !== normalizeUrl2(originalUrl);
+    if (urlChanged || currentUrl.includes("__cf_chl")) {
+      log(`Cloudflare redirect detected: ${originalUrl} \u2192 ${currentUrl}`);
+      let lastUrl = currentUrl;
+      let stableCount = 0;
+      while (Date.now() - startTime < maxWaitMs) {
+        await new Promise((resolve) => setTimeout(resolve, 500));
+        try {
+          currentUrl = await hero.url;
+          if (currentUrl === lastUrl) {
+            stableCount++;
+            if (stableCount >= 2) {
+              break;
+            }
+          } else {
+            stableCount = 0;
+            lastUrl = currentUrl;
+            log(`URL changed to: ${currentUrl}`);
+          }
+        } catch {
+        }
+      }
+      try {
+        await hero.waitForLoad("AllContentLoaded", { timeoutMs: 1e4 });
+      } catch {
+      }
+    }
+    await hero.waitForPaintingStable();
+    await new Promise((resolve) => setTimeout(resolve, 2e3));
+  }
+  /**
+   * Scrape a single URL
+   */
+  async scrapeSingleUrl(url, index) {
+    const startTime = Date.now();
+    const robotsRules = await this.getRobotsRules(url);
+    if (!isUrlAllowed(url, robotsRules)) {
+      throw new Error(`URL blocked by robots.txt: ${url}`);
+    }
+    try {
+      return await this.pool.withBrowser(async (hero) => {
+        await hero.goto(url, { timeoutMs: this.options.timeoutMs });
+        try {
+          await hero.waitForLoad("DomContentLoaded", { timeoutMs: this.options.timeoutMs });
+        } catch {
+        }
+        await hero.waitForPaintingStable();
+        let hadChallenge = false;
+        let challengeType = "none";
+        let waitTimeMs = 0;
+        const initialUrl = await hero.url;
+        const detection = await detectChallenge(hero);
+        if (detection.isChallenge) {
+          hadChallenge = true;
+          challengeType = detection.type;
+          if (this.options.verbose) {
+            this.logger.info(`Challenge detected on ${url}: ${detection.type}`);
+          }
+          const result2 = await waitForChallengeResolution(hero, {
+            maxWaitMs: 45e3,
+            pollIntervalMs: 500,
+            verbose: this.options.verbose,
+            initialUrl
+          });
+          waitTimeMs = result2.waitedMs;
+          if (!result2.resolved) {
+            throw new Error(`Challenge not resolved: ${detection.type}`);
+          }
+          if (this.options.verbose) {
+            this.logger.info(`Challenge resolved via ${result2.method} in ${waitTimeMs}ms`);
+          }
+        }
+        await this.waitForFinalPage(hero, url, this.options.verbose);
+        if (this.options.waitForSelector) {
+          try {
+            await hero.waitForElement(hero.document.querySelector(this.options.waitForSelector), {
+              timeoutMs: this.options.timeoutMs
+            });
+          } catch (error) {
+            this.logger.warn(`Selector not found: ${this.options.waitForSelector}`);
+          }
+        }
+        const pageTitle = await hero.document.title;
+        const html = await hero.document.documentElement.outerHTML;
+        const cleanedHtml = cleanContent(html, url, {
+          removeAds: this.options.removeAds,
+          removeBase64Images: this.options.removeBase64Images
+        });
+        const websiteMetadata = extractMetadata(cleanedHtml, url);
+        const duration = Date.now() - startTime;
+        const scrapedAt = (/* @__PURE__ */ new Date()).toISOString();
+        const page = {
+          url,
+          title: pageTitle,
+          markdown: "",
+          // Will be set by formatter
+          html: cleanedHtml,
+          fetchedAt: scrapedAt,
+          depth: 0,
+          hadChallenge,
+          challengeType,
+          waitTimeMs
+        };
+        const markdown = this.options.formats.includes("markdown") ? formatToMarkdown(
+          [page],
+          url,
+          scrapedAt,
+          duration,
+          websiteMetadata,
+          this.options.includeMetadata
+        ) : void 0;
+        const htmlOutput = this.options.formats.includes("html") ? formatToHTML([page], url, scrapedAt, duration, websiteMetadata) : void 0;
+        const json = this.options.formats.includes("json") ? formatToJson([page], url, scrapedAt, duration, websiteMetadata) : void 0;
+        const text = this.options.formats.includes("text") ? formatToText(
+          [page],
+          url,
+          scrapedAt,
+          duration,
+          websiteMetadata,
+          this.options.includeMetadata
+        ) : void 0;
+        if (this.options.onProgress) {
+          this.options.onProgress({
+            completed: index + 1,
+            total: this.options.urls.length,
+            currentUrl: url
+          });
+        }
+        let proxyMetadata;
+        if (this.options.proxy) {
+          const proxy = this.options.proxy;
+          if (proxy.url) {
+            try {
+              const proxyUrl = new URL(proxy.url);
+              proxyMetadata = {
+                host: proxyUrl.hostname,
+                port: parseInt(proxyUrl.port, 10) || 80,
+                country: proxy.country
+              };
+            } catch {
+            }
+          } else if (proxy.host && proxy.port) {
+            proxyMetadata = {
+              host: proxy.host,
+              port: proxy.port,
+              country: proxy.country
+            };
+          }
+        }
+        const result = {
+          markdown,
+          html: htmlOutput,
+          json,
+          text,
+          metadata: {
+            baseUrl: url,
+            totalPages: 1,
+            scrapedAt: (/* @__PURE__ */ new Date()).toISOString(),
+            duration,
+            website: websiteMetadata,
+            proxy: proxyMetadata
+          }
+        };
+        return result;
+      });
+    } catch (error) {
+      this.logger.error(`Failed to scrape ${url}: ${error.message}`);
+      if (this.options.onProgress) {
+        this.options.onProgress({
+          completed: index + 1,
+          total: this.options.urls.length,
+          currentUrl: url
+        });
+      }
+      return null;
+    }
+  }
+  /**
+   * Build final scrape result
+   */
+  buildScrapeResult(results, startTime) {
+    const successful = results.filter((r) => r.result !== null).map((r) => r.result);
+    const errors = [];
+    results.forEach((r, index) => {
+      if (r.result === null && r.error) {
+        errors.push({ url: this.options.urls[index], error: r.error });
+      }
+    });
+    const batchMetadata = {
+      totalUrls: this.options.urls.length,
+      successfulUrls: successful.length,
+      failedUrls: results.filter((r) => r.result === null).length,
+      scrapedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      totalDuration: Date.now() - startTime,
+      errors
+    };
+    return {
+      data: successful,
+      batchMetadata
+    };
+  }
+};
+async function scrape(options) {
+  const scraper = new Scraper(options);
+  return scraper.scrape();
+}
+
+// src/crawler.ts
+import { parseHTML as parseHTML4 } from "linkedom";
+
+// src/utils/rate-limiter.ts
+import pLimit2 from "p-limit";
+async function rateLimit(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// src/crawler.ts
+var Crawler = class {
+  options;
+  visited = /* @__PURE__ */ new Set();
+  queue = [];
+  urls = [];
+  pool;
+  logger = createLogger("crawler");
+  robotsRules = null;
+  constructor(options) {
+    if (!options.pool) {
+      throw new Error("Browser pool must be provided. Use ReaderClient for automatic pool management.");
+    }
+    this.pool = options.pool;
+    this.options = {
+      url: options.url,
+      depth: options.depth || 1,
+      maxPages: options.maxPages || 20,
+      scrape: options.scrape || false,
+      delayMs: options.delayMs || 1e3,
+      timeoutMs: options.timeoutMs,
+      includePatterns: options.includePatterns,
+      excludePatterns: options.excludePatterns,
+      formats: options.formats || ["markdown", "html"],
+      scrapeConcurrency: options.scrapeConcurrency || 2,
+      proxy: options.proxy,
+      userAgent: options.userAgent,
+      verbose: options.verbose || false,
+      showChrome: options.showChrome || false,
+      connectionToCore: options.connectionToCore,
+      // Content cleaning options
+      removeAds: options.removeAds,
+      removeBase64Images: options.removeBase64Images
+    };
+  }
+  /**
+   * Start crawling
+   */
+  async crawl() {
+    const startTime = Date.now();
+    this.robotsRules = await fetchRobotsTxt(this.options.url);
+    if (this.robotsRules) {
+      this.logger.info("Loaded robots.txt rules");
+    }
+    if (isUrlAllowed(this.options.url, this.robotsRules)) {
+      this.queue.push({ url: this.options.url, depth: 0 });
+    } else {
+      this.logger.warn(`Seed URL blocked by robots.txt: ${this.options.url}`);
+    }
+    while (this.queue.length > 0 && this.urls.length < this.options.maxPages) {
+      if (this.options.timeoutMs && Date.now() - startTime > this.options.timeoutMs) {
+        this.logger.warn(`Crawl timed out after ${this.options.timeoutMs}ms`);
+        break;
+      }
+      const item = this.queue.shift();
+      const urlKey = getUrlKey(item.url);
+      if (this.visited.has(urlKey)) {
+        continue;
+      }
+      const result = await this.fetchPage(item.url);
+      if (result) {
+        this.urls.push(result.crawlUrl);
+        this.visited.add(urlKey);
+        if (item.depth < this.options.depth) {
+          const links = this.extractLinks(result.html, item.url, item.depth + 1);
+          this.queue.push(...links);
+        }
+      }
+      const delay = this.robotsRules?.crawlDelay || this.options.delayMs;
+      await rateLimit(delay);
+    }
+    const metadata = {
+      totalUrls: this.urls.length,
+      maxDepth: this.options.depth,
+      totalDuration: Date.now() - startTime,
+      seedUrl: this.options.url
+    };
+    let scraped;
+    if (this.options.scrape) {
+      scraped = await this.scrapeDiscoveredUrls();
+    }
+    return {
+      urls: this.urls,
+      scraped,
+      metadata
+    };
+  }
+  /**
+   * Fetch a single page and extract basic info
+   */
+  async fetchPage(url) {
+    try {
+      return await this.pool.withBrowser(async (hero) => {
+        await hero.goto(url, { timeoutMs: 3e4 });
+        await hero.waitForPaintingStable();
+        const initialUrl = await hero.url;
+        const detection = await detectChallenge(hero);
+        if (detection.isChallenge) {
+          if (this.options.verbose) {
+            this.logger.info(`Challenge detected on ${url}`);
+          }
+          const result = await waitForChallengeResolution(hero, {
+            maxWaitMs: 45e3,
+            pollIntervalMs: 500,
+            verbose: this.options.verbose,
+            initialUrl
+          });
+          if (!result.resolved) {
+            throw new Error(`Challenge not resolved`);
+          }
+        }
+        const title = await hero.document.title;
+        const html = await hero.document.documentElement.outerHTML;
+        let description = null;
+        try {
+          const metaDesc = await hero.document.querySelector('meta[name="description"]');
+          if (metaDesc) {
+            description = await metaDesc.getAttribute("content");
+          }
+        } catch {
+        }
+        return {
+          crawlUrl: {
+            url,
+            title: title || "Untitled",
+            description
+          },
+          html
+        };
+      });
+    } catch (error) {
+      this.logger.error(`Failed to fetch ${url}: ${error.message}`);
+      return null;
+    }
+  }
+  /**
+   * Extract links from HTML content using DOM parsing
+   * Handles all href formats (single quotes, double quotes, unquoted)
+   */
+  extractLinks(html, baseUrl, depth) {
+    const links = [];
+    const { document } = parseHTML4(html);
+    document.querySelectorAll("a[href]").forEach((anchor) => {
+      const href = anchor.getAttribute("href");
+      if (!href) return;
+      const resolved = resolveUrl(href, baseUrl);
+      if (!resolved || !isValidUrl(resolved)) return;
+      if (!isSameDomain(resolved, this.options.url)) return;
+      if (!isContentUrl(resolved)) return;
+      if (!shouldIncludeUrl(resolved, this.options.includePatterns, this.options.excludePatterns)) return;
+      if (!isUrlAllowed(resolved, this.robotsRules)) return;
+      const urlKey = getUrlKey(resolved);
+      if (this.visited.has(urlKey) || this.queue.some((q) => getUrlKey(q.url) === urlKey)) {
+        return;
+      }
+      links.push({ url: resolved, depth });
+    });
+    return links;
+  }
+  /**
+   * Scrape all discovered URLs
+   */
+  async scrapeDiscoveredUrls() {
+    const urls = this.urls.map((u) => u.url);
+    return scrape({
+      urls,
+      formats: this.options.formats,
+      batchConcurrency: this.options.scrapeConcurrency,
+      proxy: this.options.proxy,
+      userAgent: this.options.userAgent,
+      verbose: this.options.verbose,
+      showChrome: this.options.showChrome,
+      pool: this.pool,
+      // Content cleaning options
+      removeAds: this.options.removeAds,
+      removeBase64Images: this.options.removeBase64Images
+    });
+  }
+};
+async function crawl(options) {
+  const crawler = new Crawler(options);
+  return crawler.crawl();
+}
+
+// src/browser/pool.ts
+import Hero from "@ulixee/hero";
+
+// src/proxy/config.ts
+function createProxyUrl(config) {
+  if (config.url) {
+    return config.url;
+  }
+  if (config.type === "residential") {
+    const sessionId = `hero_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+    return `http://customer-${config.username}_session-${sessionId}_country-${config.country || "us"}:${config.password}@${config.host}:${config.port}`;
+  }
+  return `http://${config.username}:${config.password}@${config.host}:${config.port}`;
+}
+function parseProxyUrl(url) {
+  try {
+    const parsed = new URL(url);
+    return {
+      url,
+      username: parsed.username,
+      password: parsed.password,
+      host: parsed.hostname,
+      port: parsed.port ? parseInt(parsed.port, 10) : void 0
+    };
+  } catch (error) {
+    throw new Error(`Invalid proxy URL: ${url}`);
+  }
+}
+
+// src/browser/hero-config.ts
+function createHeroConfig(options = {}) {
+  const config = {
+    // Show or hide Chrome window
+    showChrome: options.showChrome ?? false,
+    // ============================================================================
+    // CRITICAL: TLS fingerprint emulation
+    // ============================================================================
+    // Setting disableMitm to false enables TLS/TCP fingerprint emulation
+    // This is ESSENTIAL for bypassing Cloudflare and other anti-bot systems
+    disableMitm: false,
+    // ============================================================================
+    // Session management
+    // ============================================================================
+    // Use incognito for clean session state
+    disableIncognito: false,
+    // ============================================================================
+    // Docker compatibility
+    // ============================================================================
+    // Required when running in containerized environments
+    noChromeSandbox: true,
+    // ============================================================================
+    // DNS over TLS (mimics Chrome behavior)
+    // ============================================================================
+    // Using Cloudflare's DNS (1.1.1.1) over TLS makes the connection
+    // look more like a real Chrome browser
+    dnsOverTlsProvider: {
+      host: "1.1.1.1",
+      servername: "cloudflare-dns.com"
+    },
+    // ============================================================================
+    // WebRTC IP leak prevention
+    // ============================================================================
+    // Masks the real IP address in WebRTC connections
+    // Uses ipify.org to detect the public IP
+    upstreamProxyIpMask: {
+      ipLookupService: "https://api.ipify.org?format=json"
+    },
+    // ============================================================================
+    // Locale and timezone
+    // ============================================================================
+    locale: "en-US",
+    timezoneId: "America/New_York",
+    // ============================================================================
+    // Viewport (standard desktop size)
+    // ============================================================================
+    viewport: {
+      width: 1920,
+      height: 1080
+    },
+    // ============================================================================
+    // User agent (if provided)
+    // ============================================================================
+    ...options.userAgent && { userAgent: options.userAgent },
+    // ============================================================================
+    // Connection to Core (if provided)
+    // ============================================================================
+    ...options.connectionToCore && { connectionToCore: options.connectionToCore }
+  };
+  if (options.proxy) {
+    config.upstreamProxyUrl = createProxyUrl(options.proxy);
+    config.upstreamProxyUseSystemDns = false;
+  }
+  return config;
+}
+
+// src/browser/pool.ts
+var DEFAULT_POOL_CONFIG = {
+  size: 2,
+  retireAfterPageCount: 100,
+  retireAfterAgeMs: 30 * 60 * 1e3,
+  // 30 minutes
+  recycleCheckInterval: 60 * 1e3,
+  // 1 minute
+  healthCheckInterval: 5 * 60 * 1e3,
+  // 5 minutes
+  maxConsecutiveFailures: 3,
+  maxQueueSize: 100,
+  queueTimeout: 60 * 1e3
+  // 1 minute
+};
+function generateId() {
+  return `browser_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`;
+}
+var BrowserPool = class {
+  instances = [];
+  available = [];
+  inUse = /* @__PURE__ */ new Set();
+  queue = [];
+  config;
+  proxy;
+  recycleTimer;
+  healthTimer;
+  totalRequests = 0;
+  totalRequestDuration = 0;
+  showChrome;
+  connectionToCore;
+  userAgent;
+  verbose;
+  logger = createLogger("pool");
+  constructor(config = {}, proxy, showChrome = false, connectionToCore, userAgent, verbose = false) {
+    this.config = { ...DEFAULT_POOL_CONFIG, ...config };
+    this.proxy = proxy;
+    this.showChrome = showChrome;
+    this.connectionToCore = connectionToCore;
+    this.userAgent = userAgent;
+    this.verbose = verbose;
+  }
+  /**
+   * Initialize the pool by pre-launching browsers
+   */
+  async initialize() {
+    if (this.verbose) {
+      this.logger.info(`Initializing pool with ${this.config.size} browsers...`);
+    }
+    const launchPromises = [];
+    for (let i = 0; i < this.config.size; i++) {
+      launchPromises.push(this.createInstance());
+    }
+    this.instances = await Promise.all(launchPromises);
+    this.available = [...this.instances];
+    this.startRecycling();
+    this.startHealthChecks();
+    if (this.verbose) {
+      this.logger.info(`Pool ready: ${this.instances.length} browsers available`);
+    }
+  }
+  /**
+   * Shutdown the pool and close all browsers
+   */
+  async shutdown() {
+    if (this.verbose) {
+      const stats = this.getStats();
+      this.logger.info(
+        `Shutting down pool: ${stats.totalRequests} total requests processed, ${Math.round(stats.avgRequestDuration)}ms avg duration`
+      );
+    }
+    if (this.recycleTimer) clearInterval(this.recycleTimer);
+    if (this.healthTimer) clearInterval(this.healthTimer);
+    for (const item of this.queue) {
+      item.reject(new Error("Pool shutting down"));
+    }
+    this.queue = [];
+    const closePromises = this.instances.map((instance) => instance.hero.close().catch(() => {
+    }));
+    await Promise.all(closePromises);
+    if (this.connectionToCore) {
+      try {
+        await this.connectionToCore.disconnect();
+      } catch {
+      }
+      this.connectionToCore = void 0;
+    }
+    this.instances = [];
+    this.available = [];
+    this.inUse.clear();
+  }
+  /**
+   * Acquire a browser from the pool
+   */
+  async acquire() {
+    const instance = this.available.shift();
+    if (!instance) {
+      if (this.verbose) {
+        this.logger.info(`No browsers available, queuing request (queue: ${this.queue.length + 1})`);
+      }
+      return this.queueRequest();
+    }
+    instance.status = "busy";
+    instance.lastUsed = Date.now();
+    this.inUse.add(instance);
+    if (this.verbose) {
+      this.logger.info(
+        `Acquired browser ${instance.id} (available: ${this.available.length}, busy: ${this.inUse.size})`
+      );
+    }
+    return instance.hero;
+  }
+  /**
+   * Release a browser back to the pool
+   */
+  release(hero) {
+    const instance = this.instances.find((i) => i.hero === hero);
+    if (!instance) return;
+    instance.status = "idle";
+    instance.requestCount++;
+    this.inUse.delete(instance);
+    if (this.verbose) {
+      this.logger.info(
+        `Released browser ${instance.id} (requests: ${instance.requestCount}, available: ${this.available.length + 1})`
+      );
+    }
+    if (this.shouldRecycle(instance)) {
+      if (this.verbose) {
+        this.logger.info(`Recycling browser ${instance.id} (age or request limit reached)`);
+      }
+      this.recycleInstance(instance).catch(() => {
+      });
+    } else {
+      this.available.push(instance);
+      this.processQueue();
+    }
+  }
+  /**
+   * Execute callback with auto-managed browser
+   */
+  async withBrowser(callback) {
+    const startTime = Date.now();
+    const hero = await this.acquire();
+    try {
+      const result = await callback(hero);
+      this.totalRequests++;
+      this.totalRequestDuration += Date.now() - startTime;
+      return result;
+    } finally {
+      this.release(hero);
+    }
+  }
+  /**
+   * Get pool statistics
+   */
+  getStats() {
+    const recycling = this.instances.filter((i) => i.status === "recycling").length;
+    const unhealthy = this.instances.filter((i) => i.status === "unhealthy").length;
+    return {
+      total: this.instances.length,
+      available: this.available.length,
+      busy: this.inUse.size,
+      recycling,
+      unhealthy,
+      queueLength: this.queue.length,
+      totalRequests: this.totalRequests,
+      avgRequestDuration: this.totalRequests > 0 ? this.totalRequestDuration / this.totalRequests : 0
+    };
+  }
+  /**
+   * Run health check
+   */
+  async healthCheck() {
+    const issues = [];
+    const stats = this.getStats();
+    if (stats.unhealthy > 0) {
+      issues.push(`${stats.unhealthy} unhealthy instances`);
+    }
+    if (stats.queueLength > this.config.maxQueueSize * 0.8) {
+      issues.push(`Queue near capacity: ${stats.queueLength}/${this.config.maxQueueSize}`);
+    }
+    if (stats.available === 0 && stats.queueLength > 0) {
+      issues.push("Pool saturated - all browsers busy with pending requests");
+    }
+    return {
+      healthy: issues.length === 0,
+      issues,
+      stats
+    };
+  }
+  // =========================================================================
+  // Private methods
+  // =========================================================================
+  /**
+   * Create a new browser instance
+   */
+  async createInstance() {
+    const heroConfig = createHeroConfig({
+      proxy: this.proxy,
+      showChrome: this.showChrome,
+      connectionToCore: this.connectionToCore,
+      userAgent: this.userAgent
+    });
+    const hero = new Hero(heroConfig);
+    return {
+      hero,
+      id: generateId(),
+      createdAt: Date.now(),
+      lastUsed: Date.now(),
+      requestCount: 0,
+      status: "idle"
+    };
+  }
+  /**
+   * Check if instance should be recycled
+   */
+  shouldRecycle(instance) {
+    const age = Date.now() - instance.createdAt;
+    return instance.requestCount >= this.config.retireAfterPageCount || age >= this.config.retireAfterAgeMs;
+  }
+  /**
+   * Recycle an instance (close old, create new)
+   */
+  async recycleInstance(instance) {
+    instance.status = "recycling";
+    try {
+      await instance.hero.close().catch(() => {
+      });
+      const newInstance = await this.createInstance();
+      const index = this.instances.indexOf(instance);
+      if (index !== -1) {
+        this.instances[index] = newInstance;
+      }
+      this.available.push(newInstance);
+      if (this.verbose) {
+        this.logger.info(`Recycled browser: ${instance.id} \u2192 ${newInstance.id}`);
+      }
+      this.processQueue();
+    } catch (error) {
+      instance.status = "unhealthy";
+      if (this.verbose) {
+        this.logger.warn(`Failed to recycle browser ${instance.id}`);
+      }
+    }
+  }
+  /**
+   * Queue a request when no browsers available
+   */
+  queueRequest() {
+    return new Promise((resolve, reject) => {
+      if (this.queue.length >= this.config.maxQueueSize) {
+        reject(new Error("Queue full"));
+        return;
+      }
+      const item = {
+        resolve,
+        reject,
+        queuedAt: Date.now()
+      };
+      this.queue.push(item);
+      setTimeout(() => {
+        const index = this.queue.indexOf(item);
+        if (index !== -1) {
+          this.queue.splice(index, 1);
+          reject(new Error("Queue timeout"));
+        }
+      }, this.config.queueTimeout);
+    });
+  }
+  /**
+   * Process queued requests
+   */
+  processQueue() {
+    while (this.queue.length > 0 && this.available.length > 0) {
+      const item = this.queue.shift();
+      const age = Date.now() - item.queuedAt;
+      if (age > this.config.queueTimeout) {
+        item.reject(new Error("Queue timeout"));
+        continue;
+      }
+      this.acquire().then(item.resolve).catch(item.reject);
+    }
+  }
+  /**
+   * Start background recycling task
+   */
+  startRecycling() {
+    this.recycleTimer = setInterval(() => {
+      for (const instance of this.instances) {
+        if (instance.status === "idle" && this.shouldRecycle(instance)) {
+          this.recycleInstance(instance).catch(() => {
+          });
+        }
+      }
+    }, this.config.recycleCheckInterval);
+    this.recycleTimer.unref();
+  }
+  /**
+   * Start background health checks
+   */
+  startHealthChecks() {
+    this.healthTimer = setInterval(async () => {
+      const health = await this.healthCheck();
+      if (!health.healthy && health.issues.length > 0) {
+        console.warn("[BrowserPool] Health issues:", health.issues);
+      }
+    }, this.config.healthCheckInterval);
+    this.healthTimer.unref();
+  }
+};
+
+// src/client.ts
+var logger2 = createLogger("client");
+var ReaderClient = class {
+  heroCore = null;
+  pool = null;
+  initialized = false;
+  initializing = null;
+  closed = false;
+  options;
+  proxyIndex = 0;
+  cleanupHandler = null;
+  constructor(options = {}) {
+    this.options = options;
+    const skipTLS = options.skipTLSVerification ?? true;
+    if (skipTLS) {
+      process.env.MITM_ALLOW_INSECURE = "true";
+    }
+    this.registerCleanup();
+  }
+  /**
+   * Get the next proxy from the rotation pool
+   */
+  getNextProxy() {
+    const { proxies, proxyRotation = "round-robin" } = this.options;
+    if (!proxies || proxies.length === 0) {
+      return void 0;
+    }
+    if (proxyRotation === "random") {
+      return proxies[Math.floor(Math.random() * proxies.length)];
+    }
+    const proxy = proxies[this.proxyIndex % proxies.length];
+    this.proxyIndex++;
+    return proxy;
+  }
+  /**
+   * Initialize HeroCore. Called automatically on first scrape/crawl.
+   * Can be called explicitly if you want to pre-warm the client.
+   */
+  async start() {
+    if (this.closed) {
+      throw new Error("ReaderClient has been closed. Create a new instance.");
+    }
+    if (this.initialized) {
+      return;
+    }
+    if (this.initializing) {
+      await this.initializing;
+      return;
+    }
+    this.initializing = this.initializeCore();
+    await this.initializing;
+    this.initializing = null;
+  }
+  /**
+   * Internal initialization logic
+   */
+  async initializeCore() {
+    try {
+      if (this.options.verbose) {
+        logger2.info("Starting HeroCore...");
+      }
+      this.heroCore = new HeroCore();
+      await this.heroCore.start();
+      if (this.options.verbose) {
+        logger2.info("HeroCore started successfully");
+      }
+      if (this.options.verbose) {
+        logger2.info("Initializing browser pool...");
+      }
+      const browserPoolConfig = this.options.browserPool;
+      const poolConfig = {
+        size: browserPoolConfig?.size ?? 2,
+        retireAfterPageCount: browserPoolConfig?.retireAfterPages ?? 100,
+        retireAfterAgeMs: (browserPoolConfig?.retireAfterMinutes ?? 30) * 60 * 1e3,
+        maxQueueSize: browserPoolConfig?.maxQueueSize ?? 100
+      };
+      this.pool = new BrowserPool(
+        poolConfig,
+        void 0,
+        // proxy set per-request
+        this.options.showChrome,
+        this.createConnection(),
+        void 0,
+        // userAgent
+        this.options.verbose
+      );
+      await this.pool.initialize();
+      this.initialized = true;
+      if (this.options.verbose) {
+        logger2.info("Browser pool initialized successfully");
+      }
+    } catch (error) {
+      if (this.pool) {
+        await this.pool.shutdown().catch(() => {
+        });
+        this.pool = null;
+      }
+      if (this.heroCore) {
+        await this.heroCore.close().catch(() => {
+        });
+        this.heroCore = null;
+      }
+      this.initialized = false;
+      const message = error.message || String(error);
+      if (message.includes("EADDRINUSE")) {
+        throw new Error(
+          "Failed to start HeroCore: Port already in use. Another instance may be running. Close it or use a different port."
+        );
+      }
+      if (message.includes("chrome") || message.includes("Chrome")) {
+        throw new Error(
+          "Failed to start HeroCore: Chrome/Chromium not found. Please install Chrome or set CHROME_PATH environment variable."
+        );
+      }
+      throw new Error(`Failed to start HeroCore: ${message}`);
+    }
+  }
+  /**
+   * Create a connection to the HeroCore instance
+   */
+  createConnection() {
+    if (!this.heroCore) {
+      throw new Error("HeroCore not initialized. This should not happen.");
+    }
+    const bridge = new TransportBridge();
+    this.heroCore.addConnection(bridge.transportToClient);
+    return new ConnectionToHeroCore(bridge.transportToCore);
+  }
+  /**
+   * Ensure client is initialized before operation
+   */
+  async ensureInitialized() {
+    if (this.closed) {
+      throw new Error("ReaderClient has been closed. Create a new instance.");
+    }
+    if (!this.initialized) {
+      await this.start();
+    }
+  }
+  /**
+   * Scrape one or more URLs
+   *
+   * @param options - Scrape options (urls, formats, etc.)
+   * @returns Scrape result with data and metadata
+   *
+   * @example
+   * const result = await reader.scrape({
+   *   urls: ['https://example.com'],
+   *   formats: ['markdown', 'html'],
+   * });
+   */
+  async scrape(options) {
+    await this.ensureInitialized();
+    if (!this.pool) {
+      throw new Error("Browser pool not initialized. This should not happen.");
+    }
+    const proxy = options.proxy ?? this.getNextProxy();
+    return await scrape({
+      ...options,
+      proxy,
+      showChrome: options.showChrome ?? this.options.showChrome,
+      verbose: options.verbose ?? this.options.verbose,
+      pool: this.pool
+    });
+  }
+  /**
+   * Crawl a website to discover URLs
+   *
+   * @param options - Crawl options (url, depth, maxPages, etc.)
+   * @returns Crawl result with discovered URLs and optional scraped content
+   *
+   * @example
+   * const result = await reader.crawl({
+   *   url: 'https://example.com',
+   *   depth: 2,
+   *   maxPages: 50,
+   *   scrape: true,
+   * });
+   */
+  async crawl(options) {
+    await this.ensureInitialized();
+    if (!this.pool) {
+      throw new Error("Browser pool not initialized. This should not happen.");
+    }
+    const proxy = options.proxy ?? this.getNextProxy();
+    return await crawl({
+      ...options,
+      proxy,
+      pool: this.pool
+    });
+  }
+  /**
+   * Check if the client is initialized and ready
+   */
+  isReady() {
+    return this.initialized && !this.closed;
+  }
+  /**
+   * Close the client and release resources
+   *
+   * Note: This is optional - the client will auto-close on process exit.
+   */
+  async close() {
+    if (this.closed) {
+      return;
+    }
+    this.closed = true;
+    this.removeCleanupHandlers();
+    if (this.pool) {
+      if (this.options.verbose) {
+        logger2.info("Shutting down browser pool...");
+      }
+      try {
+        await this.pool.shutdown();
+      } catch (error) {
+        if (this.options.verbose) {
+          logger2.warn(`Error shutting down pool: ${error.message}`);
+        }
+      }
+      this.pool = null;
+    }
+    if (this.heroCore) {
+      if (this.options.verbose) {
+        logger2.info("Closing HeroCore...");
+      }
+      try {
+        await this.heroCore.close();
+        await HeroCore.shutdown();
+      } catch (error) {
+        if (this.options.verbose) {
+          logger2.warn(`Error closing HeroCore: ${error.message}`);
+        }
+      }
+      this.heroCore = null;
+    }
+    this.initialized = false;
+    if (this.options.verbose) {
+      logger2.info("ReaderClient closed");
+    }
+  }
+  /**
+   * Register cleanup handlers for process exit
+   */
+  registerCleanup() {
+    this.cleanupHandler = async () => {
+      await this.close();
+    };
+    process.once("beforeExit", this.cleanupHandler);
+    process.once("SIGINT", async () => {
+      await this.cleanupHandler?.();
+      process.exit(0);
+    });
+    process.once("SIGTERM", async () => {
+      await this.cleanupHandler?.();
+      process.exit(0);
+    });
+  }
+  /**
+   * Remove process cleanup handlers
+   */
+  removeCleanupHandlers() {
+    if (this.cleanupHandler) {
+      process.removeListener("beforeExit", this.cleanupHandler);
+      this.cleanupHandler = null;
+    }
+  }
+};
+
+// src/daemon/server.ts
+import http from "http";
+var logger3 = createLogger("daemon");
+var DEFAULT_DAEMON_PORT = 3847;
+var PID_FILE_NAME = ".reader-daemon.pid";
+var DaemonServer = class {
+  server = null;
+  client = null;
+  options;
+  startTime = 0;
+  constructor(options = {}) {
+    this.options = {
+      port: options.port ?? DEFAULT_DAEMON_PORT,
+      poolSize: options.poolSize ?? 5,
+      verbose: options.verbose ?? false,
+      showChrome: options.showChrome ?? false
+    };
+  }
+  /**
+   * Start the daemon server
+   */
+  async start() {
+    if (this.server) {
+      throw new Error("Daemon is already running");
+    }
+    const clientOptions = {
+      verbose: this.options.verbose,
+      showChrome: this.options.showChrome,
+      browserPool: {
+        size: this.options.poolSize
+      }
+    };
+    this.client = new ReaderClient(clientOptions);
+    await this.client.start();
+    this.server = http.createServer(this.handleRequest.bind(this));
+    await new Promise((resolve, reject) => {
+      this.server.listen(this.options.port, () => {
+        this.startTime = Date.now();
+        if (this.options.verbose) {
+          logger3.info(`Daemon started on port ${this.options.port} with pool size ${this.options.poolSize}`);
+        }
+        resolve();
+      });
+      this.server.on("error", (error) => {
+        if (error.code === "EADDRINUSE") {
+          reject(new Error(`Port ${this.options.port} is already in use. Is another daemon running?`));
+        } else {
+          reject(error);
+        }
+      });
+    });
+    await this.writePidFile();
+  }
+  /**
+   * Stop the daemon server
+   */
+  async stop() {
+    if (this.server) {
+      await new Promise((resolve) => {
+        this.server.close(() => resolve());
+      });
+      this.server = null;
+    }
+    if (this.client) {
+      await this.client.close();
+      this.client = null;
+    }
+    await this.removePidFile();
+    if (this.options.verbose) {
+      logger3.info("Daemon stopped");
+    }
+  }
+  /**
+   * Get the port the daemon is running on
+   */
+  getPort() {
+    return this.options.port;
+  }
+  /**
+   * Handle incoming HTTP requests
+   */
+  async handleRequest(req, res) {
+    if (req.method !== "POST" || req.url !== "/") {
+      res.writeHead(404, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ success: false, error: "Not found" }));
+      return;
+    }
+    let body = "";
+    for await (const chunk of req) {
+      body += chunk;
+    }
+    let request;
+    try {
+      request = JSON.parse(body);
+    } catch {
+      this.sendResponse(res, 400, { success: false, error: "Invalid JSON" });
+      return;
+    }
+    try {
+      switch (request.action) {
+        case "scrape":
+          await this.handleScrape(res, request.options);
+          break;
+        case "crawl":
+          await this.handleCrawl(res, request.options);
+          break;
+        case "status":
+          this.handleStatus(res);
+          break;
+        case "shutdown":
+          await this.handleShutdown(res);
+          break;
+        default:
+          this.sendResponse(res, 400, { success: false, error: "Unknown action" });
+      }
+    } catch (error) {
+      this.sendResponse(res, 500, { success: false, error: error.message });
+    }
+  }
+  /**
+   * Handle scrape request
+   */
+  async handleScrape(res, options) {
+    if (!this.client) {
+      this.sendResponse(res, 500, { success: false, error: "Client not initialized" });
+      return;
+    }
+    const result = await this.client.scrape(options);
+    this.sendResponse(res, 200, { success: true, data: result });
+  }
+  /**
+   * Handle crawl request
+   */
+  async handleCrawl(res, options) {
+    if (!this.client) {
+      this.sendResponse(res, 500, { success: false, error: "Client not initialized" });
+      return;
+    }
+    const result = await this.client.crawl(options);
+    this.sendResponse(res, 200, { success: true, data: result });
+  }
+  /**
+   * Handle status request
+   */
+  handleStatus(res) {
+    const status = {
+      running: true,
+      port: this.options.port,
+      poolSize: this.options.poolSize,
+      uptime: Date.now() - this.startTime,
+      pid: process.pid
+    };
+    this.sendResponse(res, 200, { success: true, data: status });
+  }
+  /**
+   * Handle shutdown request
+   */
+  async handleShutdown(res) {
+    this.sendResponse(res, 200, { success: true, data: { message: "Shutting down" } });
+    setTimeout(() => {
+      this.stop().then(() => process.exit(0));
+    }, 100);
+  }
+  /**
+   * Send JSON response
+   */
+  sendResponse(res, statusCode, data) {
+    res.writeHead(statusCode, { "Content-Type": "application/json" });
+    res.end(JSON.stringify(data));
+  }
+  /**
+   * Write PID file
+   */
+  async writePidFile() {
+    const fs = await import("fs/promises");
+    const path = await import("path");
+    const os = await import("os");
+    const pidFile = path.join(os.tmpdir(), PID_FILE_NAME);
+    const data = JSON.stringify({
+      pid: process.pid,
+      port: this.options.port,
+      startedAt: (/* @__PURE__ */ new Date()).toISOString()
+    });
+    await fs.writeFile(pidFile, data);
+  }
+  /**
+   * Remove PID file
+   */
+  async removePidFile() {
+    const fs = await import("fs/promises");
+    const path = await import("path");
+    const os = await import("os");
+    const pidFile = path.join(os.tmpdir(), PID_FILE_NAME);
+    try {
+      await fs.unlink(pidFile);
+    } catch {
+    }
+  }
+};
+async function getPidFilePath() {
+  const path = await import("path");
+  const os = await import("os");
+  return path.join(os.tmpdir(), PID_FILE_NAME);
+}
+async function getDaemonInfo() {
+  const fs = await import("fs/promises");
+  const pidFile = await getPidFilePath();
+  try {
+    const data = await fs.readFile(pidFile, "utf-8");
+    const info = JSON.parse(data);
+    try {
+      process.kill(info.pid, 0);
+      return info;
+    } catch {
+      await fs.unlink(pidFile).catch(() => {
+      });
+      return null;
+    }
+  } catch {
+    return null;
+  }
+}
+
+// src/daemon/client.ts
+import http2 from "http";
+var DaemonClient = class {
+  options;
+  constructor(options = {}) {
+    this.options = {
+      port: options.port ?? DEFAULT_DAEMON_PORT,
+      timeoutMs: options.timeoutMs ?? 6e5
+      // 10 minutes default
+    };
+  }
+  /**
+   * Scrape URLs via daemon
+   */
+  async scrape(options) {
+    return this.request({
+      action: "scrape",
+      options
+    });
+  }
+  /**
+   * Crawl URL via daemon
+   */
+  async crawl(options) {
+    return this.request({
+      action: "crawl",
+      options
+    });
+  }
+  /**
+   * Get daemon status
+   */
+  async status() {
+    return this.request({
+      action: "status"
+    });
+  }
+  /**
+   * Request daemon shutdown
+   */
+  async shutdown() {
+    await this.request({
+      action: "shutdown"
+    });
+  }
+  /**
+   * Check if daemon is reachable
+   */
+  async isRunning() {
+    try {
+      await this.status();
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  /**
+   * Make HTTP request to daemon
+   */
+  request(body) {
+    return new Promise((resolve, reject) => {
+      const data = JSON.stringify(body);
+      const req = http2.request(
+        {
+          hostname: "127.0.0.1",
+          port: this.options.port,
+          path: "/",
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            "Content-Length": Buffer.byteLength(data)
+          },
+          timeout: this.options.timeoutMs
+        },
+        (res) => {
+          let responseBody = "";
+          res.on("data", (chunk) => {
+            responseBody += chunk;
+          });
+          res.on("end", () => {
+            try {
+              const response = JSON.parse(responseBody);
+              if (response.success) {
+                resolve(response.data);
+              } else {
+                reject(new Error(response.error || "Unknown daemon error"));
+              }
+            } catch (error) {
+              reject(new Error(`Failed to parse daemon response: ${responseBody}`));
+            }
+          });
+        }
+      );
+      req.on("error", (error) => {
+        if (error.code === "ECONNREFUSED") {
+          reject(new Error(`Cannot connect to daemon on port ${this.options.port}. Is it running?`));
+        } else {
+          reject(error);
+        }
+      });
+      req.on("timeout", () => {
+        req.destroy();
+        reject(new Error(`Request to daemon timed out after ${this.options.timeoutMs}ms`));
+      });
+      req.write(data);
+      req.end();
+    });
+  }
+};
+async function isDaemonRunning(port = DEFAULT_DAEMON_PORT) {
+  const client = new DaemonClient({ port, timeoutMs: 5e3 });
+  return client.isRunning();
+}
+export {
+  BrowserPool,
+  Crawler,
+  DEFAULT_DAEMON_PORT,
+  DEFAULT_OPTIONS,
+  DaemonClient,
+  DaemonServer,
+  BrowserPool as HeroBrowserPool,
+  ReaderClient,
+  Scraper,
+  cleanContent,
+  crawl,
+  createHeroConfig,
+  createProxyUrl,
+  detectChallenge,
+  extractMetadata,
+  formatToHTML,
+  formatToJson,
+  formatToJsonLite,
+  formatToMarkdown,
+  formatToText,
+  getDaemonInfo,
+  getPidFilePath,
+  getUrlKey,
+  handleChallenge,
+  isChallengePage,
+  isDaemonRunning,
+  isSameDomain,
+  isValidFormat,
+  isValidUrl,
+  parseProxyUrl,
+  rateLimit,
+  resolveUrl,
+  scrape,
+  shouldCrawlUrl,
+  shouldCrawlUrl2 as shouldCrawlUrlFn,
+  validateUrls,
+  waitForChallengeResolution,
+  waitForSelector
+};
+//# sourceMappingURL=index.js.map