@vakra-dev/reader 0.0.1 → 0.0.3

package/dist/cli/index.js

@@ -1,4 +1,10 @@
 #!/usr/bin/env node
+var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
+  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
+}) : x)(function(x) {
+  if (typeof require !== "undefined") return require.apply(this, arguments);
+  throw Error('Dynamic require of "' + x + '" is not supported');
+});
 
 // src/cli/index.ts
 import { Command } from "commander";
@@ -12,27 +18,36 @@ import { ConnectionToHeroCore } from "@ulixee/hero";
 import pLimit from "p-limit";
 
 // src/cloudflare/detector.ts
-var CHALLENGE_DOM_SELECTORS = [
+var CLOUDFLARE_CHALLENGE_SELECTORS = [
   "#challenge-running",
   "#challenge-stage",
   "#challenge-form",
-  ".cf-browser-verification"
+  ".cf-browser-verification",
+  "#cf-wrapper",
+  "#cf-hcaptcha-container",
+  "#turnstile-wrapper"
 ];
-var CHALLENGE_TEXT_PATTERNS = [
-  "verifying you are human",
+var CLOUDFLARE_TEXT_PATTERNS = [
   "checking if the site connection is secure",
-  "this process is automatic. your browser will redirect"
+  "this process is automatic. your browser will redirect",
+  "ray id:",
+  "performance & security by cloudflare"
+];
+var CLOUDFLARE_INFRA_PATTERNS = [
+  "/cdn-cgi/",
+  "cloudflare",
+  "__cf_bm",
+  "cf-ray"
 ];
-var BLOCKED_SIGNALS = [
-  "you have been blocked",
-  "access to this page has been denied",
+var CLOUDFLARE_BLOCKED_PATTERNS = [
   "sorry, you have been blocked",
-  "access denied",
-  "403 forbidden"
+  "ray id:"
 ];
 async function detectChallenge(hero) {
   const signals = [];
   let type = "none";
+  let hasCloudflareInfra = false;
+  let hasChallengeIndicator = false;
   try {
     if (!hero.document) {
       return {
@@ -44,30 +59,51 @@ async function detectChallenge(hero) {
     }
     const html = await hero.document.documentElement.outerHTML;
     const htmlLower = html.toLowerCase();
-    for (const selector of CHALLENGE_DOM_SELECTORS) {
-      if (htmlLower.includes(selector.toLowerCase())) {
-        signals.push(`Challenge element: ${selector}`);
-        type = "js_challenge";
+    for (const pattern of CLOUDFLARE_INFRA_PATTERNS) {
+      if (htmlLower.includes(pattern)) {
+        hasCloudflareInfra = true;
+        signals.push(`Cloudflare infra: "${pattern}"`);
+        break;
+      }
+    }
+    if (!hasCloudflareInfra) {
+      return {
+        isChallenge: false,
+        type: "none",
+        confidence: 0,
+        signals: ["No Cloudflare infrastructure detected"]
+      };
+    }
+    for (const selector of CLOUDFLARE_CHALLENGE_SELECTORS) {
+      try {
+        const element = await hero.document.querySelector(selector);
+        if (element) {
+          hasChallengeIndicator = true;
+          signals.push(`Challenge element: ${selector}`);
+          type = "js_challenge";
+        }
+      } catch {
       }
     }
-    for (const pattern of CHALLENGE_TEXT_PATTERNS) {
+    for (const pattern of CLOUDFLARE_TEXT_PATTERNS) {
       if (htmlLower.includes(pattern)) {
+        hasChallengeIndicator = true;
         signals.push(`Challenge text: "${pattern}"`);
         type = type === "none" ? "js_challenge" : type;
       }
     }
     if (htmlLower.includes("waiting for") && htmlLower.includes("to respond")) {
+      hasChallengeIndicator = true;
       signals.push('Challenge text: "waiting for...to respond"');
       type = type === "none" ? "js_challenge" : type;
     }
-    for (const pattern of BLOCKED_SIGNALS) {
-      if (htmlLower.includes(pattern)) {
-        signals.push(`Blocked: "${pattern}"`);
-        type = "blocked";
-        break;
-      }
+    const hasBlocked = CLOUDFLARE_BLOCKED_PATTERNS.every((p) => htmlLower.includes(p));
+    if (hasBlocked) {
+      hasChallengeIndicator = true;
+      signals.push("Cloudflare block page detected");
+      type = "blocked";
     }
-    const isChallenge = signals.length > 0;
+    const isChallenge = hasCloudflareInfra && hasChallengeIndicator;
     const confidence = isChallenge ? 100 : 0;
     return {
       isChallenge,
@@ -150,84 +186,6 @@ var turndownService = new TurndownService({
   linkStyle: "inlined",
   linkReferenceStyle: "full"
 });
-function formatToMarkdown(pages, baseUrl, scrapedAt, duration, website, includeMetadata = true) {
-  const sections = [];
-  if (includeMetadata) {
-    sections.push(createMarkdownHeader(baseUrl, scrapedAt, duration, website, pages.length));
-  }
-  if (pages.length > 1) {
-    sections.push(createMarkdownTOC(pages));
-  }
-  sections.push(...pages.map((page, index) => createMarkdownPage(page, index + 1)));
-  return sections.join("\n\n");
-}
-function createMarkdownHeader(baseUrl, scrapedAt, duration, website, totalPages) {
-  const title = website.title || extractDomainFromUrl(baseUrl);
-  const description = website.description || "";
-  let header = `# Website Scrape: ${title}
-
-`;
-  header += `**Base URL:** ${baseUrl}
-`;
-  header += `**Scraped at:** ${new Date(scrapedAt).toLocaleString()}
-`;
-  header += `**Duration:** ${duration}ms
-`;
-  header += `**Total pages:** ${totalPages}
-`;
-  if (description) {
-    header += `**Description:** ${description}
-`;
-  }
-  if (website.author) {
-    header += `**Author:** ${website.author}
-`;
-  }
-  if (website.language) {
-    header += `**Language:** ${website.language}
-`;
-  }
-  return header;
-}
-function createMarkdownTOC(pages) {
-  let toc = "## Table of Contents\n\n";
-  pages.forEach((page, index) => {
-    const depth = "  ".repeat(page.depth);
-    const pageNumber = index + 1;
-    const title = page.title || `Page ${pageNumber}`;
-    const cleanTitle = title.replace(/[#[\]/\\:*?"<>|]/g, "").trim();
-    const anchor = cleanTitle.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
-    toc += `${depth}${pageNumber}. [${title}](#page-${pageNumber}-${anchor})
-`;
-  });
-  return toc;
-}
-function createMarkdownPage(page, pageNumber) {
-  const title = page.title || `Page ${pageNumber}`;
-  const cleanTitle = title.replace(/[#[\]/\\:*?"<>|]/g, "").trim();
-  const anchor = cleanTitle.toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "");
-  let pageContent = `---
-
-`;
-  pageContent += `## Page ${pageNumber}: ${title} {#page-${pageNumber}-${anchor}}
-
-`;
-  pageContent += `**URL:** ${page.url}
-`;
-  pageContent += `**Title:** ${page.title}
-`;
-  pageContent += `**Depth:** ${page.depth}
-`;
-  pageContent += `**Fetched at:** ${new Date(page.fetchedAt).toLocaleString()}
-
-`;
-  pageContent += `---
-
-`;
-  const markdown = htmlToMarkdown(page.html);
-  pageContent += markdown;
-  return pageContent;
-}
 function htmlToMarkdown(html) {
   try {
     return turndownService.turndown(html);
@@ -236,574 +194,339 @@ function htmlToMarkdown(html) {
     return html.replace(/<[^>]*>/g, "").trim();
   }
 }
-function extractDomainFromUrl(url) {
-  try {
-    return new URL(url).hostname;
-  } catch {
-    return "Unknown";
-  }
-}
-
-// src/formatters/html.ts
-function formatToHTML(pages, baseUrl, scrapedAt, duration, website) {
-  const html = `<!DOCTYPE html>
-<html lang="${website.language || "en"}">
-<head>
-    <meta charset="${website.charset || "UTF-8"}">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Scrape: ${website.title || extractDomainFromUrl2(baseUrl)}</title>
-    ${generateMetaTags(website)}
-    <style>
-        ${generateCSS()}
-    </style>
-</head>
-<body>
-    <header class="header">
-        <h1>Website Scrape: ${escapeHtml(website.title || extractDomainFromUrl2(baseUrl))}</h1>
-        <div class="meta-info">
-            <p><strong>Base URL:</strong> <a href="${escapeHtml(
-    baseUrl
-  )}" target="_blank">${escapeHtml(baseUrl)}</a></p>
-            <p><strong>Scraped at:</strong> ${new Date(scrapedAt).toLocaleString()}</p>
-            <p><strong>Duration:</strong> ${duration}ms</p>
-            <p><strong>Total pages:</strong> ${pages.length}</p>
-            ${website.description ? `<p><strong>Description:</strong> ${escapeHtml(website.description)}</p>` : ""}
-            ${website.author ? `<p><strong>Author:</strong> ${escapeHtml(website.author)}</p>` : ""}
-            ${website.language ? `<p><strong>Language:</strong> ${escapeHtml(website.language)}</p>` : ""}
-        </div>
-    </header>
-
-    ${pages.length > 1 ? generateTOC(pages) : ""}
-
-    <main class="content">
-        ${pages.map((page, index) => generatePageHTML(page, index + 1)).join("\n")}
-    </main>
-
-    <footer class="footer">
-        <p>Generated by Reader JS/TS SDK</p>
-    </footer>
-
-    <script>
-        ${generateJavaScript()}
-    </script>
-</body>
-</html>`;
-  return html;
-}
-function generateMetaTags(website) {
-  const tags = [];
-  if (website.description) {
-    tags.push(`<meta name="description" content="${escapeHtml(website.description)}">`);
-  }
-  if (website.author) {
-    tags.push(`<meta name="author" content="${escapeHtml(website.author)}">`);
-  }
-  if (website.keywords) {
-    tags.push(`<meta name="keywords" content="${escapeHtml(website.keywords.join(", "))}">`);
-  }
-  if (website.robots) {
-    tags.push(`<meta name="robots" content="${escapeHtml(website.robots)}">`);
-  }
-  if (website.themeColor) {
-    tags.push(`<meta name="theme-color" content="${escapeHtml(website.themeColor)}">`);
-  }
-  if (website.favicon) {
-    tags.push(`<link rel="icon" href="${escapeHtml(website.favicon)}">`);
-  }
-  if (website.canonical) {
-    tags.push(`<link rel="canonical" href="${escapeHtml(website.canonical)}">`);
-  }
-  if (website.openGraph) {
-    const og = website.openGraph;
-    if (og.title) tags.push(`<meta property="og:title" content="${escapeHtml(og.title)}">`);
-    if (og.description)
-      tags.push(`<meta property="og:description" content="${escapeHtml(og.description)}">`);
-    if (og.type) tags.push(`<meta property="og:type" content="${escapeHtml(og.type)}">`);
-    if (og.url) tags.push(`<meta property="og:url" content="${escapeHtml(og.url)}">`);
-    if (og.image) tags.push(`<meta property="og:image" content="${escapeHtml(og.image)}">`);
-    if (og.siteName)
-      tags.push(`<meta property="og:site_name" content="${escapeHtml(og.siteName)}">`);
-    if (og.locale) tags.push(`<meta property="og:locale" content="${escapeHtml(og.locale)}">`);
-  }
-  if (website.twitter) {
-    const twitter = website.twitter;
-    if (twitter.card) tags.push(`<meta name="twitter:card" content="${escapeHtml(twitter.card)}">`);
-    if (twitter.site) tags.push(`<meta name="twitter:site" content="${escapeHtml(twitter.site)}">`);
-    if (twitter.creator)
-      tags.push(`<meta name="twitter:creator" content="${escapeHtml(twitter.creator)}">`);
-    if (twitter.title)
-      tags.push(`<meta name="twitter:title" content="${escapeHtml(twitter.title)}">`);
-    if (twitter.description)
-      tags.push(`<meta name="twitter:description" content="${escapeHtml(twitter.description)}">`);
-    if (twitter.image)
-      tags.push(`<meta name="twitter:image" content="${escapeHtml(twitter.image)}">`);
-  }
-  return tags.join("\n    ");
-}
-function generateCSS() {
-  return `
-        * {
-            margin: 0;
-            padding: 0;
-            box-sizing: border-box;
-        }
-
-        body {
-            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
-            line-height: 1.6;
-            color: #333;
-            background-color: #f8f9fa;
-        }
-
-        .header {
-            background: white;
-            padding: 2rem;
-            border-bottom: 1px solid #e9ecef;
-            margin-bottom: 2rem;
-        }
-
-        .header h1 {
-            color: #2c3e50;
-            margin-bottom: 1rem;
-            font-size: 2rem;
-        }
-
-        .meta-info {
-            display: grid;
-            grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
-            gap: 0.5rem;
-        }
-
-        .meta-info p {
-            margin: 0.25rem 0;
-            font-size: 0.9rem;
-            color: #6c757d;
-        }
-
-        .toc {
-            background: white;
-            padding: 1.5rem;
-            margin: 2rem 0;
-            border-radius: 8px;
-            border: 1px solid #e9ecef;
-        }
-
-        .toc h2 {
-            color: #2c3e50;
-            margin-bottom: 1rem;
-            font-size: 1.25rem;
-        }
-
-        .toc ul {
-            list-style: none;
-        }
-
-        .toc li {
-            margin: 0.5rem 0;
-        }
-
-        .toc a {
-            color: #007bff;
-            text-decoration: none;
-            transition: color 0.2s;
-        }
-
-        .toc a:hover {
-            color: #0056b3;
-            text-decoration: underline;
-        }
-
-        .content {
-            max-width: 800px;
-            margin: 0 auto;
-            padding: 0 1rem;
-        }
-
-        .page {
-            background: white;
-            margin: 2rem 0;
-            padding: 2rem;
-            border-radius: 8px;
-            border: 1px solid #e9ecef;
-            box-shadow: 0 2px 4px rgba(0,0,0,0.05);
-        }
-
-        .page-header {
-            border-bottom: 2px solid #e9ecef;
-            padding-bottom: 1rem;
-            margin-bottom: 2rem;
-        }
-
-        .page-header h2 {
-            color: #2c3e50;
-            margin-bottom: 0.5rem;
-            font-size: 1.5rem;
-        }
-
-        .page-meta {
-            display: flex;
-            flex-wrap: wrap;
-            gap: 1rem;
-            font-size: 0.9rem;
-            color: #6c757d;
-        }
-
-        .page-content {
-            line-height: 1.8;
-        }
-
-        .page-content h1, .page-content h2, .page-content h3,
-        .page-content h4, .page-content h5, .page-content h6 {
-            color: #2c3e50;
-            margin: 1.5rem 0 0.5rem 0;
-        }
-
-        .page-content p {
-            margin: 1rem 0;
-        }
-
-        .page-content a {
-            color: #007bff;
-            text-decoration: none;
-        }
-
-        .page-content a:hover {
-            text-decoration: underline;
-        }
-
-        .page-content code {
-            background: #f8f9fa;
-            padding: 0.2rem 0.4rem;
-            border-radius: 4px;
-            font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
-            font-size: 0.9em;
-        }
-
-        .page-content pre {
-            background: #f8f9fa;
-            padding: 1rem;
-            border-radius: 4px;
-            overflow-x: auto;
-            margin: 1rem 0;
-        }
-
-        .page-content blockquote {
-            border-left: 4px solid #007bff;
-            padding-left: 1rem;
-            margin: 1rem 0;
-            color: #6c757d;
-        }
-
-        .footer {
-            text-align: center;
-            padding: 2rem;
-            margin-top: 3rem;
-            border-top: 1px solid #e9ecef;
-            color: #6c757d;
-            font-size: 0.9rem;
-        }
-
-        @media (max-width: 768px) {
-            .header {
-                padding: 1rem;
-            }
-
-            .header h1 {
-                font-size: 1.5rem;
-            }
-
-            .page {
-                padding: 1rem;
-            }
-
-            .page-meta {
-                flex-direction: column;
-                gap: 0.5rem;
-            }
-        }
-    `.trim();
-}
-function generateTOC(pages) {
-  const tocItems = pages.map((page, index) => {
-    const pageNumber = index + 1;
-    const title = page.title || `Page ${pageNumber}`;
-    const id = `page-${pageNumber}`;
-    return `<li><a href="#${id}">${pageNumber}. ${escapeHtml(title)}</a></li>`;
-  }).join("\n");
-  return `
-    <nav class="toc">
-        <h2>Table of Contents</h2>
-        <ul>
-            ${tocItems}
-        </ul>
-    </nav>`;
-}
-function generatePageHTML(page, pageNumber) {
-  const id = `page-${pageNumber}`;
-  const title = page.title || `Page ${pageNumber}`;
-  return `
-    <article class="page" id="${id}">
-        <div class="page-header">
-            <h2>${pageNumber}. ${escapeHtml(title)}</h2>
-            <div class="page-meta">
-                <span><strong>URL:</strong> <a href="${escapeHtml(
-    page.url
-  )}" target="_blank">${escapeHtml(page.url)}</a></span>
-                <span><strong>Depth:</strong> ${page.depth}</span>
-                <span><strong>Fetched:</strong> ${new Date(page.fetchedAt).toLocaleString()}</span>
-            </div>
-        </div>
-        <div class="page-content">
-            ${page.html}
-        </div>
-    </article>`;
-}
-function generateJavaScript() {
-  return `
-        // Smooth scrolling for TOC links
-        document.querySelectorAll('a[href^="#"]').forEach(anchor => {
-            anchor.addEventListener('click', function (e) {
-                e.preventDefault();
-                const target = document.querySelector(this.getAttribute('href'));
-                if (target) {
-                    target.scrollIntoView({
-                        behavior: 'smooth',
-                        block: 'start'
-                    });
-                }
-            });
-        });
-
-        // Highlight current section in TOC
-        window.addEventListener('scroll', function() {
-            const pages = document.querySelectorAll('.page');
-            const tocLinks = document.querySelectorAll('.toc a');
-            
-            let currentPage = null;
-            pages.forEach(page => {
-                const rect = page.getBoundingClientRect();
-                if (rect.top <= 100) {
-                    currentPage = page;
-                }
-            });
-
-            tocLinks.forEach(link => {
-                link.style.fontWeight = 'normal';
-                const target = document.querySelector(link.getAttribute('href'));
-                if (target === currentPage) {
-                    link.style.fontWeight = 'bold';
-                }
-            });
-        });
-    `;
-}
-function escapeHtml(text) {
-  return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, "&quot;").replace(/'/g, "&#039;").replace(/\//g, "&#x2F;");
-}
-function extractDomainFromUrl2(url) {
-  try {
-    return new URL(url).hostname;
-  } catch {
-    return "Unknown";
-  }
-}
-
-// src/formatters/json.ts
-function formatToJson(pages, baseUrl, scrapedAt, duration, website) {
-  const jsonResult = {
-    metadata: {
-      baseUrl,
-      totalPages: pages.length,
-      scrapedAt,
-      duration,
-      website
-    },
-    pages: pages.map((page, index) => ({
-      index: index + 1,
-      url: page.url,
-      title: page.title,
-      markdown: page.markdown,
-      html: page.html,
-      fetchedAt: page.fetchedAt,
-      depth: page.depth,
-      wordCount: countWords(page.markdown),
-      readingTime: estimateReadingTime(page.markdown)
-    }))
-  };
-  return JSON.stringify(jsonResult, null, 2);
-}
-function countWords(markdown) {
-  const plainText = markdown.replace(/#{1,6}\s+/g, "").replace(/\*\*(.*?)\*\*/g, "$1").replace(/\*(.*?)\*/g, "$1").replace(/`(.*?)`/g, "$1").replace(/```[\s\S]*?```/g, "").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replace(/^\s*[-*+]\s+/gm, "").replace(/^\s*\d+\.\s+/gm, "").replace(/^\s*>\s+/gm, "").replace(/\n{3,}/g, "\n\n").trim();
-  return plainText.split(/\s+/).filter((word) => word.length > 0).length;
-}
-function estimateReadingTime(markdown) {
-  const wordCount = countWords(markdown);
-  return Math.ceil(wordCount / 200);
-}
-
-// src/formatters/text.ts
-import { parseHTML } from "linkedom";
-function formatToText(pages, baseUrl, scrapedAt, duration, website, includeMetadata = true) {
-  const sections = [];
-  if (includeMetadata) {
-    sections.push(createTextHeader(baseUrl, scrapedAt, duration, website, pages.length));
-  }
-  sections.push(...pages.map((page, index) => createTextPage(page, index + 1, pages.length > 1)));
-  return sections.join("\n\n");
-}
-function createTextHeader(baseUrl, scrapedAt, duration, website, totalPages) {
-  const title = website.title || extractDomainFromUrl3(baseUrl);
-  const lines = [];
-  lines.push(`=== ${title} ===`);
-  lines.push("");
-  lines.push(`URL: ${baseUrl}`);
-  lines.push(`Scraped: ${new Date(scrapedAt).toLocaleString()}`);
-  lines.push(`Duration: ${duration}ms`);
-  lines.push(`Pages: ${totalPages}`);
-  if (website.description) {
-    lines.push(`Description: ${website.description}`);
-  }
-  if (website.author) {
-    lines.push(`Author: ${website.author}`);
-  }
-  if (website.language) {
-    lines.push(`Language: ${website.language}`);
-  }
-  return lines.join("\n");
-}
-function createTextPage(page, pageNumber, showSeparator) {
-  const lines = [];
-  if (showSeparator) {
-    lines.push("\u2500".repeat(60));
-    lines.push(`Page ${pageNumber}: ${page.title || "Untitled"}`);
-    lines.push(`URL: ${page.url}`);
-    lines.push("\u2500".repeat(60));
-  }
-  const plainText = htmlToPlainText(page.html);
-  lines.push(plainText);
-  return lines.join("\n");
-}
-function htmlToPlainText(html) {
-  const { document } = parseHTML(html);
-  const elementsToRemove = ["script", "style", "noscript", "svg", "canvas", "template"];
-  elementsToRemove.forEach((tag) => {
-    document.querySelectorAll(tag).forEach((el) => el.remove());
-  });
-  let text = document.body?.textContent || document.documentElement?.textContent || "";
-  text = text.replace(/[ \t]+/g, " ");
-  text = text.replace(/\n[ \t]+/g, "\n");
-  text = text.replace(/[ \t]+\n/g, "\n");
-  text = text.replace(/\n{3,}/g, "\n\n");
-  text = text.trim();
-  return text;
-}
-function extractDomainFromUrl3(url) {
-  try {
-    return new URL(url).hostname;
-  } catch {
-    return "Unknown";
-  }
-}
 
 // src/utils/content-cleaner.ts
-import { parseHTML as parseHTML2 } from "linkedom";
+import { parseHTML } from "linkedom";
 var ALWAYS_REMOVE_SELECTORS = [
-  // Navigation and menus
-  "nav",
-  "header nav",
-  "footer nav",
-  ".nav",
-  ".navigation",
-  ".menu",
-  ".navbar",
-  ".sidebar",
-  ".aside",
-  // Header and footer elements
-  "header",
-  "footer",
-  ".site-header",
-  ".page-header",
-  ".site-footer",
-  ".page-footer",
-  // Social media and sharing
-  ".social",
-  ".share",
-  ".sharing",
-  ".twitter",
-  ".facebook",
-  ".linkedin",
-  ".instagram",
-  // Comments and discussions
-  ".comments",
-  ".comment",
-  ".discussion",
-  ".disqus",
-  // Forms and interactive elements
-  "form",
-  "input",
-  "button:not([type='submit'])",
-  "select",
-  "textarea",
   // Scripts and styles
   "script",
   "style",
   "noscript",
+  "link[rel='stylesheet']",
   // Hidden elements
   "[hidden]",
+  "[aria-hidden='true']",
   "[style*='display: none']",
   "[style*='display:none']",
-  // Common utility classes
-  ".cookie",
-  ".cookie-banner",
-  ".popup",
+  "[style*='visibility: hidden']",
+  "[style*='visibility:hidden']",
+  // SVG icons and decorative elements
+  "svg[aria-hidden='true']",
+  "svg.icon",
+  "svg[class*='icon']",
+  // Template and metadata
+  "template",
+  "meta",
+  // Embeds that don't convert to text
+  "iframe",
+  "canvas",
+  "object",
+  "embed",
+  // Forms (usually not main content)
+  "form",
+  "input",
+  "select",
+  "textarea",
+  "button"
+];
+var OVERLAY_SELECTORS = [
+  "[class*='modal']",
+  "[class*='popup']",
+  "[class*='overlay']",
+  "[class*='dialog']",
+  "[role='dialog']",
+  "[role='alertdialog']",
+  "[class*='cookie']",
+  "[class*='consent']",
+  "[class*='gdpr']",
+  "[class*='privacy-banner']",
+  "[class*='notification-bar']",
+  "[id*='cookie']",
+  "[id*='consent']",
+  "[id*='gdpr']",
+  // Fixed/sticky positioned elements
+  "[style*='position: fixed']",
+  "[style*='position:fixed']",
+  "[style*='position: sticky']",
+  "[style*='position:sticky']"
+];
+var NAVIGATION_SELECTORS = [
+  // Semantic elements
+  "header",
+  "footer",
+  "nav",
+  "aside",
+  // Header variations
+  ".header",
+  ".top",
+  ".navbar",
+  "#header",
+  // Footer variations
+  ".footer",
+  ".bottom",
+  "#footer",
+  // Sidebars
+  ".sidebar",
+  ".side",
+  ".aside",
+  "#sidebar",
+  // Modals/popups (backup if not caught by OVERLAY_SELECTORS)
   ".modal",
+  ".popup",
+  "#modal",
   ".overlay",
-  ".notification",
+  // Ads
+  ".ad",
+  ".ads",
+  ".advert",
+  "#ad",
+  // Language selectors
+  ".lang-selector",
+  ".language",
+  "#language-selector",
+  // Social
+  ".social",
+  ".social-media",
+  ".social-links",
+  "#social",
+  // Navigation/menus
+  ".menu",
+  ".navigation",
+  "#nav",
   // Breadcrumbs
-  ".breadcrumb",
   ".breadcrumbs",
-  ".breadcrumb-trail"
+  "#breadcrumbs",
+  // Share buttons
+  ".share",
+  "#share",
+  // Widgets
+  ".widget",
+  "#widget",
+  // Cookie notices (backup)
+  ".cookie",
+  "#cookie"
+];
+var FORCE_INCLUDE_SELECTORS = [
+  // IDs
+  "#main",
+  "#content",
+  "#main-content",
+  "#article",
+  "#post",
+  "#page-content",
+  // Semantic elements
+  "main",
+  "article",
+  "[role='main']",
+  // Classes
+  ".main-content",
+  ".content",
+  ".post-content",
+  ".article-content",
+  ".entry-content",
+  ".page-content",
+  ".article-body",
+  ".post-body",
+  ".story-content",
+  ".blog-content"
 ];
 var AD_SELECTORS = [
-  // Ads and promotions
-  ".ad",
-  ".ads",
-  ".advertisement",
-  ".promotion",
-  ".sponsored",
-  "[class*='ad-']",
-  "[id*='ad-']",
-  "[class*='advert']",
-  "[id*='advert']",
-  "[class*='banner']",
-  "[id*='banner']",
+  // Google ads
+  "ins.adsbygoogle",
   ".google-ad",
   ".adsense",
+  // Generic ad containers
   "[data-ad]",
   "[data-ads]",
-  "ins.adsbygoogle",
-  // Tracking
-  "[class*='tracking']",
-  "[id*='tracking']",
-  "[class*='analytics']",
-  "[id*='analytics']"
+  "[data-ad-slot]",
+  "[data-ad-client]",
+  // Common ad class patterns
+  ".ad-container",
+  ".ad-wrapper",
+  ".advertisement",
+  ".sponsored-content",
+  // Tracking pixels
+  "img[width='1'][height='1']",
+  "img[src*='pixel']",
+  "img[src*='tracking']",
+  "img[src*='analytics']"
 ];
-function cleanHtml(html, baseUrl, options = {}) {
-  const { removeAds = true, removeBase64Images = true } = options;
-  const { document } = parseHTML2(html);
-  for (const selector of ALWAYS_REMOVE_SELECTORS) {
+function getLinkDensity(element) {
+  const text = element.textContent || "";
+  const textLength = text.trim().length;
+  if (textLength === 0) return 1;
+  let linkLength = 0;
+  element.querySelectorAll("a").forEach((link) => {
+    linkLength += (link.textContent || "").trim().length;
+  });
+  return linkLength / textLength;
+}
+function getContentScore(element) {
+  let score = 0;
+  const text = element.textContent || "";
+  const textLength = text.trim().length;
+  score += Math.min(textLength / 100, 50);
+  score += element.querySelectorAll("p").length * 3;
+  score += element.querySelectorAll("h1, h2, h3, h4, h5, h6").length * 2;
+  score += element.querySelectorAll("img").length * 1;
+  score -= element.querySelectorAll("a").length * 0.5;
+  score -= element.querySelectorAll("li").length * 0.2;
+  const linkDensity = getLinkDensity(element);
+  if (linkDensity > 0.5) score -= 30;
+  else if (linkDensity > 0.3) score -= 15;
+  const classAndId = (element.className || "") + " " + (element.id || "");
+  if (/article|content|post|body|main|entry/i.test(classAndId)) score += 25;
+  if (/comment|sidebar|footer|nav|menu|header|widget|ad/i.test(classAndId)) score -= 25;
+  return score;
+}
+function looksLikeNavigation(element) {
+  const linkDensity = getLinkDensity(element);
+  if (linkDensity > 0.5) return true;
+  const listItems = element.querySelectorAll("li");
+  const links = element.querySelectorAll("a");
+  if (listItems.length > 5 && links.length > listItems.length * 0.8) return true;
+  return false;
+}
+function removeElements(document, selectors) {
+  for (const selector of selectors) {
     try {
       document.querySelectorAll(selector).forEach((el) => el.remove());
     } catch {
     }
   }
+}
+function removeWithProtection(document, selectorsToRemove, protectedSelectors) {
+  for (const selector of selectorsToRemove) {
+    try {
+      document.querySelectorAll(selector).forEach((element) => {
+        const isProtected = protectedSelectors.some((ps) => {
+          try {
+            return element.matches(ps);
+          } catch {
+            return false;
+          }
+        });
+        if (isProtected) return;
+        const containsProtected = protectedSelectors.some((ps) => {
+          try {
+            return element.querySelector(ps) !== null;
+          } catch {
+            return false;
+          }
+        });
+        if (containsProtected) return;
+        element.remove();
+      });
+    } catch {
+    }
+  }
+}
+function findMainContent(document) {
+  const isValidContent = (el) => {
+    if (!el) return false;
+    const text = el.textContent || "";
+    if (text.trim().length < 100) return false;
+    if (looksLikeNavigation(el)) return false;
+    return true;
+  };
+  const main = document.querySelector("main");
+  if (isValidContent(main) && getLinkDensity(main) < 0.4) {
+    return main;
+  }
+  const roleMain = document.querySelector('[role="main"]');
+  if (isValidContent(roleMain) && getLinkDensity(roleMain) < 0.4) {
+    return roleMain;
+  }
+  const articles = document.querySelectorAll("article");
+  if (articles.length === 1 && isValidContent(articles[0])) {
+    return articles[0];
+  }
+  const contentSelectors = [
+    "#content",
+    "#main-content",
+    "#main",
+    ".content",
+    ".main-content",
+    ".post-content",
+    ".article-content",
+    ".entry-content",
+    ".page-content",
+    ".article-body",
+    ".post-body",
+    ".story-content",
+    ".blog-content"
+  ];
+  for (const selector of contentSelectors) {
+    try {
+      const el = document.querySelector(selector);
+      if (isValidContent(el) && getLinkDensity(el) < 0.4) {
+        return el;
+      }
+    } catch {
+    }
+  }
+  const candidates = [];
+  const containers = document.querySelectorAll("div, section, article");
+  containers.forEach((el) => {
+    const text = el.textContent || "";
+    if (text.trim().length < 200) return;
+    const score = getContentScore(el);
+    if (score > 0) {
+      candidates.push({ el, score });
+    }
+  });
+  candidates.sort((a, b) => b.score - a.score);
+  if (candidates.length > 0 && candidates[0].score > 20) {
+    return candidates[0].el;
+  }
+  return null;
+}
+function cleanHtml(html, baseUrl, options = {}) {
+  const {
+    removeAds = true,
+    removeBase64Images = true,
+    onlyMainContent = true,
+    includeTags,
+    excludeTags
+  } = options;
+  const { document } = parseHTML(html);
+  removeElements(document, ALWAYS_REMOVE_SELECTORS);
+  removeElements(document, OVERLAY_SELECTORS);
   if (removeAds) {
-    for (const selector of AD_SELECTORS) {
+    removeElements(document, AD_SELECTORS);
+  }
+  if (excludeTags && excludeTags.length > 0) {
+    removeElements(document, excludeTags);
+  }
+  if (onlyMainContent) {
+    removeWithProtection(document, NAVIGATION_SELECTORS, FORCE_INCLUDE_SELECTORS);
+    const mainContent = findMainContent(document);
+    if (mainContent) {
+      const body = document.body;
+      if (body) {
+        const clone = mainContent.cloneNode(true);
+        body.innerHTML = "";
+        body.appendChild(clone);
+      }
+    }
+  }
+  if (includeTags && includeTags.length > 0) {
+    const matchedElements = [];
+    for (const selector of includeTags) {
       try {
-        document.querySelectorAll(selector).forEach((el) => el.remove());
+        document.querySelectorAll(selector).forEach((el) => {
+          matchedElements.push(el.cloneNode(true));
+        });
       } catch {
       }
     }
+    if (matchedElements.length > 0) {
+      const body = document.body;
+      if (body) {
+        body.innerHTML = "";
+        matchedElements.forEach((el) => body.appendChild(el));
+      }
+    }
   }
   if (removeBase64Images) {
     removeBase64ImagesFromDocument(document);
@@ -828,7 +551,10 @@ function removeBase64ImagesFromDocument(document) {
   document.querySelectorAll("[style*='data:image']").forEach((el) => {
     const style = el.getAttribute("style");
     if (style) {
-      const cleanedStyle = style.replace(/background(-image)?:\s*url\([^)]*data:image[^)]*\)[^;]*;?/gi, "");
+      const cleanedStyle = style.replace(
+        /background(-image)?:\s*url\([^)]*data:image[^)]*\)[^;]*;?/gi,
+        ""
+      );
       if (cleanedStyle.trim()) {
         el.setAttribute("style", cleanedStyle);
       } else {
@@ -865,7 +591,7 @@ function cleanContent(html, baseUrl, options = {}) {
 }
 
 // src/utils/metadata-extractor.ts
-import { parseHTML as parseHTML3 } from "linkedom";
+import { parseHTML as parseHTML2 } from "linkedom";
 
 // src/utils/url-helpers.ts
 import { URL as URL2 } from "url";
@@ -938,8 +664,26 @@ function isSameDomain(url, baseUrl) {
 function getUrlKey(url) {
   try {
     const parsedUrl = new URL2(url);
+    parsedUrl.hash = "";
     parsedUrl.search = "";
-    return parsedUrl.toString().toLowerCase();
+    if (parsedUrl.hostname.startsWith("www.")) {
+      parsedUrl.hostname = parsedUrl.hostname.slice(4);
+    }
+    if (parsedUrl.protocol === "http:" && parsedUrl.port === "80" || parsedUrl.protocol === "https:" && parsedUrl.port === "443") {
+      parsedUrl.port = "";
+    }
+    const indexFiles = ["index.html", "index.htm", "default.html", "default.htm", "index.php"];
+    for (const indexFile of indexFiles) {
+      if (parsedUrl.pathname.endsWith(`/${indexFile}`)) {
+        parsedUrl.pathname = parsedUrl.pathname.slice(0, -indexFile.length);
+        break;
+      }
+    }
+    let normalized = parsedUrl.toString().toLowerCase();
+    if (normalized.endsWith("/") && parsedUrl.pathname !== "/") {
+      normalized = normalized.slice(0, -1);
+    }
+    return normalized;
   } catch {
     return url.toLowerCase();
   }
@@ -1006,7 +750,7 @@ function extractMetadata(html, baseUrl) {
   return extractWebsiteMetadata(html, baseUrl);
 }
 function extractWebsiteMetadata(html, baseUrl) {
-  const { document } = parseHTML3(html);
+  const { document } = parseHTML2(html);
   const metadata = {
     title: null,
     description: null,
@@ -1161,11 +905,20 @@ function extractTwitterCard(document) {
 
 // src/utils/logger.ts
 import pino from "pino";
+function hasPinoPretty() {
+  try {
+    __require.resolve("pino-pretty");
+    return true;
+  } catch {
+    return false;
+  }
+}
 function createLogger(name = "reader", level = process.env.LOG_LEVEL || "info") {
+  const usePretty = process.env.NODE_ENV !== "production" && hasPinoPretty();
   return pino({
     name,
     level,
-    transport: process.env.NODE_ENV !== "production" ? {
+    transport: usePretty ? {
       target: "pino-pretty",
       options: {
         colorize: true,
@@ -1279,13 +1032,15 @@ function isUrlAllowed(url, rules) {
 var DEFAULT_OPTIONS = {
   urls: [],
   formats: ["markdown"],
-  includeMetadata: true,
   timeoutMs: 3e4,
   includePatterns: [],
   excludePatterns: [],
   // Content cleaning defaults
   removeAds: true,
   removeBase64Images: true,
+  onlyMainContent: true,
+  includeTags: [],
+  excludeTags: [],
   skipTLSVerification: true,
   // Batch defaults
   batchConcurrency: 1,
@@ -1442,14 +1197,9 @@ var Scraper = class {
         } catch {
         }
         await hero.waitForPaintingStable();
-        let hadChallenge = false;
-        let challengeType = "none";
-        let waitTimeMs = 0;
         const initialUrl = await hero.url;
         const detection = await detectChallenge(hero);
         if (detection.isChallenge) {
-          hadChallenge = true;
-          challengeType = detection.type;
           if (this.options.verbose) {
             this.logger.info(`Challenge detected on ${url}: ${detection.type}`);
           }
@@ -1459,12 +1209,11 @@ var Scraper = class {
             verbose: this.options.verbose,
             initialUrl
           });
-          waitTimeMs = result2.waitedMs;
           if (!result2.resolved) {
             throw new Error(`Challenge not resolved: ${detection.type}`);
           }
           if (this.options.verbose) {
-            this.logger.info(`Challenge resolved via ${result2.method} in ${waitTimeMs}ms`);
+            this.logger.info(`Challenge resolved via ${result2.method} in ${result2.waitedMs}ms`);
           }
         }
         await this.waitForFinalPage(hero, url, this.options.verbose);
@@ -1477,45 +1226,18 @@ var Scraper = class {
             this.logger.warn(`Selector not found: ${this.options.waitForSelector}`);
           }
         }
-        const pageTitle = await hero.document.title;
         const html = await hero.document.documentElement.outerHTML;
         const cleanedHtml = cleanContent(html, url, {
           removeAds: this.options.removeAds,
-          removeBase64Images: this.options.removeBase64Images
+          removeBase64Images: this.options.removeBase64Images,
+          onlyMainContent: this.options.onlyMainContent,
+          includeTags: this.options.includeTags,
+          excludeTags: this.options.excludeTags
         });
         const websiteMetadata = extractMetadata(cleanedHtml, url);
         const duration = Date.now() - startTime;
-        const scrapedAt = (/* @__PURE__ */ new Date()).toISOString();
-        const page = {
-          url,
-          title: pageTitle,
-          markdown: "",
-          // Will be set by formatter
-          html: cleanedHtml,
-          fetchedAt: scrapedAt,
-          depth: 0,
-          hadChallenge,
-          challengeType,
-          waitTimeMs
-        };
-        const markdown = this.options.formats.includes("markdown") ? formatToMarkdown(
-          [page],
-          url,
-          scrapedAt,
-          duration,
-          websiteMetadata,
-          this.options.includeMetadata
-        ) : void 0;
-        const htmlOutput = this.options.formats.includes("html") ? formatToHTML([page], url, scrapedAt, duration, websiteMetadata) : void 0;
-        const json = this.options.formats.includes("json") ? formatToJson([page], url, scrapedAt, duration, websiteMetadata) : void 0;
-        const text = this.options.formats.includes("text") ? formatToText(
-          [page],
-          url,
-          scrapedAt,
-          duration,
-          websiteMetadata,
-          this.options.includeMetadata
-        ) : void 0;
+        const markdown = this.options.formats.includes("markdown") ? htmlToMarkdown(cleanedHtml) : void 0;
+        const htmlOutput = this.options.formats.includes("html") ? cleanedHtml : void 0;
         if (this.options.onProgress) {
           this.options.onProgress({
             completed: index + 1,
@@ -1547,8 +1269,6 @@ var Scraper = class {
         const result = {
           markdown,
           html: htmlOutput,
-          json,
-          text,
           metadata: {
             baseUrl: url,
             totalPages: 1,
@@ -1603,7 +1323,7 @@ async function scrape(options) {
 }
 
 // src/crawler.ts
-import { parseHTML as parseHTML4 } from "linkedom";
+import { parseHTML as parseHTML3 } from "linkedom";
 
 // src/utils/rate-limiter.ts
 import pLimit2 from "p-limit";
@@ -1752,12 +1472,26 @@ var Crawler = class {
    */
   extractLinks(html, baseUrl, depth) {
     const links = [];
-    const { document } = parseHTML4(html);
+    const { document } = parseHTML3(html);
     document.querySelectorAll("a[href]").forEach((anchor) => {
-      const href = anchor.getAttribute("href");
+      const rawHref = anchor.getAttribute("href");
+      if (!rawHref) return;
+      const href = rawHref.trim();
       if (!href) return;
-      const resolved = resolveUrl(href, baseUrl);
+      if (href.startsWith("#")) return;
+      const lowerHref = href.toLowerCase();
+      if (lowerHref.startsWith("javascript:") || lowerHref.startsWith("mailto:") || lowerHref.startsWith("tel:") || lowerHref.startsWith("data:") || lowerHref.startsWith("blob:") || lowerHref.startsWith("ftp:")) {
+        return;
+      }
+      let resolved = resolveUrl(href, baseUrl);
       if (!resolved || !isValidUrl(resolved)) return;
+      try {
+        const parsed = new URL(resolved);
+        parsed.hash = "";
+        resolved = parsed.toString();
+      } catch {
+        return;
+      }
       if (!isSameDomain(resolved, this.options.url)) return;
       if (!isContentUrl(resolved)) return;
       if (!shouldIncludeUrl(resolved, this.options.includePatterns, this.options.excludePatterns)) return;
@@ -2868,9 +2602,9 @@ program.command("status").description("Check daemon status").option("-p, --port
 });
 program.command("scrape <urls...>").description("Scrape one or more URLs").option(
   "-f, --format <formats>",
-  "Output formats (comma-separated: markdown,html,json,text)",
+  "Content formats to include (comma-separated: markdown,html)",
   "markdown"
-).option("-o, --output <file>", "Output file (stdout if omitted)").option("-c, --concurrency <n>", "Parallel requests", "1").option("-t, --timeout <ms>", "Request timeout in milliseconds", "30000").option("--proxy <url>", "Proxy URL (e.g., http://user:pass@host:port)").option("--user-agent <string>", "Custom user agent string").option("--batch-timeout <ms>", "Total timeout for entire batch operation", "300000").option("--show-chrome", "Show browser window for debugging").option("--no-metadata", "Exclude metadata from output").option("--standalone", "Force standalone mode (bypass daemon)").option("-p, --port <n>", `Daemon port (default: ${DEFAULT_DAEMON_PORT})`, String(DEFAULT_DAEMON_PORT)).option("-v, --verbose", "Enable verbose logging").action(async (urls, options) => {
+).option("-o, --output <file>", "Output file (stdout if omitted)").option("-c, --concurrency <n>", "Parallel requests", "1").option("-t, --timeout <ms>", "Request timeout in milliseconds", "30000").option("--proxy <url>", "Proxy URL (e.g., http://user:pass@host:port)").option("--user-agent <string>", "Custom user agent string").option("--batch-timeout <ms>", "Total timeout for entire batch operation", "300000").option("--show-chrome", "Show browser window for debugging").option("--standalone", "Force standalone mode (bypass daemon)").option("-p, --port <n>", `Daemon port (default: ${DEFAULT_DAEMON_PORT})`, String(DEFAULT_DAEMON_PORT)).option("-v, --verbose", "Enable verbose logging").option("--no-main-content", "Disable main content extraction (include full page)").option("--include-tags <selectors>", "CSS selectors for elements to include (comma-separated)").option("--exclude-tags <selectors>", "CSS selectors for elements to exclude (comma-separated)").action(async (urls, options) => {
   const port = parseInt(options.port, 10);
   const useStandalone = options.standalone || false;
   let useDaemon = false;
@@ -2887,7 +2621,7 @@ program.command("scrape <urls...>").description("Scrape one or more URLs").optio
   }) : null;
   try {
     const formats = options.format.split(",").map((f) => f.trim());
-    const validFormats = ["markdown", "html", "json", "text"];
+    const validFormats = ["markdown", "html"];
     for (const format of formats) {
       if (!validFormats.includes(format)) {
         console.error(
@@ -2900,6 +2634,8 @@ program.command("scrape <urls...>").description("Scrape one or more URLs").optio
       console.error(`Scraping ${urls.length} URL(s)...`);
       console.error(`Formats: ${formats.join(", ")}`);
     }
+    const includeTags = options.includeTags ? options.includeTags.split(",").map((s) => s.trim()) : void 0;
+    const excludeTags = options.excludeTags ? options.excludeTags.split(",").map((s) => s.trim()) : void 0;
     const scrapeOptions = {
       urls,
       formats,
@@ -2908,33 +2644,26 @@ program.command("scrape <urls...>").description("Scrape one or more URLs").optio
       batchTimeoutMs: parseInt(options.batchTimeout, 10),
       proxy: options.proxy ? { url: options.proxy } : void 0,
       userAgent: options.userAgent,
-      includeMetadata: options.metadata !== false,
       verbose: options.verbose || false,
       showChrome: options.showChrome || false,
+      // Content cleaning options
+      onlyMainContent: options.mainContent !== false,
+      // --no-main-content sets this to false
+      includeTags,
+      excludeTags,
       onProgress: options.verbose ? ({ completed, total, currentUrl }) => {
         console.error(`[${completed}/${total}] ${currentUrl}`);
       } : void 0
     };
     const result = useDaemon ? await daemonClient.scrape(scrapeOptions) : await standaloneClient.scrape(scrapeOptions);
-    let output = "";
-    for (const site of result.data) {
-      if (formats.includes("markdown") && site.markdown) {
-        output += site.markdown + "\n\n";
-      } else if (formats.includes("text") && site.text) {
-        output += site.text + "\n\n";
-      } else if (formats.includes("html") && site.html) {
-        output += site.html + "\n\n";
-      } else if (formats.includes("json") && site.json) {
-        output += site.json + "\n\n";
-      }
-    }
+    const output = JSON.stringify(result, null, 2);
     if (options.output) {
-      writeFileSync(options.output, output.trim());
+      writeFileSync(options.output, output);
       if (options.verbose) {
         console.error(`Output written to ${options.output}`);
       }
     } else {
-      console.log(output.trim());
+      console.log(output);
     }
     if (options.verbose) {
       console.error(`
@@ -2957,7 +2686,7 @@ Summary:`);
     }
   }
 });
-program.command("crawl <url>").description("Crawl a website to discover and optionally scrape pages").option("-d, --depth <n>", "Maximum crawl depth", "1").option("-m, --max-pages <n>", "Maximum pages to discover", "20").option("-s, --scrape", "Also scrape content of discovered pages").option("-f, --format <formats>", "Output formats when scraping (comma-separated)", "markdown").option("-o, --output <file>", "Output file (stdout if omitted)").option("--delay <ms>", "Delay between requests in milliseconds", "1000").option("-t, --timeout <ms>", "Total timeout for crawl operation in milliseconds").option("--include <patterns>", "URL patterns to include (comma-separated regex)").option("--exclude <patterns>", "URL patterns to exclude (comma-separated regex)").option("--proxy <url>", "Proxy URL (e.g., http://user:pass@host:port)").option("--user-agent <string>", "Custom user agent string").option("--show-chrome", "Show browser window for debugging").option("--standalone", "Force standalone mode (bypass daemon)").option("-p, --port <n>", `Daemon port (default: ${DEFAULT_DAEMON_PORT})`, String(DEFAULT_DAEMON_PORT)).option("-v, --verbose", "Enable verbose logging").action(async (url, options) => {
+program.command("crawl <url>").description("Crawl a website to discover and optionally scrape pages").option("-d, --depth <n>", "Maximum crawl depth", "1").option("-m, --max-pages <n>", "Maximum pages to discover", "20").option("-s, --scrape", "Also scrape content of discovered pages").option("-f, --format <formats>", "Content formats when scraping (comma-separated: markdown,html)", "markdown").option("-o, --output <file>", "Output file (stdout if omitted)").option("--delay <ms>", "Delay between requests in milliseconds", "1000").option("-t, --timeout <ms>", "Total timeout for crawl operation in milliseconds").option("--include <patterns>", "URL patterns to include (comma-separated regex)").option("--exclude <patterns>", "URL patterns to exclude (comma-separated regex)").option("--proxy <url>", "Proxy URL (e.g., http://user:pass@host:port)").option("--user-agent <string>", "Custom user agent string").option("--show-chrome", "Show browser window for debugging").option("--standalone", "Force standalone mode (bypass daemon)").option("-p, --port <n>", `Daemon port (default: ${DEFAULT_DAEMON_PORT})`, String(DEFAULT_DAEMON_PORT)).option("-v, --verbose", "Enable verbose logging").action(async (url, options) => {
   const port = parseInt(options.port, 10);
   const useStandalone = options.standalone || false;
   let useDaemon = false;
@@ -2993,38 +2722,20 @@ program.command("crawl <url>").description("Crawl a website to discover and opti
       verbose: options.verbose || false,
       showChrome: options.showChrome || false
     };
-    const result = useDaemon ? await daemonClient.crawl(crawlOptions) : await standaloneClient.crawl(crawlOptions);
-    let output = "";
-    if (options.scrape && result.scraped) {
-      const formats = options.format.split(",").map((f) => f.trim());
-      for (const site of result.scraped.data) {
-        if (formats.includes("markdown") && site.markdown) {
-          output += site.markdown + "\n\n";
-        } else if (formats.includes("text") && site.text) {
-          output += site.text + "\n\n";
-        } else if (formats.includes("html") && site.html) {
-          output += site.html + "\n\n";
-        } else if (formats.includes("json") && site.json) {
-          output += site.json + "\n\n";
-        }
-      }
-    } else {
-      output = JSON.stringify(
-        {
-          urls: result.urls,
-          metadata: result.metadata
-        },
-        null,
-        2
-      );
-    }
+    const formats = options.format.split(",").map((f) => f.trim());
+    const crawlOptionsWithFormats = {
+      ...crawlOptions,
+      formats
+    };
+    const result = useDaemon ? await daemonClient.crawl(crawlOptionsWithFormats) : await standaloneClient.crawl(crawlOptionsWithFormats);
+    const output = JSON.stringify(result, null, 2);
     if (options.output) {
-      writeFileSync(options.output, output.trim());
+      writeFileSync(options.output, output);
       if (options.verbose) {
         console.error(`Output written to ${options.output}`);
       }
     } else {
-      console.log(output.trim());
+      console.log(output);
     }
     if (options.verbose) {
       console.error(`