@robot-resources/scraper 0.1.3 → 0.2.1

package/dist/mcp-entry.js

@@ -0,0 +1,1040 @@
+#!/usr/bin/env node
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { z } from 'zod';
+import { Readability } from '@mozilla/readability';
+import { parseHTML } from 'linkedom';
+import TurndownService from 'turndown';
+import robotsParser from 'robots-parser';
+
+// src/fetch.ts
+var USER_AGENTS = [
+  "Mozilla/5.0 (compatible; ScraperBot/1.0; +https://scraper.robotresources.ai)",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+];
+var DEFAULT_TIMEOUT = 1e4;
+var DEFAULT_MAX_RETRIES = 3;
+var BASE_BACKOFF_MS = 1e3;
+var FetchError = class extends Error {
+  constructor(message, statusCode, retryable = false) {
+    super(message);
+    this.statusCode = statusCode;
+    this.retryable = retryable;
+    this.name = "FetchError";
+  }
+};
+function isValidUrl(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+function getRandomUserAgent() {
+  return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
+}
+function buildHeaders(userAgent) {
+  return {
+    "User-Agent": userAgent || getRandomUserAgent(),
+    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate",
+    Connection: "keep-alive",
+    "Cache-Control": "no-cache"
+  };
+}
+function headersToObject(headers) {
+  const result = {};
+  headers.forEach((value, key) => {
+    result[key.toLowerCase()] = value;
+  });
+  return result;
+}
+function isRetryableStatus(status) {
+  return status >= 500 && status < 600;
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function getBackoffDelay(attempt) {
+  return BASE_BACKOFF_MS * Math.pow(2, attempt);
+}
+async function fetchWithTimeout(url, headers, timeout) {
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  try {
+    const response = await fetch(url, {
+      method: "GET",
+      headers,
+      signal: controller.signal,
+      redirect: "follow"
+    });
+    return response;
+  } catch (error) {
+    if (error instanceof Error && error.name === "AbortError") {
+      throw new FetchError("Request timeout", void 0, true);
+    }
+    throw error;
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+async function fetchUrl(url, options = {}) {
+  const {
+    timeout = DEFAULT_TIMEOUT,
+    maxRetries = DEFAULT_MAX_RETRIES,
+    userAgent
+  } = options;
+  if (!isValidUrl(url)) {
+    throw new FetchError("Invalid URL", void 0, false);
+  }
+  const headers = buildHeaders(userAgent);
+  let lastError = null;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const response = await fetchWithTimeout(url, headers, timeout);
+      if (!response.ok) {
+        const statusCode = response.status;
+        if (statusCode >= 400 && statusCode < 500) {
+          throw new FetchError(`HTTP ${statusCode}`, statusCode, false);
+        }
+        if (isRetryableStatus(statusCode)) {
+          throw new FetchError(`HTTP ${statusCode}`, statusCode, true);
+        }
+      }
+      const html = await response.text();
+      const responseHeaders = headersToObject(response.headers);
+      return {
+        html,
+        url: response.url,
+        statusCode: response.status,
+        headers: responseHeaders
+      };
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      const isRetryable = error instanceof FetchError ? error.retryable : !(error instanceof FetchError);
+      const hasRetriesLeft = attempt < maxRetries;
+      if (isRetryable && hasRetriesLeft) {
+        const delay = getBackoffDelay(attempt);
+        await sleep(delay);
+        continue;
+      }
+      break;
+    }
+  }
+  throw lastError || new FetchError("Unknown fetch error");
+}
+
+// src/fetch-stealth.ts
+var DEFAULT_TIMEOUT2 = 1e4;
+var DEFAULT_MAX_RETRIES2 = 2;
+var BASE_BACKOFF_MS2 = 1e3;
+function isValidUrl2(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+function isRetryableStatus2(status) {
+  return status >= 500 && status < 600;
+}
+function sleep2(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function getBackoffDelay2(attempt) {
+  return BASE_BACKOFF_MS2 * Math.pow(2, attempt);
+}
+async function fetchStealth(url, options = {}) {
+  const {
+    timeout = DEFAULT_TIMEOUT2,
+    maxRetries = DEFAULT_MAX_RETRIES2
+  } = options;
+  if (!isValidUrl2(url)) {
+    throw new FetchError("Invalid URL", void 0, false);
+  }
+  let Impit;
+  try {
+    ({ Impit } = await import('impit'));
+  } catch {
+    throw new FetchError(
+      "impit is required for stealth mode. Install: npm install impit (requires Node >= 20)",
+      void 0,
+      false
+    );
+  }
+  const client = new Impit({ browser: "chrome" });
+  let lastError = null;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const response = await client.fetch(url, {
+        signal: AbortSignal.timeout(timeout)
+      });
+      if (!response.ok) {
+        const statusCode = response.status;
+        if (statusCode >= 400 && statusCode < 500) {
+          throw new FetchError(`HTTP ${statusCode}`, statusCode, false);
+        }
+        if (isRetryableStatus2(statusCode)) {
+          throw new FetchError(`HTTP ${statusCode}`, statusCode, true);
+        }
+      }
+      const html = await response.text();
+      const headers = {};
+      response.headers.forEach((value, key) => {
+        headers[key] = value;
+      });
+      return {
+        html,
+        url: response.url ?? url,
+        statusCode: response.status,
+        headers
+      };
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      const isRetryable = error instanceof FetchError ? error.retryable : !(error instanceof FetchError);
+      const hasRetriesLeft = attempt < maxRetries;
+      if (isRetryable && hasRetriesLeft) {
+        const delay = getBackoffDelay2(attempt);
+        await sleep2(delay);
+        continue;
+      }
+      break;
+    }
+  }
+  throw lastError || new FetchError("Unknown stealth fetch error");
+}
+
+// src/fetch-render.ts
+var DEFAULT_TIMEOUT3 = 3e4;
+function isValidUrl3(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+async function fetchRender(url, options = {}) {
+  const { timeout = DEFAULT_TIMEOUT3 } = options;
+  if (!isValidUrl3(url)) {
+    throw new FetchError("Invalid URL", void 0, false);
+  }
+  let chromium;
+  try {
+    ({ chromium } = await import('playwright'));
+  } catch {
+    throw new FetchError(
+      "Playwright is required for render mode. Install: npm install playwright",
+      void 0,
+      false
+    );
+  }
+  const browser = await chromium.launch({ headless: true });
+  try {
+    const page = await browser.newPage();
+    page.on("dialog", (dialog) => dialog.dismiss());
+    const response = await page.goto(url, {
+      waitUntil: "networkidle",
+      timeout
+    });
+    if (!response) {
+      throw new FetchError(
+        "Navigation returned no response (about:blank or same-URL redirect)",
+        void 0,
+        false
+      );
+    }
+    const statusCode = response.status();
+    if (statusCode >= 400 && statusCode < 500) {
+      throw new FetchError(`HTTP ${statusCode}`, statusCode, false);
+    }
+    if (statusCode >= 500) {
+      throw new FetchError(`HTTP ${statusCode}`, statusCode, true);
+    }
+    const html = await page.content();
+    const headers = response.headers();
+    return {
+      html,
+      url: page.url(),
+      statusCode,
+      headers
+    };
+  } catch (error) {
+    if (error instanceof Error && error.name === "TimeoutError") {
+      throw new FetchError("Navigation timeout", void 0, true);
+    }
+    throw error;
+  } finally {
+    await browser.close();
+  }
+}
+
+// src/fetch-mode.ts
+var CHALLENGE_MARKERS = [
+  "cf-browser-verification",
+  "Just a moment",
+  "_cf_chl_opt",
+  "akamai-challenge",
+  "ak-challenge"
+];
+function isChallengeResponse(fetchResult) {
+  return CHALLENGE_MARKERS.some((marker) => fetchResult.html.includes(marker));
+}
+var VALID_MODES = ["fast", "stealth", "render", "auto"];
+async function fetchWithMode(url, mode, options) {
+  if (!VALID_MODES.includes(mode)) {
+    throw new FetchError(
+      `Invalid fetch mode: '${mode}'. Valid modes: ${VALID_MODES.join(", ")}`,
+      void 0,
+      false
+    );
+  }
+  if (mode === "stealth") return fetchStealth(url, options);
+  if (mode === "render") return fetchRender(url, options);
+  if (mode === "fast") return fetchUrl(url, options);
+  try {
+    const result = await fetchUrl(url, options);
+    if (isChallengeResponse(result)) {
+      return fetchStealth(url, options);
+    }
+    return result;
+  } catch (err) {
+    if (err instanceof FetchError) {
+      if (err.statusCode === 403) {
+        return fetchStealth(url, options);
+      }
+      throw err;
+    }
+    return fetchStealth(url, options);
+  }
+}
+var ExtractionError = class extends Error {
+  constructor(message, code) {
+    super(message);
+    this.code = code;
+    this.name = "ExtractionError";
+  }
+};
+async function extractContent(fetchResult) {
+  const { html } = fetchResult;
+  if (!html || !html.trim()) {
+    throw new ExtractionError("Empty HTML content", "EMPTY_HTML");
+  }
+  const { document } = parseHTML(html);
+  const reader = new Readability(document, {
+    charThreshold: 50
+  });
+  const article = reader.parse();
+  if (!article || !article.content || article.content.trim().length < 20) {
+    throw new ExtractionError(
+      "No content could be extracted from the page",
+      "NO_CONTENT"
+    );
+  }
+  const result = {
+    content: cleanContent(article.content),
+    title: article.title || extractFallbackTitle(document),
+    author: article.byline || void 0,
+    publishedAt: article.publishedTime || extractPublishedTime(document),
+    siteName: article.siteName || extractSiteName(document)
+  };
+  return result;
+}
+function cleanContent(content) {
+  return content.replace(/>\s+</g, "><").replace(/\s{2,}/g, " ").trim();
+}
+function extractFallbackTitle(document) {
+  const ogTitle = document.querySelector('meta[property="og:title"]');
+  if (ogTitle) {
+    const content = ogTitle.getAttribute("content");
+    if (content) return content;
+  }
+  const titleEl = document.querySelector("title");
+  if (titleEl && titleEl.textContent) {
+    return titleEl.textContent.trim();
+  }
+  const h1 = document.querySelector("h1");
+  if (h1 && h1.textContent) {
+    return h1.textContent.trim();
+  }
+  return void 0;
+}
+function extractPublishedTime(document) {
+  const ogTime = document.querySelector(
+    'meta[property="article:published_time"]'
+  );
+  if (ogTime) {
+    const content = ogTime.getAttribute("content");
+    if (content) return content;
+  }
+  const schemaTime = document.querySelector('[itemprop="datePublished"]');
+  if (schemaTime) {
+    const datetime = schemaTime.getAttribute("datetime");
+    if (datetime) return datetime;
+    const content = schemaTime.getAttribute("content");
+    if (content) return content;
+  }
+  const timeEl = document.querySelector("time[datetime]");
+  if (timeEl) {
+    const datetime = timeEl.getAttribute("datetime");
+    if (datetime) return datetime;
+  }
+  return void 0;
+}
+function extractSiteName(document) {
+  const ogSiteName = document.querySelector('meta[property="og:site_name"]');
+  if (ogSiteName) {
+    const content = ogSiteName.getAttribute("content");
+    if (content) return content;
+  }
+  const appName = document.querySelector('meta[name="application-name"]');
+  if (appName) {
+    const content = appName.getAttribute("content");
+    if (content) return content;
+  }
+  return void 0;
+}
+function createTurndownService() {
+  const turndown = new TurndownService({
+    headingStyle: "atx",
+    hr: "---",
+    bulletListMarker: "-",
+    codeBlockStyle: "fenced",
+    fence: "```",
+    emDelimiter: "*",
+    strongDelimiter: "**",
+    linkStyle: "inlined"
+  });
+  turndown.remove(["script", "style", "noscript", "iframe"]);
+  turndown.addRule("removeEmpty", {
+    filter: (node) => {
+      if (node.nodeType === 1) {
+        const text = node.textContent || "";
+        const isEmptyBlock = text.trim() === "" && !["IMG", "BR", "HR", "INPUT"].includes(node.nodeName);
+        return isEmptyBlock;
+      }
+      return false;
+    },
+    replacement: () => ""
+  });
+  turndown.addRule("fencedCodeBlock", {
+    filter: (node, options) => {
+      return options.codeBlockStyle === "fenced" && node.nodeName === "PRE" && node.firstChild !== null && node.firstChild.nodeName === "CODE";
+    },
+    replacement: (_content, node, options) => {
+      const codeNode = node.firstChild;
+      const code = codeNode.textContent || "";
+      const className = codeNode.getAttribute("class") || "";
+      const langMatch = className.match(/language-(\w+)/);
+      const lang = langMatch ? langMatch[1] : "";
+      const fence = options.fence || "```";
+      return `
+
+${fence}${lang}
+${code}
+${fence}
+
+`;
+    }
+  });
+  turndown.addRule("strikethrough", {
+    filter: ["del", "s"],
+    replacement: (content) => `~~${content}~~`
+  });
+  return turndown;
+}
+var turndownInstance = null;
+function getTurndown() {
+  if (!turndownInstance) {
+    turndownInstance = createTurndownService();
+  }
+  return turndownInstance;
+}
+function cleanMarkdown(markdown) {
+  return markdown.replace(/\n{3,}/g, "\n\n").replace(/[ \t]+$/gm, "").replace(/^\n+/, "").replace(/\n+$/, "").trim();
+}
+async function convertToMarkdown(extractResult) {
+  const { content } = extractResult;
+  if (!content || !content.trim()) {
+    return {
+      markdown: "",
+      tokenCount: 0
+    };
+  }
+  const turndown = getTurndown();
+  let markdown = turndown.turndown(content);
+  markdown = cleanMarkdown(markdown);
+  const tokenCount = estimateTokens(markdown);
+  return {
+    markdown,
+    tokenCount
+  };
+}
+function estimateTokens(text) {
+  if (!text) return 0;
+  let tokens = 0;
+  let remaining = text;
+  remaining = remaining.replace(/```[\s\S]*?```/g, (match) => {
+    tokens += Math.ceil(match.length / 3.2);
+    return " ";
+  });
+  remaining = remaining.replace(/`[^`]+`/g, (match) => {
+    tokens += Math.ceil(match.length / 3.5);
+    return " ";
+  });
+  remaining = remaining.replace(/https?:\/\/\S+/g, (match) => {
+    tokens += Math.ceil(match.length / 5);
+    return " ";
+  });
+  const proseLength = remaining.replace(/\s+/g, " ").trim().length;
+  if (proseLength > 0) {
+    tokens += Math.ceil(proseLength / 4.3);
+  }
+  return Math.max(1, tokens);
+}
+var DEFAULT_TTL_MS = 60 * 60 * 1e3;
+var DEFAULT_TIMEOUT_MS = 5e3;
+var BOT_USER_AGENT = "ScraperBot";
+var cache = /* @__PURE__ */ new Map();
+function getRobotsUrl(url) {
+  const parsed = new URL(url);
+  return `${parsed.protocol}//${parsed.host}/robots.txt`;
+}
+async function getRobotsParser(url, timeout = DEFAULT_TIMEOUT_MS) {
+  const robotsUrl = getRobotsUrl(url);
+  const cached = cache.get(robotsUrl);
+  if (cached && cached.expiresAt > Date.now()) {
+    return cached.parser;
+  }
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    const response = await fetch(robotsUrl, {
+      signal: controller.signal,
+      headers: { "User-Agent": BOT_USER_AGENT }
+    });
+    clearTimeout(timeoutId);
+    const text = response.ok ? await response.text() : "";
+    const parser = robotsParser(robotsUrl, text);
+    cache.set(robotsUrl, {
+      parser,
+      expiresAt: Date.now() + DEFAULT_TTL_MS
+    });
+    return parser;
+  } catch {
+    const parser = robotsParser(robotsUrl, "");
+    cache.set(robotsUrl, {
+      parser,
+      expiresAt: Date.now() + DEFAULT_TTL_MS
+    });
+    return parser;
+  }
+}
+async function isAllowedByRobots(url, timeout) {
+  const parser = await getRobotsParser(url, timeout);
+  return parser.isAllowed(url, BOT_USER_AGENT) !== false;
+}
+async function getCrawlDelay(url, timeout) {
+  const parser = await getRobotsParser(url, timeout);
+  const delay = parser.getCrawlDelay(BOT_USER_AGENT);
+  return delay === void 0 ? null : delay;
+}
+
+// src/sitemap.ts
+var DEFAULT_TIMEOUT_MS2 = 1e4;
+var DEFAULT_TTL_MS2 = 60 * 60 * 1e3;
+var MAX_RECURSION_DEPTH = 2;
+var cache2 = /* @__PURE__ */ new Map();
+function getOrigin(url) {
+  const parsed = new URL(url);
+  return `${parsed.protocol}//${parsed.host}`;
+}
+function isSitemapIndex(xml) {
+  return /<sitemapindex[\s>]/i.test(xml);
+}
+function extractSitemapIndexUrls(xml) {
+  const urls = [];
+  const sitemapBlockRegex = /<(?:\w+:)?sitemap\b[^>]*>([\s\S]*?)<\/(?:\w+:)?sitemap>/gi;
+  let blockMatch;
+  while ((blockMatch = sitemapBlockRegex.exec(xml)) !== null) {
+    const block = blockMatch[1];
+    const locMatch = /<(?:\w+:)?loc\b[^>]*>([\s\S]*?)<\/(?:\w+:)?loc>/i.exec(block);
+    if (locMatch) {
+      const url = locMatch[1].trim();
+      if (url) urls.push(url);
+    }
+  }
+  return urls;
+}
+function extractUrlEntries(xml, origin) {
+  const entries = [];
+  const urlBlockRegex = /<(?:\w+:)?url\b[^>]*>([\s\S]*?)<\/(?:\w+:)?url>/gi;
+  let blockMatch;
+  while ((blockMatch = urlBlockRegex.exec(xml)) !== null) {
+    const block = blockMatch[1];
+    const locMatch = /<(?:\w+:)?loc\b[^>]*>([\s\S]*?)<\/(?:\w+:)?loc>/i.exec(block);
+    if (!locMatch) continue;
+    const loc = locMatch[1].trim();
+    if (!loc) continue;
+    try {
+      if (getOrigin(loc) !== origin) continue;
+    } catch {
+      continue;
+    }
+    const entry = { loc };
+    const lastmodMatch = /<(?:\w+:)?lastmod\b[^>]*>([\s\S]*?)<\/(?:\w+:)?lastmod>/i.exec(block);
+    if (lastmodMatch) {
+      const lastmod = lastmodMatch[1].trim();
+      if (lastmod) entry.lastmod = lastmod;
+    }
+    const priorityMatch = /<(?:\w+:)?priority\b[^>]*>([\s\S]*?)<\/(?:\w+:)?priority>/i.exec(block);
+    if (priorityMatch) {
+      const priority = parseFloat(priorityMatch[1].trim());
+      if (!isNaN(priority)) entry.priority = priority;
+    }
+    entries.push(entry);
+  }
+  return entries;
+}
+async function fetchSitemapXml(url, timeout) {
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: { "User-Agent": "ScraperBot/1.0" }
+    });
+    clearTimeout(timeoutId);
+    if (!response.ok) return null;
+    return await response.text();
+  } catch {
+    return null;
+  }
+}
+async function parseSitemapInternal(url, origin, timeout, depth) {
+  if (depth >= MAX_RECURSION_DEPTH) return [];
+  const xml = await fetchSitemapXml(url, timeout);
+  if (!xml) return [];
+  if (isSitemapIndex(xml)) {
+    const sitemapUrls = extractSitemapIndexUrls(xml);
+    const allEntries = [];
+    for (const sitemapUrl of sitemapUrls) {
+      const entries = await parseSitemapInternal(
+        sitemapUrl,
+        origin,
+        timeout,
+        depth + 1
+      );
+      allEntries.push(...entries);
+    }
+    return allEntries;
+  }
+  return extractUrlEntries(xml, origin);
+}
+async function parseSitemap(url, timeout = DEFAULT_TIMEOUT_MS2) {
+  const cached = cache2.get(url);
+  if (cached && cached.expiresAt > Date.now()) {
+    return cached.entries;
+  }
+  let origin;
+  try {
+    origin = getOrigin(url);
+  } catch {
+    return [];
+  }
+  const entries = await parseSitemapInternal(url, origin, timeout, 0);
+  cache2.set(url, {
+    entries,
+    expiresAt: Date.now() + DEFAULT_TTL_MS2
+  });
+  return entries;
+}
+
+// src/crawl.ts
+function isValidUrl4(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+function normalizeUrl(url) {
+  try {
+    const parsed = new URL(url);
+    parsed.hash = "";
+    let pathname = parsed.pathname;
+    if (pathname.length > 1 && pathname.endsWith("/")) {
+      pathname = pathname.slice(0, -1);
+    }
+    parsed.pathname = pathname;
+    return parsed.toString();
+  } catch {
+    return url;
+  }
+}
+var SKIP_EXTENSIONS = /* @__PURE__ */ new Set([
+  ".pdf",
+  ".jpg",
+  ".jpeg",
+  ".png",
+  ".gif",
+  ".svg",
+  ".webp",
+  ".ico",
+  ".mp4",
+  ".mp3",
+  ".wav",
+  ".avi",
+  ".zip",
+  ".tar",
+  ".gz",
+  ".rar",
+  ".css",
+  ".js",
+  ".xml",
+  ".json",
+  ".woff",
+  ".woff2",
+  ".ttf",
+  ".eot"
+]);
+function extractLinks(html, baseUrl) {
+  const links = [];
+  let origin;
+  try {
+    origin = new URL(baseUrl).origin;
+  } catch {
+    return [];
+  }
+  const regex = /<a\s+[^>]*href\s*=\s*["']([^"']+)["'][^>]*>/gi;
+  let match;
+  while ((match = regex.exec(html)) !== null) {
+    const href = match[1].trim();
+    if (href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:") || href.startsWith("#")) continue;
+    try {
+      const resolved = new URL(href, baseUrl);
+      if (resolved.origin !== origin) continue;
+      const ext = resolved.pathname.toLowerCase().match(/\.\w+$/)?.[0];
+      if (ext && SKIP_EXTENSIONS.has(ext)) continue;
+      links.push(normalizeUrl(resolved.toString()));
+    } catch {
+    }
+  }
+  return [...new Set(links)];
+}
+function matchGlob(url, pattern) {
+  const regex = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*\*/g, "{{DOUBLESTAR}}").replace(/\*/g, "[^/]*").replace(/\{\{DOUBLESTAR\}\}/g, ".*");
+  return new RegExp(regex).test(url);
+}
+function matchesFilter(url, include, exclude) {
+  if (exclude?.length) {
+    for (const pattern of exclude) {
+      if (matchGlob(url, pattern)) return false;
+    }
+  }
+  if (include?.length) {
+    return include.some((pattern) => matchGlob(url, pattern));
+  }
+  return true;
+}
+function sleep3(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function crawl(options) {
+  const startTime = Date.now();
+  const {
+    url: startUrl,
+    depth: maxDepth = 2,
+    limit = 50,
+    mode = "auto",
+    include,
+    exclude,
+    timeout,
+    concurrency = 3,
+    respectRobots = true
+  } = options;
+  if (!isValidUrl4(startUrl)) {
+    throw new FetchError("Invalid URL", void 0, false);
+  }
+  if (maxDepth < 0) throw new FetchError("depth must be >= 0", void 0, false);
+  if (limit < 1) throw new FetchError("limit must be >= 1", void 0, false);
+  if (concurrency < 1) throw new FetchError("concurrency must be >= 1", void 0, false);
+  if (timeout !== void 0 && (timeout <= 0 || Number.isNaN(timeout))) {
+    throw new FetchError("timeout must be a positive number", void 0, false);
+  }
+  const pages = [];
+  const errors = [];
+  const visited = /* @__PURE__ */ new Set();
+  let totalDiscovered = 0;
+  let totalSkipped = 0;
+  const normalizedStart = normalizeUrl(startUrl);
+  const origin = new URL(startUrl).origin;
+  let crawlDelay = null;
+  if (respectRobots) {
+    crawlDelay = await getCrawlDelay(startUrl, timeout);
+    const allowed = await isAllowedByRobots(startUrl, timeout);
+    if (!allowed) {
+      return {
+        pages: [],
+        totalDiscovered: 1,
+        totalCrawled: 0,
+        totalSkipped: 1,
+        errors: [],
+        duration: Date.now() - startTime
+      };
+    }
+  }
+  const queue = [
+    { url: normalizedStart, depth: 0 }
+  ];
+  if (maxDepth > 0) {
+    try {
+      const sitemapEntries = await parseSitemap(`${origin}/sitemap.xml`, timeout);
+      const seen = /* @__PURE__ */ new Set([normalizedStart]);
+      for (const entry of sitemapEntries) {
+        const normalized = normalizeUrl(entry.loc);
+        if (!seen.has(normalized)) {
+          seen.add(normalized);
+          queue.push({ url: normalized, depth: 1 });
+        }
+      }
+    } catch (err) {
+      console.debug(`[scraper] Sitemap unavailable for ${origin}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  totalDiscovered = queue.length;
+  while (queue.length > 0 && pages.length < limit) {
+    const batchSize = Math.min(concurrency, limit - pages.length, queue.length);
+    const batch = queue.splice(0, batchSize);
+    const tasks = batch.map(async ({ url, depth }) => {
+      const normalized = normalizeUrl(url);
+      if (visited.has(normalized)) {
+        totalSkipped++;
+        return;
+      }
+      visited.add(normalized);
+      if (normalized !== normalizedStart && !matchesFilter(normalized, include, exclude)) {
+        totalSkipped++;
+        return;
+      }
+      if (respectRobots) {
+        const allowed = await isAllowedByRobots(url, timeout);
+        if (!allowed) {
+          totalSkipped++;
+          return;
+        }
+      }
+      try {
+        const fetchResult = await fetchWithMode(url, mode, { timeout });
+        const extractResult = await extractContent(fetchResult);
+        const convertResult = await convertToMarkdown(extractResult);
+        const pageResult = {
+          markdown: convertResult.markdown,
+          tokenCount: convertResult.tokenCount,
+          title: extractResult.title,
+          author: extractResult.author,
+          siteName: extractResult.siteName,
+          publishedAt: extractResult.publishedAt,
+          url: fetchResult.url,
+          depth
+        };
+        pages.push(pageResult);
+        if (depth < maxDepth) {
+          const links = extractLinks(fetchResult.html, fetchResult.url);
+          for (const link of links) {
+            if (!visited.has(link)) {
+              queue.push({ url: link, depth: depth + 1 });
+              totalDiscovered++;
+            }
+          }
+        }
+      } catch (err) {
+        errors.push({
+          url,
+          error: err instanceof Error ? err.message : String(err),
+          depth
+        });
+      }
+    });
+    await Promise.allSettled(tasks);
+    if (crawlDelay && crawlDelay > 0 && queue.length > 0) {
+      await sleep3(crawlDelay * 1e3);
+    }
+  }
+  return {
+    pages,
+    totalDiscovered,
+    totalCrawled: pages.length,
+    totalSkipped,
+    errors,
+    duration: Date.now() - startTime
+  };
+}
+
+// src/mcp-server.ts
+function createServer() {
+  const server2 = new McpServer({
+    name: "scraper-mcp",
+    version: "0.2.0"
+  });
+  server2.tool(
+    "scraper_compress_url",
+    "Compress web content from a URL for reduced token usage. Returns markdown with 70-90% fewer tokens than raw HTML.",
+    {
+      url: z.string().url().describe("URL to compress"),
+      mode: z.enum(["fast", "stealth", "render", "auto"]).optional().describe("Fetch mode: 'fast' (plain HTTP), 'stealth' (TLS fingerprint), 'render' (headless browser), 'auto' (fast with fallback). Default: 'auto'"),
+      timeout: z.number().positive().optional().describe("Fetch timeout in milliseconds (default: 10000)"),
+      maxRetries: z.number().int().min(0).max(10).optional().describe("Max retry attempts (default: 3)")
+    },
+    async (args) => compressUrl(args)
+  );
+  server2.tool(
+    "scraper_crawl_url",
+    "Crawl multiple pages from a starting URL using BFS link discovery. Returns compressed markdown for each page with 70-90% fewer tokens than raw HTML.",
+    {
+      url: z.string().url().describe("Starting URL to crawl"),
+      maxPages: z.number().int().min(1).max(100).optional().describe("Max pages to crawl (default: 10)"),
+      maxDepth: z.number().int().min(0).max(5).optional().describe("Max link depth (default: 2)"),
+      mode: z.enum(["fast", "stealth", "render", "auto"]).optional().describe("Fetch mode: 'fast' (plain HTTP), 'stealth' (TLS fingerprint), 'render' (headless browser), 'auto' (fast with fallback). Default: 'auto'"),
+      include: z.array(z.string()).optional().describe("URL patterns to include (glob)"),
+      exclude: z.array(z.string()).optional().describe("URL patterns to exclude (glob)"),
+      timeout: z.number().positive().optional().describe("Per-page timeout in milliseconds (default: 10000)")
+    },
+    async (args) => crawlUrl(args)
+  );
+  return server2;
+}
+async function compressUrl({
+  url,
+  mode,
+  timeout,
+  maxRetries
+}) {
+  try {
+    const fetchResult = await fetchWithMode(url, mode ?? "auto", { timeout, maxRetries });
+    const originalTokens = estimateTokens(fetchResult.html);
+    const extractResult = await extractContent(fetchResult);
+    const convertResult = await convertToMarkdown(extractResult);
+    const compressionRatio = originalTokens > 0 ? Math.round((1 - convertResult.tokenCount / originalTokens) * 100) : 0;
+    return {
+      content: [{ type: "text", text: convertResult.markdown }],
+      structuredContent: {
+        markdown: convertResult.markdown,
+        tokenCount: convertResult.tokenCount,
+        title: extractResult.title ?? null,
+        author: extractResult.author ?? null,
+        siteName: extractResult.siteName ?? null,
+        url: fetchResult.url,
+        compressionRatio
+      }
+    };
+  } catch (error) {
+    return formatError(url, error);
+  }
+}
+async function crawlUrl({
+  url,
+  maxPages,
+  maxDepth,
+  mode,
+  include,
+  exclude,
+  timeout
+}) {
+  try {
+    const result = await crawl({
+      url,
+      limit: maxPages ?? 10,
+      depth: maxDepth ?? 2,
+      mode,
+      include,
+      exclude,
+      timeout
+    });
+    const host = new URL(url).host;
+    const errorSuffix = result.errors.length > 0 ? ` (${result.errors.length} error${result.errors.length > 1 ? "s" : ""})` : "";
+    const summary = `Crawled ${result.totalCrawled} pages from ${host}${errorSuffix}`;
+    const content = [
+      { type: "text", text: summary }
+    ];
+    for (const page of result.pages) {
+      const header = page.title ? `## ${page.title}
+
+` : "";
+      content.push({
+        type: "text",
+        text: `${header}${page.markdown}`
+      });
+    }
+    return {
+      content,
+      structuredContent: {
+        pages: result.pages,
+        totalCrawled: result.totalCrawled,
+        totalDiscovered: result.totalDiscovered,
+        totalSkipped: result.totalSkipped,
+        errors: result.errors,
+        duration: result.duration
+      }
+    };
+  } catch (error) {
+    return formatError(url, error);
+  }
+}
+function formatError(url, error) {
+  if (error instanceof FetchError) {
+    let message;
+    if (error.statusCode === 403) {
+      message = `Access denied (HTTP 403) for ${url}. The site may block automated access. Try a different URL or check if authentication is required.`;
+    } else if (error.statusCode === 404) {
+      message = `Page not found (HTTP 404) at ${url}. Verify the URL is correct and the page exists.`;
+    } else if (error.statusCode && error.statusCode >= 500) {
+      message = `Server error (HTTP ${error.statusCode}) from ${url}. The site may be experiencing issues. Try again later.`;
+    } else if (error.message.includes("timeout") || error.message.includes("Timeout")) {
+      message = `Request timed out fetching ${url}. Try increasing the timeout parameter or check if the site is accessible.`;
+    } else if (error.message.includes("Invalid URL")) {
+      message = `Invalid URL: ${url}. Provide a full URL starting with http:// or https://`;
+    } else {
+      message = `Failed to fetch ${url}: ${error.message}${error.retryable ? ". Retries exhausted \u2014 try again later." : ""}`;
+    }
+    return {
+      content: [{ type: "text", text: message }],
+      isError: true
+    };
+  }
+  if (error instanceof ExtractionError) {
+    let message;
+    if (error.code === "EMPTY_HTML") {
+      message = `The page at ${url} returned empty HTML. It may require JavaScript rendering \u2014 try using mode: 'render' (requires Playwright peer dependency) or try a URL that serves static HTML content.`;
+    } else if (error.code === "NO_CONTENT") {
+      message = `Could not extract meaningful content from ${url}. The page may be a login wall, contain only images/video, or rely entirely on JavaScript rendering.`;
+    } else {
+      message = `Content extraction failed for ${url}: ${error.message}`;
+    }
+    return {
+      content: [{ type: "text", text: message }],
+      isError: true
+    };
+  }
+  const msg = error instanceof Error ? error.message : String(error);
+  return {
+    content: [
+      {
+        type: "text",
+        text: `Unexpected error processing ${url}: ${msg}`
+      }
+    ],
+    isError: true
+  };
+}
+
+// src/mcp-entry.ts
+var server = createServer();
+var transport = new StdioServerTransport();
+await server.connect(transport);