@robot-resources/scraper 0.1.0

package/dist/index.js

@@ -0,0 +1,976 @@
+import { Readability } from '@mozilla/readability';
+import { parseHTML } from 'linkedom';
+import TurndownService from 'turndown';
+import robotsParser from 'robots-parser';
+import { readFileSync } from 'fs';
+import { homedir } from 'os';
+import { join } from 'path';
+
+// src/fetch.ts
+var USER_AGENTS = [
+  "Mozilla/5.0 (compatible; ScraperBot/1.0; +https://scraper.robotresources.ai)",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+];
+var DEFAULT_TIMEOUT = 1e4;
+var DEFAULT_MAX_RETRIES = 3;
+var BASE_BACKOFF_MS = 1e3;
+var FetchError = class extends Error {
+  constructor(message, statusCode, retryable = false) {
+    super(message);
+    this.statusCode = statusCode;
+    this.retryable = retryable;
+    this.name = "FetchError";
+  }
+};
+function isValidUrl(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+function getRandomUserAgent() {
+  return USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)];
+}
+function buildHeaders(userAgent) {
+  return {
+    "User-Agent": userAgent || getRandomUserAgent(),
+    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate",
+    Connection: "keep-alive",
+    "Cache-Control": "no-cache"
+  };
+}
+function headersToObject(headers) {
+  const result = {};
+  headers.forEach((value, key) => {
+    result[key.toLowerCase()] = value;
+  });
+  return result;
+}
+function isRetryableStatus(status) {
+  return status >= 500 && status < 600;
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function getBackoffDelay(attempt) {
+  return BASE_BACKOFF_MS * Math.pow(2, attempt);
+}
+async function fetchWithTimeout(url, headers, timeout) {
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  try {
+    const response = await fetch(url, {
+      method: "GET",
+      headers,
+      signal: controller.signal,
+      redirect: "follow"
+    });
+    return response;
+  } catch (error) {
+    if (error instanceof Error && error.name === "AbortError") {
+      throw new FetchError("Request timeout", void 0, true);
+    }
+    throw error;
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+async function fetchUrl(url, options = {}) {
+  const {
+    timeout = DEFAULT_TIMEOUT,
+    maxRetries = DEFAULT_MAX_RETRIES,
+    userAgent
+  } = options;
+  if (!isValidUrl(url)) {
+    throw new FetchError("Invalid URL", void 0, false);
+  }
+  const headers = buildHeaders(userAgent);
+  let lastError = null;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const response = await fetchWithTimeout(url, headers, timeout);
+      if (!response.ok) {
+        const statusCode = response.status;
+        if (statusCode >= 400 && statusCode < 500) {
+          throw new FetchError(`HTTP ${statusCode}`, statusCode, false);
+        }
+        if (isRetryableStatus(statusCode)) {
+          throw new FetchError(`HTTP ${statusCode}`, statusCode, true);
+        }
+      }
+      const html = await response.text();
+      const responseHeaders = headersToObject(response.headers);
+      return {
+        html,
+        url: response.url,
+        statusCode: response.status,
+        headers: responseHeaders
+      };
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      const isRetryable = error instanceof FetchError ? error.retryable : !(error instanceof FetchError);
+      const hasRetriesLeft = attempt < maxRetries;
+      if (isRetryable && hasRetriesLeft) {
+        const delay = getBackoffDelay(attempt);
+        await sleep(delay);
+        continue;
+      }
+      break;
+    }
+  }
+  throw lastError || new FetchError("Unknown fetch error");
+}
+
+// src/fetch-stealth.ts
+var DEFAULT_TIMEOUT2 = 1e4;
+var DEFAULT_MAX_RETRIES2 = 2;
+var BASE_BACKOFF_MS2 = 1e3;
+function isValidUrl2(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+function isRetryableStatus2(status) {
+  return status >= 500 && status < 600;
+}
+function sleep2(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function getBackoffDelay2(attempt) {
+  return BASE_BACKOFF_MS2 * Math.pow(2, attempt);
+}
+async function fetchStealth(url, options = {}) {
+  const {
+    timeout = DEFAULT_TIMEOUT2,
+    maxRetries = DEFAULT_MAX_RETRIES2
+  } = options;
+  if (!isValidUrl2(url)) {
+    throw new FetchError("Invalid URL", void 0, false);
+  }
+  let Impit;
+  try {
+    ({ Impit } = await import('impit'));
+  } catch {
+    throw new FetchError(
+      "impit is required for stealth mode. Install: npm install impit (requires Node >= 20)",
+      void 0,
+      false
+    );
+  }
+  const client = new Impit({ browser: "chrome" });
+  let lastError = null;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const response = await client.fetch(url, {
+        signal: AbortSignal.timeout(timeout)
+      });
+      if (!response.ok) {
+        const statusCode = response.status;
+        if (statusCode >= 400 && statusCode < 500) {
+          throw new FetchError(`HTTP ${statusCode}`, statusCode, false);
+        }
+        if (isRetryableStatus2(statusCode)) {
+          throw new FetchError(`HTTP ${statusCode}`, statusCode, true);
+        }
+      }
+      const html = await response.text();
+      const headers = {};
+      response.headers.forEach((value, key) => {
+        headers[key] = value;
+      });
+      return {
+        html,
+        url: response.url ?? url,
+        statusCode: response.status,
+        headers
+      };
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      const isRetryable = error instanceof FetchError ? error.retryable : !(error instanceof FetchError);
+      const hasRetriesLeft = attempt < maxRetries;
+      if (isRetryable && hasRetriesLeft) {
+        const delay = getBackoffDelay2(attempt);
+        await sleep2(delay);
+        continue;
+      }
+      break;
+    }
+  }
+  throw lastError || new FetchError("Unknown stealth fetch error");
+}
+
+// src/fetch-render.ts
+var DEFAULT_TIMEOUT3 = 3e4;
+function isValidUrl3(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+async function fetchRender(url, options = {}) {
+  const { timeout = DEFAULT_TIMEOUT3 } = options;
+  if (!isValidUrl3(url)) {
+    throw new FetchError("Invalid URL", void 0, false);
+  }
+  let chromium;
+  try {
+    ({ chromium } = await import('playwright'));
+  } catch {
+    throw new FetchError(
+      "Playwright is required for render mode. Install: npm install playwright",
+      void 0,
+      false
+    );
+  }
+  const browser = await chromium.launch({ headless: true });
+  try {
+    const page = await browser.newPage();
+    page.on("dialog", (dialog) => dialog.dismiss());
+    const response = await page.goto(url, {
+      waitUntil: "networkidle",
+      timeout
+    });
+    if (!response) {
+      throw new FetchError(
+        "Navigation returned no response (about:blank or same-URL redirect)",
+        void 0,
+        false
+      );
+    }
+    const statusCode = response.status();
+    if (statusCode >= 400 && statusCode < 500) {
+      throw new FetchError(`HTTP ${statusCode}`, statusCode, false);
+    }
+    if (statusCode >= 500) {
+      throw new FetchError(`HTTP ${statusCode}`, statusCode, true);
+    }
+    const html = await page.content();
+    const headers = response.headers();
+    return {
+      html,
+      url: page.url(),
+      statusCode,
+      headers
+    };
+  } catch (error) {
+    if (error instanceof Error && error.name === "TimeoutError") {
+      throw new FetchError("Navigation timeout", void 0, true);
+    }
+    throw error;
+  } finally {
+    await browser.close();
+  }
+}
+var ExtractionError = class extends Error {
+  constructor(message, code) {
+    super(message);
+    this.code = code;
+    this.name = "ExtractionError";
+  }
+};
+async function extractContent(fetchResult) {
+  const { html } = fetchResult;
+  if (!html || !html.trim()) {
+    throw new ExtractionError("Empty HTML content", "EMPTY_HTML");
+  }
+  const { document } = parseHTML(html);
+  const reader = new Readability(document, {
+    charThreshold: 50
+  });
+  const article = reader.parse();
+  if (!article || !article.content || article.content.trim().length < 20) {
+    throw new ExtractionError(
+      "No content could be extracted from the page",
+      "NO_CONTENT"
+    );
+  }
+  const result = {
+    content: cleanContent(article.content),
+    title: article.title || extractFallbackTitle(document),
+    author: article.byline || void 0,
+    publishedAt: article.publishedTime || extractPublishedTime(document),
+    siteName: article.siteName || extractSiteName(document)
+  };
+  return result;
+}
+function cleanContent(content) {
+  return content.replace(/>\s+</g, "><").replace(/\s{2,}/g, " ").trim();
+}
+function extractFallbackTitle(document) {
+  const ogTitle = document.querySelector('meta[property="og:title"]');
+  if (ogTitle) {
+    const content = ogTitle.getAttribute("content");
+    if (content) return content;
+  }
+  const titleEl = document.querySelector("title");
+  if (titleEl && titleEl.textContent) {
+    return titleEl.textContent.trim();
+  }
+  const h1 = document.querySelector("h1");
+  if (h1 && h1.textContent) {
+    return h1.textContent.trim();
+  }
+  return void 0;
+}
+function extractPublishedTime(document) {
+  const ogTime = document.querySelector(
+    'meta[property="article:published_time"]'
+  );
+  if (ogTime) {
+    const content = ogTime.getAttribute("content");
+    if (content) return content;
+  }
+  const schemaTime = document.querySelector('[itemprop="datePublished"]');
+  if (schemaTime) {
+    const datetime = schemaTime.getAttribute("datetime");
+    if (datetime) return datetime;
+    const content = schemaTime.getAttribute("content");
+    if (content) return content;
+  }
+  const timeEl = document.querySelector("time[datetime]");
+  if (timeEl) {
+    const datetime = timeEl.getAttribute("datetime");
+    if (datetime) return datetime;
+  }
+  return void 0;
+}
+function extractSiteName(document) {
+  const ogSiteName = document.querySelector('meta[property="og:site_name"]');
+  if (ogSiteName) {
+    const content = ogSiteName.getAttribute("content");
+    if (content) return content;
+  }
+  const appName = document.querySelector('meta[name="application-name"]');
+  if (appName) {
+    const content = appName.getAttribute("content");
+    if (content) return content;
+  }
+  return void 0;
+}
+function createTurndownService() {
+  const turndown = new TurndownService({
+    headingStyle: "atx",
+    hr: "---",
+    bulletListMarker: "-",
+    codeBlockStyle: "fenced",
+    fence: "```",
+    emDelimiter: "*",
+    strongDelimiter: "**",
+    linkStyle: "inlined"
+  });
+  turndown.remove(["script", "style", "noscript", "iframe"]);
+  turndown.addRule("removeEmpty", {
+    filter: (node) => {
+      if (node.nodeType === 1) {
+        const text = node.textContent || "";
+        const isEmptyBlock = text.trim() === "" && !["IMG", "BR", "HR", "INPUT"].includes(node.nodeName);
+        return isEmptyBlock;
+      }
+      return false;
+    },
+    replacement: () => ""
+  });
+  turndown.addRule("fencedCodeBlock", {
+    filter: (node, options) => {
+      return options.codeBlockStyle === "fenced" && node.nodeName === "PRE" && node.firstChild !== null && node.firstChild.nodeName === "CODE";
+    },
+    replacement: (_content, node, options) => {
+      const codeNode = node.firstChild;
+      const code = codeNode.textContent || "";
+      const className = codeNode.getAttribute("class") || "";
+      const langMatch = className.match(/language-(\w+)/);
+      const lang = langMatch ? langMatch[1] : "";
+      const fence = options.fence || "```";
+      return `
+
+${fence}${lang}
+${code}
+${fence}
+
+`;
+    }
+  });
+  turndown.addRule("strikethrough", {
+    filter: ["del", "s"],
+    replacement: (content) => `~~${content}~~`
+  });
+  return turndown;
+}
+var turndownInstance = null;
+function getTurndown() {
+  if (!turndownInstance) {
+    turndownInstance = createTurndownService();
+  }
+  return turndownInstance;
+}
+function cleanMarkdown(markdown) {
+  return markdown.replace(/\n{3,}/g, "\n\n").replace(/[ \t]+$/gm, "").replace(/^\n+/, "").replace(/\n+$/, "").trim();
+}
+async function convertToMarkdown(extractResult) {
+  const { content } = extractResult;
+  if (!content || !content.trim()) {
+    return {
+      markdown: "",
+      tokenCount: 0
+    };
+  }
+  const turndown = getTurndown();
+  let markdown = turndown.turndown(content);
+  markdown = cleanMarkdown(markdown);
+  const tokenCount = estimateTokens(markdown);
+  return {
+    markdown,
+    tokenCount
+  };
+}
+function estimateTokens(text) {
+  if (!text) return 0;
+  let tokens = 0;
+  let remaining = text;
+  remaining = remaining.replace(/```[\s\S]*?```/g, (match) => {
+    tokens += Math.ceil(match.length / 3.2);
+    return " ";
+  });
+  remaining = remaining.replace(/`[^`]+`/g, (match) => {
+    tokens += Math.ceil(match.length / 3.5);
+    return " ";
+  });
+  remaining = remaining.replace(/https?:\/\/\S+/g, (match) => {
+    tokens += Math.ceil(match.length / 5);
+    return " ";
+  });
+  const proseLength = remaining.replace(/\s+/g, " ").trim().length;
+  if (proseLength > 0) {
+    tokens += Math.ceil(proseLength / 4.3);
+  }
+  return Math.max(1, tokens);
+}
+var DEFAULT_TTL_MS = 60 * 60 * 1e3;
+var DEFAULT_TIMEOUT_MS = 5e3;
+var BOT_USER_AGENT = "ScraperBot";
+var cache = /* @__PURE__ */ new Map();
+function getRobotsUrl(url) {
+  const parsed = new URL(url);
+  return `${parsed.protocol}//${parsed.host}/robots.txt`;
+}
+async function getRobotsParser(url, timeout = DEFAULT_TIMEOUT_MS) {
+  const robotsUrl = getRobotsUrl(url);
+  const cached = cache.get(robotsUrl);
+  if (cached && cached.expiresAt > Date.now()) {
+    return cached.parser;
+  }
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    const response = await fetch(robotsUrl, {
+      signal: controller.signal,
+      headers: { "User-Agent": BOT_USER_AGENT }
+    });
+    clearTimeout(timeoutId);
+    const text = response.ok ? await response.text() : "";
+    const parser = robotsParser(robotsUrl, text);
+    cache.set(robotsUrl, {
+      parser,
+      expiresAt: Date.now() + DEFAULT_TTL_MS
+    });
+    return parser;
+  } catch {
+    const parser = robotsParser(robotsUrl, "");
+    cache.set(robotsUrl, {
+      parser,
+      expiresAt: Date.now() + DEFAULT_TTL_MS
+    });
+    return parser;
+  }
+}
+async function isAllowedByRobots(url, timeout) {
+  const parser = await getRobotsParser(url, timeout);
+  return parser.isAllowed(url, BOT_USER_AGENT) !== false;
+}
+async function getSitemapUrls(url, timeout) {
+  const parser = await getRobotsParser(url, timeout);
+  return parser.getSitemaps();
+}
+async function getCrawlDelay(url, timeout) {
+  const parser = await getRobotsParser(url, timeout);
+  const delay = parser.getCrawlDelay(BOT_USER_AGENT);
+  return delay === void 0 ? null : delay;
+}
+function clearRobotsCache() {
+  cache.clear();
+}
+
+// src/sitemap.ts
+var DEFAULT_TIMEOUT_MS2 = 1e4;
+var DEFAULT_TTL_MS2 = 60 * 60 * 1e3;
+var MAX_RECURSION_DEPTH = 2;
+var cache2 = /* @__PURE__ */ new Map();
+function getOrigin(url) {
+  const parsed = new URL(url);
+  return `${parsed.protocol}//${parsed.host}`;
+}
+function isSitemapIndex(xml) {
+  return /<sitemapindex[\s>]/i.test(xml);
+}
+function extractSitemapIndexUrls(xml) {
+  const urls = [];
+  const sitemapBlockRegex = /<(?:\w+:)?sitemap\b[^>]*>([\s\S]*?)<\/(?:\w+:)?sitemap>/gi;
+  let blockMatch;
+  while ((blockMatch = sitemapBlockRegex.exec(xml)) !== null) {
+    const block = blockMatch[1];
+    const locMatch = /<(?:\w+:)?loc\b[^>]*>([\s\S]*?)<\/(?:\w+:)?loc>/i.exec(block);
+    if (locMatch) {
+      const url = locMatch[1].trim();
+      if (url) urls.push(url);
+    }
+  }
+  return urls;
+}
+function extractUrlEntries(xml, origin) {
+  const entries = [];
+  const urlBlockRegex = /<(?:\w+:)?url\b[^>]*>([\s\S]*?)<\/(?:\w+:)?url>/gi;
+  let blockMatch;
+  while ((blockMatch = urlBlockRegex.exec(xml)) !== null) {
+    const block = blockMatch[1];
+    const locMatch = /<(?:\w+:)?loc\b[^>]*>([\s\S]*?)<\/(?:\w+:)?loc>/i.exec(block);
+    if (!locMatch) continue;
+    const loc = locMatch[1].trim();
+    if (!loc) continue;
+    try {
+      if (getOrigin(loc) !== origin) continue;
+    } catch {
+      continue;
+    }
+    const entry = { loc };
+    const lastmodMatch = /<(?:\w+:)?lastmod\b[^>]*>([\s\S]*?)<\/(?:\w+:)?lastmod>/i.exec(block);
+    if (lastmodMatch) {
+      const lastmod = lastmodMatch[1].trim();
+      if (lastmod) entry.lastmod = lastmod;
+    }
+    const priorityMatch = /<(?:\w+:)?priority\b[^>]*>([\s\S]*?)<\/(?:\w+:)?priority>/i.exec(block);
+    if (priorityMatch) {
+      const priority = parseFloat(priorityMatch[1].trim());
+      if (!isNaN(priority)) entry.priority = priority;
+    }
+    entries.push(entry);
+  }
+  return entries;
+}
+async function fetchSitemapXml(url, timeout) {
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: { "User-Agent": "ScraperBot/1.0" }
+    });
+    clearTimeout(timeoutId);
+    if (!response.ok) return null;
+    return await response.text();
+  } catch {
+    return null;
+  }
+}
+async function parseSitemapInternal(url, origin, timeout, depth) {
+  if (depth >= MAX_RECURSION_DEPTH) return [];
+  const xml = await fetchSitemapXml(url, timeout);
+  if (!xml) return [];
+  if (isSitemapIndex(xml)) {
+    const sitemapUrls = extractSitemapIndexUrls(xml);
+    const allEntries = [];
+    for (const sitemapUrl of sitemapUrls) {
+      const entries = await parseSitemapInternal(
+        sitemapUrl,
+        origin,
+        timeout,
+        depth + 1
+      );
+      allEntries.push(...entries);
+    }
+    return allEntries;
+  }
+  return extractUrlEntries(xml, origin);
+}
+async function parseSitemap(url, timeout = DEFAULT_TIMEOUT_MS2) {
+  const cached = cache2.get(url);
+  if (cached && cached.expiresAt > Date.now()) {
+    return cached.entries;
+  }
+  let origin;
+  try {
+    origin = getOrigin(url);
+  } catch {
+    return [];
+  }
+  const entries = await parseSitemapInternal(url, origin, timeout, 0);
+  cache2.set(url, {
+    entries,
+    expiresAt: Date.now() + DEFAULT_TTL_MS2
+  });
+  return entries;
+}
+function clearSitemapCache() {
+  cache2.clear();
+}
+
+// src/fetch-mode.ts
+var CHALLENGE_MARKERS = [
+  "cf-browser-verification",
+  "Just a moment",
+  "_cf_chl_opt",
+  "akamai-challenge",
+  "ak-challenge"
+];
+function isChallengeResponse(fetchResult) {
+  return CHALLENGE_MARKERS.some((marker) => fetchResult.html.includes(marker));
+}
+var VALID_MODES = ["fast", "stealth", "render", "auto"];
+async function fetchWithMode(url, mode, options) {
+  if (!VALID_MODES.includes(mode)) {
+    throw new FetchError(
+      `Invalid fetch mode: '${mode}'. Valid modes: ${VALID_MODES.join(", ")}`,
+      void 0,
+      false
+    );
+  }
+  if (mode === "stealth") return fetchStealth(url, options);
+  if (mode === "render") return fetchRender(url, options);
+  if (mode === "fast") return fetchUrl(url, options);
+  try {
+    const result = await fetchUrl(url, options);
+    if (isChallengeResponse(result)) {
+      return fetchStealth(url, options);
+    }
+    return result;
+  } catch (err) {
+    if (err instanceof FetchError && err.statusCode === 403) {
+      return fetchStealth(url, options);
+    }
+    throw err;
+  }
+}
+
+// src/crawl.ts
+function isValidUrl4(url) {
+  try {
+    const parsed = new URL(url);
+    return parsed.protocol === "http:" || parsed.protocol === "https:";
+  } catch {
+    return false;
+  }
+}
+function normalizeUrl(url) {
+  try {
+    const parsed = new URL(url);
+    parsed.hash = "";
+    let pathname = parsed.pathname;
+    if (pathname.length > 1 && pathname.endsWith("/")) {
+      pathname = pathname.slice(0, -1);
+    }
+    parsed.pathname = pathname;
+    return parsed.toString();
+  } catch {
+    return url;
+  }
+}
+var SKIP_EXTENSIONS = /* @__PURE__ */ new Set([
+  ".pdf",
+  ".jpg",
+  ".jpeg",
+  ".png",
+  ".gif",
+  ".svg",
+  ".webp",
+  ".ico",
+  ".mp4",
+  ".mp3",
+  ".wav",
+  ".avi",
+  ".zip",
+  ".tar",
+  ".gz",
+  ".rar",
+  ".css",
+  ".js",
+  ".xml",
+  ".json",
+  ".woff",
+  ".woff2",
+  ".ttf",
+  ".eot"
+]);
+function extractLinks(html, baseUrl) {
+  const links = [];
+  let origin;
+  try {
+    origin = new URL(baseUrl).origin;
+  } catch {
+    return [];
+  }
+  const regex = /<a\s+[^>]*href\s*=\s*["']([^"']+)["'][^>]*>/gi;
+  let match;
+  while ((match = regex.exec(html)) !== null) {
+    const href = match[1].trim();
+    if (href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:") || href.startsWith("#")) continue;
+    try {
+      const resolved = new URL(href, baseUrl);
+      if (resolved.origin !== origin) continue;
+      const ext = resolved.pathname.toLowerCase().match(/\.\w+$/)?.[0];
+      if (ext && SKIP_EXTENSIONS.has(ext)) continue;
+      links.push(normalizeUrl(resolved.toString()));
+    } catch {
+    }
+  }
+  return [...new Set(links)];
+}
+function matchGlob(url, pattern) {
+  const regex = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*\*/g, "{{DOUBLESTAR}}").replace(/\*/g, "[^/]*").replace(/\{\{DOUBLESTAR\}\}/g, ".*");
+  return new RegExp(regex).test(url);
+}
+function matchesFilter(url, include, exclude) {
+  if (exclude?.length) {
+    for (const pattern of exclude) {
+      if (matchGlob(url, pattern)) return false;
+    }
+  }
+  if (include?.length) {
+    return include.some((pattern) => matchGlob(url, pattern));
+  }
+  return true;
+}
+function sleep3(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function crawl(options) {
+  const startTime = Date.now();
+  const {
+    url: startUrl,
+    depth: maxDepth = 2,
+    limit = 50,
+    mode = "auto",
+    include,
+    exclude,
+    timeout,
+    concurrency = 3,
+    respectRobots = true
+  } = options;
+  if (!isValidUrl4(startUrl)) {
+    throw new FetchError("Invalid URL", void 0, false);
+  }
+  if (maxDepth < 0) throw new FetchError("depth must be >= 0", void 0, false);
+  if (limit < 1) throw new FetchError("limit must be >= 1", void 0, false);
+  if (concurrency < 1) throw new FetchError("concurrency must be >= 1", void 0, false);
+  if (timeout !== void 0 && (timeout <= 0 || Number.isNaN(timeout))) {
+    throw new FetchError("timeout must be a positive number", void 0, false);
+  }
+  const pages = [];
+  const errors = [];
+  const visited = /* @__PURE__ */ new Set();
+  let totalDiscovered = 0;
+  let totalSkipped = 0;
+  const normalizedStart = normalizeUrl(startUrl);
+  const origin = new URL(startUrl).origin;
+  let crawlDelay = null;
+  if (respectRobots) {
+    crawlDelay = await getCrawlDelay(startUrl, timeout);
+    const allowed = await isAllowedByRobots(startUrl, timeout);
+    if (!allowed) {
+      return {
+        pages: [],
+        totalDiscovered: 1,
+        totalCrawled: 0,
+        totalSkipped: 1,
+        errors: [],
+        duration: Date.now() - startTime
+      };
+    }
+  }
+  const queue = [
+    { url: normalizedStart, depth: 0 }
+  ];
+  if (maxDepth > 0) {
+    try {
+      const sitemapEntries = await parseSitemap(`${origin}/sitemap.xml`, timeout);
+      const seen = /* @__PURE__ */ new Set([normalizedStart]);
+      for (const entry of sitemapEntries) {
+        const normalized = normalizeUrl(entry.loc);
+        if (!seen.has(normalized)) {
+          seen.add(normalized);
+          queue.push({ url: normalized, depth: 1 });
+        }
+      }
+    } catch (err) {
+      console.debug(`[scraper] Sitemap unavailable for ${origin}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  totalDiscovered = queue.length;
+  while (queue.length > 0 && pages.length < limit) {
+    const batchSize = Math.min(concurrency, limit - pages.length, queue.length);
+    const batch = queue.splice(0, batchSize);
+    const tasks = batch.map(async ({ url, depth }) => {
+      const normalized = normalizeUrl(url);
+      if (visited.has(normalized)) {
+        totalSkipped++;
+        return;
+      }
+      visited.add(normalized);
+      if (normalized !== normalizedStart && !matchesFilter(normalized, include, exclude)) {
+        totalSkipped++;
+        return;
+      }
+      if (respectRobots) {
+        const allowed = await isAllowedByRobots(url, timeout);
+        if (!allowed) {
+          totalSkipped++;
+          return;
+        }
+      }
+      try {
+        const fetchResult = await fetchWithMode(url, mode, { timeout });
+        const extractResult = await extractContent(fetchResult);
+        const convertResult = await convertToMarkdown(extractResult);
+        const pageResult = {
+          markdown: convertResult.markdown,
+          tokenCount: convertResult.tokenCount,
+          title: extractResult.title,
+          author: extractResult.author,
+          siteName: extractResult.siteName,
+          publishedAt: extractResult.publishedAt,
+          url: fetchResult.url,
+          depth
+        };
+        pages.push(pageResult);
+        if (depth < maxDepth) {
+          const links = extractLinks(fetchResult.html, fetchResult.url);
+          for (const link of links) {
+            if (!visited.has(link)) {
+              queue.push({ url: link, depth: depth + 1 });
+              totalDiscovered++;
+            }
+          }
+        }
+      } catch (err) {
+        errors.push({
+          url,
+          error: err instanceof Error ? err.message : String(err),
+          depth
+        });
+      }
+    });
+    await Promise.allSettled(tasks);
+    if (crawlDelay && crawlDelay > 0 && queue.length > 0) {
+      await sleep3(crawlDelay * 1e3);
+    }
+  }
+  return {
+    pages,
+    totalDiscovered,
+    totalCrawled: pages.length,
+    totalSkipped,
+    errors,
+    duration: Date.now() - startTime
+  };
+}
+var CONFIG_PATH = join(homedir(), ".robot-resources", "config.json");
+var PLATFORM_URL = process.env.RR_PLATFORM_URL || "https://api.robotresources.ai";
+var apiKey = null;
+var keyLoaded = false;
+function loadApiKey() {
+  if (keyLoaded) return apiKey;
+  keyLoaded = true;
+  if (process.env.RR_TELEMETRY === "off") {
+    return null;
+  }
+  try {
+    const config = JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
+    if (config.telemetry === false) {
+      return null;
+    }
+    apiKey = config.api_key || null;
+    return apiKey;
+  } catch {
+    return null;
+  }
+}
+function reportScraperEvent(payload) {
+  const key = loadApiKey();
+  if (!key) return;
+  fetch(`${PLATFORM_URL}/v1/telemetry`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${key}`
+    },
+    body: JSON.stringify({
+      product: "scraper",
+      event_type: payload.success ? "compress" : "error",
+      payload
+    })
+  }).catch(() => {
+  });
+}
+
+// src/index.ts
+async function scrape(url, options = {}) {
+  const startTime = Date.now();
+  const mode = options.mode ?? "auto";
+  if (options.timeout !== void 0 && (options.timeout <= 0 || Number.isNaN(options.timeout))) {
+    throw new FetchError("timeout must be a positive number", void 0, false);
+  }
+  try {
+    if (options.respectRobots) {
+      const allowed = await isAllowedByRobots(url, options.timeout);
+      if (!allowed) {
+        throw new FetchError(`Blocked by robots.txt: ${url}`, void 0, false);
+      }
+    }
+    const fetchResult = await fetchWithMode(url, mode, {
+      timeout: options.timeout,
+      maxRetries: options.maxRetries,
+      userAgent: options.userAgent
+    });
+    const extractResult = await extractContent(fetchResult);
+    const convertResult = await convertToMarkdown(extractResult);
+    const result = {
+      markdown: convertResult.markdown,
+      tokenCount: convertResult.tokenCount,
+      title: extractResult.title,
+      author: extractResult.author,
+      siteName: extractResult.siteName,
+      publishedAt: extractResult.publishedAt,
+      url: fetchResult.url
+    };
+    reportScraperEvent({
+      url,
+      tokenCount: convertResult.tokenCount,
+      title: extractResult.title,
+      latencyMs: Date.now() - startTime,
+      success: true
+    });
+    return result;
+  } catch (err) {
+    reportScraperEvent({
+      url,
+      tokenCount: 0,
+      latencyMs: Date.now() - startTime,
+      success: false,
+      error: err instanceof Error ? err.message : String(err)
+    });
+    throw err;
+  }
+}
+
+export { ExtractionError, FetchError, clearRobotsCache, clearSitemapCache, convertToMarkdown, crawl, estimateTokens, extractContent, extractLinks, fetchRender, fetchStealth, fetchUrl, fetchWithMode, getCrawlDelay, getSitemapUrls, isAllowedByRobots, isChallengeResponse, normalizeUrl, parseSitemap, scrape };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map