@detiq/crawler 1.1.8 → 1.1.9

package/dist/sitemap.js

@@ -1,154 +1,215 @@
-import { XMLParser } from "fast-xml-parser";
-import { ProxyAgent } from "undici";
-import { assertCrawlUrlSafe } from "@detiq/core";
-import { logger } from "./logger.js";
-const FETCH_TIMEOUT_MS = 1e4;
-const MAX_SITEMAP_INDEX_DEPTH = 3;
-const MAX_SUB_SITEMAPS = 50;
-const MAX_URLS = 5e3;
+/**
+ * Sitemap.xml discovery — seeds the crawl queue from a project's sitemap
+ * instead of (or alongside) plain link-following, matching crawl4AI's
+ * sitemap-seeding capability. Best-effort throughout: most sites don't have
+ * a sitemap, or it 404s, or it's malformed — none of that should ever fail
+ * a crawl, it just means sitemap seeding contributes nothing this run.
+ */
+import { XMLParser } from 'fast-xml-parser';
+import { ProxyAgent } from 'undici';
+import { assertCrawlUrlSafe } from '@detiq/core';
+import { logger } from './logger.js';
+const FETCH_TIMEOUT_MS = 10_000;
+const MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)
+const MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index
+const MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory
 function toArray(x) {
-  if (x == null) return [];
-  return Array.isArray(x) ? x : [x];
+    if (x == null)
+        return [];
+    return Array.isArray(x) ? x : [x];
 }
+/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */
 function hostKey(hostname) {
-  return hostname.replace(/^www\./i, "").toLowerCase();
+    return hostname.replace(/^www\./i, '').toLowerCase();
 }
-function sameSite(candidateUrl, siteOrigin) {
-  try {
-    const originHost = /^https?:\/\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;
-    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);
-  } catch {
-    return false;
-  }
+/**
+ * Same-site check that tolerates www/non-www and http/https differences — a sitemap on
+ * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).
+ */
+export function sameSite(candidateUrl, siteOrigin) {
+    try {
+        const originHost = /^https?:\/\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;
+        return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);
+    }
+    catch {
+        return false;
+    }
 }
-function extractUrlsFromParsedSitemap(parsed, origin) {
-  const rawPageUrls = [];
-  let subSitemapUrls = [];
-  if (parsed?.sitemapindex) {
-    subSitemapUrls = toArray(parsed.sitemapindex.sitemap).map((s) => s?.loc).filter((loc) => typeof loc === "string").slice(0, MAX_SUB_SITEMAPS);
-  }
-  if (parsed?.urlset) {
-    rawPageUrls.push(
-      ...toArray(parsed.urlset.url).map((u) => u?.loc).filter((loc) => typeof loc === "string")
-    );
-  }
-  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));
-  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };
+/**
+ * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap
+ * document (either a <urlset> or a <sitemapindex>), filtered to same-origin
+ * and deduped. Pure function — no I/O — for direct unit testing without a
+ * real HTTP fetch.
+ */
+export function extractUrlsFromParsedSitemap(parsed, origin) {
+    const rawPageUrls = [];
+    let subSitemapUrls = [];
+    if (parsed?.sitemapindex) {
+        subSitemapUrls = toArray(parsed.sitemapindex.sitemap)
+            .map((s) => s?.loc)
+            .filter((loc) => typeof loc === 'string')
+            .slice(0, MAX_SUB_SITEMAPS);
+    }
+    if (parsed?.urlset) {
+        rawPageUrls.push(...toArray(parsed.urlset.url)
+            .map((u) => u?.loc)
+            .filter((loc) => typeof loc === 'string'));
+    }
+    // Same-site only (www/protocol-tolerant) — a sitemap listing external URLs shouldn't seed this project's queue.
+    const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));
+    return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };
 }
 async function fetchAndParseXml(url, dispatcher) {
-  try {
-    assertCrawlUrlSafe(url);
-  } catch {
-    return null;
-  }
-  try {
-    const res = await fetch(url, {
-      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
-      headers: { "User-Agent": "ZeTa-AI-Crawler/1.0 (+sitemap-discovery)" },
-      ...dispatcher ? { dispatcher } : {}
-    });
-    if (!res.ok) return null;
-    const text = await res.text();
-    return new XMLParser().parse(text);
-  } catch {
-    return null;
-  }
+    // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the
+    // stricter webhook-oriented assertResolvesToPublicAddress) — a sitemap
+    // living on a tenant's internal/VPN-only staging server is a legitimate
+    // crawl target, not an attack; only loopback/cloud-metadata are blocked.
+    try {
+        assertCrawlUrlSafe(url);
+    }
+    catch {
+        return null;
+    }
+    try {
+        // Redirects followed normally here (unlike the webhook-dispatch.ts
+        // 'manual' pattern) — an http→https or bare→www redirect on the
+        // sitemap is completely ordinary, and Playwright's actual page
+        // navigation later in the crawl already follows redirects to whatever
+        // this same site sends it to; a sitemap fetch isn't a more sensitive
+        // operation than that.
+        const res = await fetch(url, {
+            signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+            headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },
+            ...(dispatcher ? { dispatcher } : {}),
+        });
+        if (!res.ok)
+            return null;
+        const text = await res.text();
+        return new XMLParser().parse(text);
+    }
+    catch {
+        return null;
+    }
 }
-async function fetchSitemapUrls(sitemapUrl, origin, depth = 0, dispatcher) {
-  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];
-  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);
-  if (!parsed) return [];
-  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
-  const collected = [...pageUrls];
-  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {
-    const subResults = await Promise.all(
-      subSitemapUrls.slice(0, 10).map((subUrl) => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))
-    );
-    for (const sub of subResults) collected.push(...sub);
-  }
-  return [...new Set(collected)].slice(0, MAX_URLS);
+/**
+ * Fetches and parses a sitemap (or sitemap index, recursed up to a small
+ * depth) into a flat, deduped list of same-origin URLs.
+ */
+export async function fetchSitemapUrls(sitemapUrl, origin, depth = 0, dispatcher) {
+    if (depth > MAX_SITEMAP_INDEX_DEPTH)
+        return [];
+    const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);
+    if (!parsed)
+        return [];
+    const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
+    const collected = [...pageUrls];
+    if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {
+        const subResults = await Promise.all(subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher)));
+        for (const sub of subResults)
+            collected.push(...sub);
+    }
+    return [...new Set(collected)].slice(0, MAX_URLS);
 }
-async function discoverSitemapUrls(appUrl, sitemapUrl) {
-  let origin;
-  try {
-    origin = new URL(appUrl).origin;
-  } catch {
-    return [];
-  }
-  const target = sitemapUrl || `${origin}/sitemap.xml`;
-  try {
-    return await fetchSitemapUrls(target, origin);
-  } catch (err) {
-    logger.warn({ err: String(err), target }, "[sitemap] discovery failed, continuing without it");
-    return [];
-  }
+/**
+ * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an
+ * explicit override) and returns the URLs it lists. Never throws.
+ */
+export async function discoverSitemapUrls(appUrl, sitemapUrl) {
+    let origin;
+    try {
+        origin = new URL(appUrl).origin;
+    }
+    catch {
+        return [];
+    }
+    const target = sitemapUrl || `${origin}/sitemap.xml`;
+    try {
+        return await fetchSitemapUrls(target, origin);
+    }
+    catch (err) {
+        logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');
+        return [];
+    }
 }
-function buildProxyDispatcher(proxyUrl) {
-  if (!proxyUrl || proxyUrl === "auto") return void 0;
-  try {
-    const m = proxyUrl.match(/^(https?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
-    if (!m) return new ProxyAgent(proxyUrl);
-    const [, scheme, user, pass, hostPort] = m;
-    const uri = `${scheme}://${hostPort}`;
-    if (user) {
-      const token = "Basic " + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? "")}`).toString("base64");
-      return new ProxyAgent({ uri, token });
-    }
-    return new ProxyAgent({ uri });
-  } catch {
-    return void 0;
-  }
+/**
+ * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because
+ * DataImpulse-style usernames contain commas (country codes) that break new URL().
+ * Returns undefined for no/auto proxy — the caller then fetches directly.
+ */
+export function buildProxyDispatcher(proxyUrl) {
+    if (!proxyUrl || proxyUrl === 'auto')
+        return undefined;
+    try {
+        const m = proxyUrl.match(/^(https?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
+        if (!m)
+            return new ProxyAgent(proxyUrl);
+        const [, scheme, user, pass, hostPort] = m;
+        const uri = `${scheme}://${hostPort}`;
+        if (user) {
+            const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');
+            return new ProxyAgent({ uri, token });
+        }
+        return new ProxyAgent({ uri });
+    }
+    catch {
+        return undefined;
+    }
 }
-function parseRobotsSitemaps(robotsTxt, origin) {
-  const out = [];
-  for (const line of robotsTxt.split(/\r?\n/)) {
-    const m = line.match(/^\s*sitemap:\s*(\S+)/i);
-    if (m && sameSite(m[1], origin)) out.push(m[1]);
-  }
-  return [...new Set(out)];
+/** Extract same-origin `Sitemap:` directives from a robots.txt body. */
+export function parseRobotsSitemaps(robotsTxt, origin) {
+    const out = [];
+    for (const line of robotsTxt.split(/\r?\n/)) {
+        const m = line.match(/^\s*sitemap:\s*(\S+)/i);
+        if (m && sameSite(m[1], origin))
+            out.push(m[1]);
+    }
+    return [...new Set(out)];
 }
-async function discoverProjectUrls(appUrl, opts) {
-  let origin;
-  try {
-    origin = new URL(appUrl).origin;
-  } catch {
-    return { candidates: [], sitemapFound: false };
-  }
-  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);
-  let robotsSitemaps = [];
-  try {
-    assertCrawlUrlSafe(`${origin}/robots.txt`);
-    const res = await fetch(`${origin}/robots.txt`, {
-      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
-      headers: { "User-Agent": "ZeTa-AI-Crawler/1.0 (+url-discovery)" },
-      ...dispatcher ? { dispatcher } : {}
-    });
-    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);
-  } catch {
-  }
-  const robotsSet = new Set(robotsSitemaps);
-  const sitemapTargets = [.../* @__PURE__ */ new Set([
-    ...opts?.sitemapUrl ? [opts.sitemapUrl] : [],
-    ...robotsSitemaps,
-    `${origin}/sitemap.xml`
-  ])];
-  const seen = /* @__PURE__ */ new Map();
-  let sitemapFound = false;
-  for (const sm of sitemapTargets) {
-    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);
-    if (urls.length) sitemapFound = true;
-    const source = robotsSet.has(sm) ? "robots" : "sitemap";
-    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });
-  }
-  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };
+/**
+ * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap
+ * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped
+ * candidate list the user can review and seed before launching the Playwright crawl.
+ * Never throws — bot-protected sites (Akamai/Cloudflare) simply yield an empty list.
+ */
+export async function discoverProjectUrls(appUrl, opts) {
+    let origin;
+    try {
+        origin = new URL(appUrl).origin;
+    }
+    catch {
+        return { candidates: [], sitemapFound: false };
+    }
+    const dispatcher = buildProxyDispatcher(opts?.proxyUrl);
+    // 1. robots.txt → declared sitemaps
+    let robotsSitemaps = [];
+    try {
+        assertCrawlUrlSafe(`${origin}/robots.txt`);
+        const res = await fetch(`${origin}/robots.txt`, {
+            signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+            headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },
+            ...(dispatcher ? { dispatcher } : {}),
+        });
+        if (res.ok)
+            robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);
+    }
+    catch { /* best-effort */ }
+    // 2. sitemap targets: explicit override + robots-declared + conventional default
+    const robotsSet = new Set(robotsSitemaps);
+    const sitemapTargets = [...new Set([
+            ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),
+            ...robotsSitemaps,
+            `${origin}/sitemap.xml`,
+        ])];
+    const seen = new Map();
+    let sitemapFound = false;
+    for (const sm of sitemapTargets) {
+        const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);
+        if (urls.length)
+            sitemapFound = true;
+        const source = robotsSet.has(sm) ? 'robots' : 'sitemap';
+        for (const u of urls)
+            if (!seen.has(u))
+                seen.set(u, { url: u, source });
+    }
+    return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };
 }
-export {
-  buildProxyDispatcher,
-  discoverProjectUrls,
-  discoverSitemapUrls,
-  extractUrlsFromParsedSitemap,
-  fetchSitemapUrls,
-  parseRobotsSitemaps,
-  sameSite
-};
-//# sourceMappingURL=sitemap.js.map