@detiq/crawler 1.1.9 → 1.1.10

package/dist/sitemap.js

@@ -1,215 +1,154 @@
-/**
- * Sitemap.xml discovery — seeds the crawl queue from a project's sitemap
- * instead of (or alongside) plain link-following, matching crawl4AI's
- * sitemap-seeding capability. Best-effort throughout: most sites don't have
- * a sitemap, or it 404s, or it's malformed — none of that should ever fail
- * a crawl, it just means sitemap seeding contributes nothing this run.
- */
-import { XMLParser } from 'fast-xml-parser';
-import { ProxyAgent } from 'undici';
-import { assertCrawlUrlSafe } from '@detiq/core';
-import { logger } from './logger.js';
-const FETCH_TIMEOUT_MS = 10_000;
-const MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)
-const MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index
-const MAX_URLS = 5000; // sane upper bound so a huge/malicious sitemap can't blow up memory
+import { XMLParser } from "fast-xml-parser";
+import { ProxyAgent } from "undici";
+import { assertCrawlUrlSafe } from "@detiq/core";
+import { logger } from "./logger.js";
+const FETCH_TIMEOUT_MS = 1e4;
+const MAX_SITEMAP_INDEX_DEPTH = 3;
+const MAX_SUB_SITEMAPS = 50;
+const MAX_URLS = 5e3;
 function toArray(x) {
-    if (x == null)
-        return [];
-    return Array.isArray(x) ? x : [x];
+  if (x == null) return [];
+  return Array.isArray(x) ? x : [x];
 }
-/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */
 function hostKey(hostname) {
-    return hostname.replace(/^www\./i, '').toLowerCase();
+  return hostname.replace(/^www\./i, "").toLowerCase();
 }
-/**
- * Same-site check that tolerates www/non-www and http/https differences — a sitemap on
- * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).
- */
-export function sameSite(candidateUrl, siteOrigin) {
-    try {
-        const originHost = /^https?:\/\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;
-        return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);
-    }
-    catch {
-        return false;
-    }
+function sameSite(candidateUrl, siteOrigin) {
+  try {
+    const originHost = /^https?:\/\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;
+    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);
+  } catch {
+    return false;
+  }
 }
-/**
- * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap
- * document (either a <urlset> or a <sitemapindex>), filtered to same-origin
- * and deduped. Pure function — no I/O — for direct unit testing without a
- * real HTTP fetch.
- */
-export function extractUrlsFromParsedSitemap(parsed, origin) {
-    const rawPageUrls = [];
-    let subSitemapUrls = [];
-    if (parsed?.sitemapindex) {
-        subSitemapUrls = toArray(parsed.sitemapindex.sitemap)
-            .map((s) => s?.loc)
-            .filter((loc) => typeof loc === 'string')
-            .slice(0, MAX_SUB_SITEMAPS);
-    }
-    if (parsed?.urlset) {
-        rawPageUrls.push(...toArray(parsed.urlset.url)
-            .map((u) => u?.loc)
-            .filter((loc) => typeof loc === 'string'));
-    }
-    // Same-site only (www/protocol-tolerant) — a sitemap listing external URLs shouldn't seed this project's queue.
-    const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));
-    return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };
+function extractUrlsFromParsedSitemap(parsed, origin) {
+  const rawPageUrls = [];
+  let subSitemapUrls = [];
+  if (parsed?.sitemapindex) {
+    subSitemapUrls = toArray(parsed.sitemapindex.sitemap).map((s) => s?.loc).filter((loc) => typeof loc === "string").slice(0, MAX_SUB_SITEMAPS);
+  }
+  if (parsed?.urlset) {
+    rawPageUrls.push(
+      ...toArray(parsed.urlset.url).map((u) => u?.loc).filter((loc) => typeof loc === "string")
+    );
+  }
+  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));
+  return { pageUrls: [...new Set(sameOrigin)].slice(0, MAX_URLS), subSitemapUrls };
 }
 async function fetchAndParseXml(url, dispatcher) {
-    // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the
-    // stricter webhook-oriented assertResolvesToPublicAddress) — a sitemap
-    // living on a tenant's internal/VPN-only staging server is a legitimate
-    // crawl target, not an attack; only loopback/cloud-metadata are blocked.
-    try {
-        assertCrawlUrlSafe(url);
-    }
-    catch {
-        return null;
-    }
-    try {
-        // Redirects followed normally here (unlike the webhook-dispatch.ts
-        // 'manual' pattern) — an http→https or bare→www redirect on the
-        // sitemap is completely ordinary, and Playwright's actual page
-        // navigation later in the crawl already follows redirects to whatever
-        // this same site sends it to; a sitemap fetch isn't a more sensitive
-        // operation than that.
-        const res = await fetch(url, {
-            signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
-            headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },
-            ...(dispatcher ? { dispatcher } : {}),
-        });
-        if (!res.ok)
-            return null;
-        const text = await res.text();
-        return new XMLParser().parse(text);
-    }
-    catch {
-        return null;
-    }
+  try {
+    assertCrawlUrlSafe(url);
+  } catch {
+    return null;
+  }
+  try {
+    const res = await fetch(url, {
+      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      headers: { "User-Agent": "ZeTa-AI-Crawler/1.0 (+sitemap-discovery)" },
+      ...dispatcher ? { dispatcher } : {}
+    });
+    if (!res.ok) return null;
+    const text = await res.text();
+    return new XMLParser().parse(text);
+  } catch {
+    return null;
+  }
 }
-/**
- * Fetches and parses a sitemap (or sitemap index, recursed up to a small
- * depth) into a flat, deduped list of same-origin URLs.
- */
-export async function fetchSitemapUrls(sitemapUrl, origin, depth = 0, dispatcher) {
-    if (depth > MAX_SITEMAP_INDEX_DEPTH)
-        return [];
-    const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);
-    if (!parsed)
-        return [];
-    const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
-    const collected = [...pageUrls];
-    if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {
-        const subResults = await Promise.all(subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher)));
-        for (const sub of subResults)
-            collected.push(...sub);
-    }
-    return [...new Set(collected)].slice(0, MAX_URLS);
+async function fetchSitemapUrls(sitemapUrl, origin, depth = 0, dispatcher) {
+  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];
+  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);
+  if (!parsed) return [];
+  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin);
+  const collected = [...pageUrls];
+  if (subSitemapUrls.length > 0 && collected.length < MAX_URLS) {
+    const subResults = await Promise.all(
+      subSitemapUrls.slice(0, 10).map((subUrl) => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher))
+    );
+    for (const sub of subResults) collected.push(...sub);
+  }
+  return [...new Set(collected)].slice(0, MAX_URLS);
 }
-/**
- * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an
- * explicit override) and returns the URLs it lists. Never throws.
- */
-export async function discoverSitemapUrls(appUrl, sitemapUrl) {
-    let origin;
-    try {
-        origin = new URL(appUrl).origin;
-    }
-    catch {
-        return [];
-    }
-    const target = sitemapUrl || `${origin}/sitemap.xml`;
-    try {
-        return await fetchSitemapUrls(target, origin);
-    }
-    catch (err) {
-        logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');
-        return [];
-    }
+async function discoverSitemapUrls(appUrl, sitemapUrl) {
+  let origin;
+  try {
+    origin = new URL(appUrl).origin;
+  } catch {
+    return [];
+  }
+  const target = sitemapUrl || `${origin}/sitemap.xml`;
+  try {
+    return await fetchSitemapUrls(target, origin);
+  } catch (err) {
+    logger.warn({ err: String(err), target }, "[sitemap] discovery failed, continuing without it");
+    return [];
+  }
 }
-/**
- * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because
- * DataImpulse-style usernames contain commas (country codes) that break new URL().
- * Returns undefined for no/auto proxy — the caller then fetches directly.
- */
-export function buildProxyDispatcher(proxyUrl) {
-    if (!proxyUrl || proxyUrl === 'auto')
-        return undefined;
-    try {
-        const m = proxyUrl.match(/^(https?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
-        if (!m)
-            return new ProxyAgent(proxyUrl);
-        const [, scheme, user, pass, hostPort] = m;
-        const uri = `${scheme}://${hostPort}`;
-        if (user) {
-            const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');
-            return new ProxyAgent({ uri, token });
-        }
-        return new ProxyAgent({ uri });
-    }
-    catch {
-        return undefined;
-    }
+function buildProxyDispatcher(proxyUrl) {
+  if (!proxyUrl || proxyUrl === "auto") return void 0;
+  try {
+    const m = proxyUrl.match(/^(https?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
+    if (!m) return new ProxyAgent(proxyUrl);
+    const [, scheme, user, pass, hostPort] = m;
+    const uri = `${scheme}://${hostPort}`;
+    if (user) {
+      const token = "Basic " + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? "")}`).toString("base64");
+      return new ProxyAgent({ uri, token });
+    }
+    return new ProxyAgent({ uri });
+  } catch {
+    return void 0;
+  }
 }
-/** Extract same-origin `Sitemap:` directives from a robots.txt body. */
-export function parseRobotsSitemaps(robotsTxt, origin) {
-    const out = [];
-    for (const line of robotsTxt.split(/\r?\n/)) {
-        const m = line.match(/^\s*sitemap:\s*(\S+)/i);
-        if (m && sameSite(m[1], origin))
-            out.push(m[1]);
-    }
-    return [...new Set(out)];
+function parseRobotsSitemaps(robotsTxt, origin) {
+  const out = [];
+  for (const line of robotsTxt.split(/\r?\n/)) {
+    const m = line.match(/^\s*sitemap:\s*(\S+)/i);
+    if (m && sameSite(m[1], origin)) out.push(m[1]);
+  }
+  return [...new Set(out)];
 }
-/**
- * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap
- * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped
- * candidate list the user can review and seed before launching the Playwright crawl.
- * Never throws — bot-protected sites (Akamai/Cloudflare) simply yield an empty list.
- */
-export async function discoverProjectUrls(appUrl, opts) {
-    let origin;
-    try {
-        origin = new URL(appUrl).origin;
-    }
-    catch {
-        return { candidates: [], sitemapFound: false };
-    }
-    const dispatcher = buildProxyDispatcher(opts?.proxyUrl);
-    // 1. robots.txt → declared sitemaps
-    let robotsSitemaps = [];
-    try {
-        assertCrawlUrlSafe(`${origin}/robots.txt`);
-        const res = await fetch(`${origin}/robots.txt`, {
-            signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
-            headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },
-            ...(dispatcher ? { dispatcher } : {}),
-        });
-        if (res.ok)
-            robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);
-    }
-    catch { /* best-effort */ }
-    // 2. sitemap targets: explicit override + robots-declared + conventional default
-    const robotsSet = new Set(robotsSitemaps);
-    const sitemapTargets = [...new Set([
-            ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),
-            ...robotsSitemaps,
-            `${origin}/sitemap.xml`,
-        ])];
-    const seen = new Map();
-    let sitemapFound = false;
-    for (const sm of sitemapTargets) {
-        const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);
-        if (urls.length)
-            sitemapFound = true;
-        const source = robotsSet.has(sm) ? 'robots' : 'sitemap';
-        for (const u of urls)
-            if (!seen.has(u))
-                seen.set(u, { url: u, source });
-    }
-    return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };
+async function discoverProjectUrls(appUrl, opts) {
+  let origin;
+  try {
+    origin = new URL(appUrl).origin;
+  } catch {
+    return { candidates: [], sitemapFound: false };
+  }
+  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);
+  let robotsSitemaps = [];
+  try {
+    assertCrawlUrlSafe(`${origin}/robots.txt`);
+    const res = await fetch(`${origin}/robots.txt`, {
+      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      headers: { "User-Agent": "ZeTa-AI-Crawler/1.0 (+url-discovery)" },
+      ...dispatcher ? { dispatcher } : {}
+    });
+    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);
+  } catch {
+  }
+  const robotsSet = new Set(robotsSitemaps);
+  const sitemapTargets = [.../* @__PURE__ */ new Set([
+    ...opts?.sitemapUrl ? [opts.sitemapUrl] : [],
+    ...robotsSitemaps,
+    `${origin}/sitemap.xml`
+  ])];
+  const seen = /* @__PURE__ */ new Map();
+  let sitemapFound = false;
+  for (const sm of sitemapTargets) {
+    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher);
+    if (urls.length) sitemapFound = true;
+    const source = robotsSet.has(sm) ? "robots" : "sitemap";
+    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });
+  }
+  return { candidates: [...seen.values()].slice(0, MAX_URLS), sitemapFound };
 }
+export {
+  buildProxyDispatcher,
+  discoverProjectUrls,
+  discoverSitemapUrls,
+  extractUrlsFromParsedSitemap,
+  fetchSitemapUrls,
+  parseRobotsSitemaps,
+  sameSite
+};
+//# sourceMappingURL=sitemap.js.map