@staticn0va/wigolo 0.1.0 → 0.1.2

package/dist/crawl/mapper.js

@@ -1,178 +1,150 @@
-import { parseHTML } from 'linkedom';
-import { matchesPatterns } from './url-utils.js';
-import { parseSitemap, parseSitemapIndex, extractSitemapUrlFromRobots } from './sitemap.js';
-import { createLogger } from '../logger.js';
-const log = createLogger('crawl');
-const IGNORED_PROTOCOLS = ['javascript:', 'mailto:', 'tel:', 'data:', 'blob:', 'ftp:'];
-export function extractLinks(html, origin) {
-    if (!html || !html.trim())
-        return [];
-    try {
-        const { document: doc } = parseHTML(html);
-        const anchors = doc.querySelectorAll('a[href]');
-        const seen = new Set();
-        const links = [];
-        const parsedOrigin = new URL(origin).origin;
-        for (const anchor of Array.from(anchors)) {
-            const href = anchor.getAttribute('href');
-            if (!href)
-                continue;
-            const trimmed = href.trim();
-            if (!trimmed)
-                continue;
-            // Skip fragment-only links
-            if (trimmed.startsWith('#'))
-                continue;
-            // Skip non-http protocols
-            if (IGNORED_PROTOCOLS.some((p) => trimmed.toLowerCase().startsWith(p)))
-                continue;
-            try {
-                const resolved = new URL(trimmed, origin);
-                // Same-origin check
-                if (resolved.origin !== parsedOrigin)
-                    continue;
-                // Strip fragment, keep path + query
-                resolved.hash = '';
-                const normalized = resolved.href;
-                if (!seen.has(normalized)) {
-                    seen.add(normalized);
-                    links.push(normalized);
-                }
-            }
-            catch {
-                log.debug('Failed to resolve URL', { href: trimmed, origin });
-            }
+import { parseHTML } from "linkedom";
+import { matchesPatterns } from "./url-utils.js";
+import { parseSitemap, parseSitemapIndex, extractSitemapUrlFromRobots } from "./sitemap.js";
+import { createLogger } from "../logger.js";
+const log = createLogger("crawl");
+const IGNORED_PROTOCOLS = ["javascript:", "mailto:", "tel:", "data:", "blob:", "ftp:"];
+function extractLinks(html, origin) {
+  if (!html || !html.trim()) return [];
+  try {
+    const { document: doc } = parseHTML(html);
+    const anchors = doc.querySelectorAll("a[href]");
+    const seen = /* @__PURE__ */ new Set();
+    const links = [];
+    const parsedOrigin = new URL(origin).origin;
+    for (const anchor of Array.from(anchors)) {
+      const href = anchor.getAttribute("href");
+      if (!href) continue;
+      const trimmed = href.trim();
+      if (!trimmed) continue;
+      if (trimmed.startsWith("#")) continue;
+      if (IGNORED_PROTOCOLS.some((p) => trimmed.toLowerCase().startsWith(p))) continue;
+      try {
+        const resolved = new URL(trimmed, origin);
+        if (resolved.origin !== parsedOrigin) continue;
+        resolved.hash = "";
+        const normalized = resolved.href;
+        if (!seen.has(normalized)) {
+          seen.add(normalized);
+          links.push(normalized);
         }
-        return links;
-    }
-    catch (err) {
-        log.debug('Failed to parse HTML for link extraction', { error: String(err) });
-        return [];
+      } catch {
+        log.debug("Failed to resolve URL", { href: trimmed, origin });
+      }
     }
+    return links;
+  } catch (err) {
+    log.debug("Failed to parse HTML for link extraction", { error: String(err) });
+    return [];
+  }
 }
-export async function mapUrls(input, fetchFn) {
-    const maxDepth = input.max_depth ?? 3;
-    const maxPages = input.max_pages ?? 200;
-    let origin;
-    try {
-        origin = new URL(input.url).origin;
-    }
-    catch (err) {
-        return {
-            urls: [],
-            total_found: 0,
-            sitemap_found: false,
-            error: `Invalid seed URL: ${String(err)}`,
-        };
-    }
-    const discovered = new Set([input.url]);
-    const queued = new Set([input.url]);
-    const queue = [{ url: input.url, depth: 0 }];
-    let sitemapFound = false;
-    // Phase 1: Sitemap discovery (best-effort, errors are non-fatal)
-    try {
-        const sitemapUrls = await discoverSitemapUrls(origin, fetchFn);
-        if (sitemapUrls.length > 0) {
-            sitemapFound = true;
-            for (const sitemapUrl of sitemapUrls) {
-                if (discovered.size >= maxPages)
-                    break;
-                if (matchesPatterns(sitemapUrl, input.include_patterns, input.exclude_patterns)) {
-                    discovered.add(sitemapUrl);
-                    // Also queue sitemap URLs for BFS traversal so their links are explored
-                    if (!queued.has(sitemapUrl)) {
-                        queued.add(sitemapUrl);
-                        queue.push({ url: sitemapUrl, depth: 0 });
-                    }
-                }
-            }
+async function mapUrls(input, fetchFn) {
+  const maxDepth = input.max_depth ?? 3;
+  const maxPages = input.max_pages ?? 200;
+  let origin;
+  try {
+    origin = new URL(input.url).origin;
+  } catch (err) {
+    return {
+      urls: [],
+      total_found: 0,
+      sitemap_found: false,
+      error: `Invalid seed URL: ${String(err)}`
+    };
+  }
+  const discovered = /* @__PURE__ */ new Set([input.url]);
+  const queued = /* @__PURE__ */ new Set([input.url]);
+  const queue = [{ url: input.url, depth: 0 }];
+  let sitemapFound = false;
+  try {
+    const sitemapUrls = await discoverSitemapUrls(origin, fetchFn);
+    if (sitemapUrls.length > 0) {
+      sitemapFound = true;
+      for (const sitemapUrl of sitemapUrls) {
+        if (discovered.size >= maxPages) break;
+        if (matchesPatterns(sitemapUrl, input.include_patterns, input.exclude_patterns)) {
+          discovered.add(sitemapUrl);
+          if (!queued.has(sitemapUrl)) {
+            queued.add(sitemapUrl);
+            queue.push({ url: sitemapUrl, depth: 0 });
+          }
         }
+      }
     }
-    catch (err) {
-        log.debug('Sitemap discovery failed, continuing with BFS only', { error: String(err) });
-    }
-    // Phase 2: BFS link traversal
-    let seedError;
-    while (queue.length > 0 && discovered.size < maxPages) {
-        const current = queue.shift();
-        if (current.depth > maxDepth)
-            continue;
-        try {
-            const { html } = await fetchFn(current.url);
-            const links = extractLinks(html, origin);
-            for (const link of links) {
-                if (discovered.size >= maxPages)
-                    break;
-                if (discovered.has(link))
-                    continue;
-                if (!matchesPatterns(link, input.include_patterns, input.exclude_patterns))
-                    continue;
-                discovered.add(link);
-                // Only queue for further traversal if we haven't hit max depth
-                if (current.depth + 1 <= maxDepth && !queued.has(link)) {
-                    queued.add(link);
-                    queue.push({ url: link, depth: current.depth + 1 });
-                }
-            }
-        }
-        catch (err) {
-            if (current.url === input.url && current.depth === 0) {
-                seedError = String(err);
-                log.warn('Seed URL fetch failed during map', { url: current.url, error: String(err) });
-            }
-            else {
-                log.debug('Child page fetch failed during map, skipping', { url: current.url, error: String(err) });
-            }
+  } catch (err) {
+    log.debug("Sitemap discovery failed, continuing with BFS only", { error: String(err) });
+  }
+  let seedError;
+  while (queue.length > 0 && discovered.size < maxPages) {
+    const current = queue.shift();
+    if (current.depth > maxDepth) continue;
+    try {
+      const { html } = await fetchFn(current.url);
+      const links = extractLinks(html, origin);
+      for (const link of links) {
+        if (discovered.size >= maxPages) break;
+        if (discovered.has(link)) continue;
+        if (!matchesPatterns(link, input.include_patterns, input.exclude_patterns)) continue;
+        discovered.add(link);
+        if (current.depth + 1 <= maxDepth && !queued.has(link)) {
+          queued.add(link);
+          queue.push({ url: link, depth: current.depth + 1 });
         }
+      }
+    } catch (err) {
+      if (current.url === input.url && current.depth === 0) {
+        seedError = String(err);
+        log.warn("Seed URL fetch failed during map", { url: current.url, error: String(err) });
+      } else {
+        log.debug("Child page fetch failed during map, skipping", { url: current.url, error: String(err) });
+      }
     }
-    const urls = Array.from(discovered);
-    return {
-        urls,
-        total_found: urls.length,
-        sitemap_found: sitemapFound,
-        ...(seedError !== undefined ? { error: seedError } : {}),
-    };
+  }
+  const urls = Array.from(discovered);
+  return {
+    urls,
+    total_found: urls.length,
+    sitemap_found: sitemapFound,
+    ...seedError !== void 0 ? { error: seedError } : {}
+  };
 }
 async function discoverSitemapUrls(origin, fetchFn) {
-    const sitemapLocations = [];
-    // Try robots.txt first for Sitemap directives
+  const sitemapLocations = [];
+  try {
+    const { html: robotsTxt } = await fetchFn(`${origin}/robots.txt`);
+    const fromRobots = extractSitemapUrlFromRobots(robotsTxt);
+    sitemapLocations.push(...fromRobots);
+  } catch {
+    log.debug("robots.txt not found, trying default sitemap location");
+  }
+  if (sitemapLocations.length === 0) {
+    sitemapLocations.push(`${origin}/sitemap.xml`);
+  }
+  const allUrls = [];
+  for (const sitemapUrl of sitemapLocations) {
     try {
-        const { html: robotsTxt } = await fetchFn(`${origin}/robots.txt`);
-        const fromRobots = extractSitemapUrlFromRobots(robotsTxt);
-        sitemapLocations.push(...fromRobots);
-    }
-    catch {
-        log.debug('robots.txt not found, trying default sitemap location');
-    }
-    // Fallback to default /sitemap.xml
-    if (sitemapLocations.length === 0) {
-        sitemapLocations.push(`${origin}/sitemap.xml`);
-    }
-    const allUrls = [];
-    for (const sitemapUrl of sitemapLocations) {
-        try {
-            const { html: sitemapXml } = await fetchFn(sitemapUrl);
-            // Check if it's a sitemap index (contains sub-sitemaps)
-            const indexUrls = parseSitemapIndex(sitemapXml);
-            if (indexUrls.length > 0) {
-                for (const subUrl of indexUrls) {
-                    try {
-                        const { html: subXml } = await fetchFn(subUrl);
-                        allUrls.push(...parseSitemap(subXml));
-                    }
-                    catch {
-                        log.debug('Failed to fetch sub-sitemap', { url: subUrl });
-                    }
-                }
-            }
-            else {
-                allUrls.push(...parseSitemap(sitemapXml));
-            }
-        }
-        catch {
-            log.debug('Sitemap fetch failed', { url: sitemapUrl });
+      const { html: sitemapXml } = await fetchFn(sitemapUrl);
+      const indexUrls = parseSitemapIndex(sitemapXml);
+      if (indexUrls.length > 0) {
+        for (const subUrl of indexUrls) {
+          try {
+            const { html: subXml } = await fetchFn(subUrl);
+            allUrls.push(...parseSitemap(subXml));
+          } catch {
+            log.debug("Failed to fetch sub-sitemap", { url: subUrl });
+          }
         }
+      } else {
+        allUrls.push(...parseSitemap(sitemapXml));
+      }
+    } catch {
+      log.debug("Sitemap fetch failed", { url: sitemapUrl });
     }
-    return allUrls;
+  }
+  return allUrls;
 }
+export {
+  extractLinks,
+  mapUrls
+};
 //# sourceMappingURL=mapper.js.map

package/dist/crawl/mapper.js.map

@@ -1 +1 @@
-{"version":3,"file":"mapper.js","sourceRoot":"","sources":["../../src/crawl/mapper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,MAAM,cAAc,CAAC;AAC5F,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAYlC,MAAM,iBAAiB,GAAG,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AAEvF,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,MAAc;IACvD,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAErC,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,GAAG,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAE5C,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,2BAA2B;YAC3B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEtC,0BAA0B;YAC1B,IAAI,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBAAE,SAAS;YAEjF,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAE1C,oBAAoB;gBACpB,IAAI,QAAQ,CAAC,MAAM,KAAK,YAAY;oBAAE,SAAS;gBAE/C,oCAAoC;gBACpC,QAAQ,CAAC,IAAI,GAAG,EAAE,CAAC;gBACnB,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC;gBAEjC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC1B,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;oBACrB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,GAAG,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,0CAA0C,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9E,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAe,EAAE,OAAqB;IAClE,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,IAAI,GAAG,CAAC;IAExC,IAAI,MAAc,CAAC;IACnB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,EAAE;YACR,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,KAAK;YACpB,KAAK,EAAE,qBAAqB,MAAM,CAAC,GAAG,CAAC,EAAE;SAC1C,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5C,MAAM,KAAK,GAA0C,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IACpF,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,iEAAiE;IACjE,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC/D,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,IAAI,UAAU,CAAC,IAAI,IAAI,QAAQ;oBAAE,MAAM;gBACvC,IAAI,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC;oBAChF,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;oBAC3B,wEAAwE;oBACxE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;wBAC5B,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;wBACvB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC5C,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,oDAAoD,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC1F,CAAC;IAED,8BAA8B;IAC9B,IAAI,SAA6B,CAAC;IAElC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;QACtD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;QAE/B,IAAI,OAAO,CAAC,KAAK,GAAG,QAAQ;YAAE,SAAS;QAEvC,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,UAAU,CAAC,IAAI,IAAI,QAAQ;oBAAE,MAAM;gBACvC,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC;oBAAE,SAAS;gBACnC,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,CAAC;oBAAE,SAAS;gBAErF,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAErB,+DAA+D;gBAC/D,IAAI,OAAO,CAAC,KAAK,GAAG,CAAC,IAAI,QAAQ,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACvD,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACjB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;gBACtD,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,OAAO,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;gBACrD,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;gBACxB,GAAG,CAAC,IAAI,CAAC,kCAAkC,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzF,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,KAAK,CAAC,8CAA8C,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtG,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEpC,OAAO;QACL,IAAI;QACJ,WAAW,EAAE,IAAI,CAAC,MAAM;QACxB,aAAa,EAAE,YAAY;QAC3B,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACzD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,MAAc,EAAE,OAAqB;IACtE,MAAM,gBAAgB,GAAa,EAAE,CAAC;IAEtC,8CAA8C;IAC9C,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,MAAM,aAAa,CAAC,CAAC;QAClE,MAAM,UAAU,GAAG,2BAA2B,CAAC,SAAS,CAAC,CAAC;QAC1D,gBAAgB,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,GAAG,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;IACrE,CAAC;IAED,mCAAmC;IACnC,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClC,gBAAgB,CAAC,IAAI,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,UAAU,IAAI,gBAAgB,EAAE,CAAC;QAC1C,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;YAEvD,wDAAwD;YACxD,MAAM,SAAS,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;YAChD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;oBAC/B,IAAI,CAAC;wBACH,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;wBAC/C,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;oBACxC,CAAC;oBAAC,MAAM,CAAC;wBACP,GAAG,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;oBAC5D,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,GAAG,CAAC,KAAK,CAAC,sBAAsB,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"sources":["../../src/crawl/mapper.ts"],"sourcesContent":["import { parseHTML } from 'linkedom';\nimport { matchesPatterns } from './url-utils.js';\nimport { parseSitemap, parseSitemapIndex, extractSitemapUrlFromRobots } from './sitemap.js';\nimport { createLogger } from '../logger.js';\nimport type { MapOutput } from '../types.js';\n\nconst log = createLogger('crawl');\n\ninterface MapInput {\n  url: string;\n  max_depth?: number;\n  max_pages?: number;\n  include_patterns?: string[];\n  exclude_patterns?: string[];\n}\n\nexport type LightFetchFn = (url: string) => Promise<{ html: string; finalUrl: string; statusCode: number }>;\n\nconst IGNORED_PROTOCOLS = ['javascript:', 'mailto:', 'tel:', 'data:', 'blob:', 'ftp:'];\n\nexport function extractLinks(html: string, origin: string): string[] {\n  if (!html || !html.trim()) return [];\n\n  try {\n    const { document: doc } = parseHTML(html);\n    const anchors = doc.querySelectorAll('a[href]');\n    const seen = new Set<string>();\n    const links: string[] = [];\n    const parsedOrigin = new URL(origin).origin;\n\n    for (const anchor of Array.from(anchors)) {\n      const href = anchor.getAttribute('href');\n      if (!href) continue;\n\n      const trimmed = href.trim();\n      if (!trimmed) continue;\n\n      // Skip fragment-only links\n      if (trimmed.startsWith('#')) continue;\n\n      // Skip non-http protocols\n      if (IGNORED_PROTOCOLS.some((p) => trimmed.toLowerCase().startsWith(p))) continue;\n\n      try {\n        const resolved = new URL(trimmed, origin);\n\n        // Same-origin check\n        if (resolved.origin !== parsedOrigin) continue;\n\n        // Strip fragment, keep path + query\n        resolved.hash = '';\n        const normalized = resolved.href;\n\n        if (!seen.has(normalized)) {\n          seen.add(normalized);\n          links.push(normalized);\n        }\n      } catch {\n        log.debug('Failed to resolve URL', { href: trimmed, origin });\n      }\n    }\n\n    return links;\n  } catch (err) {\n    log.debug('Failed to parse HTML for link extraction', { error: String(err) });\n    return [];\n  }\n}\n\nexport async function mapUrls(input: MapInput, fetchFn: LightFetchFn): Promise<MapOutput> {\n  const maxDepth = input.max_depth ?? 3;\n  const maxPages = input.max_pages ?? 200;\n\n  let origin: string;\n  try {\n    origin = new URL(input.url).origin;\n  } catch (err) {\n    return {\n      urls: [],\n      total_found: 0,\n      sitemap_found: false,\n      error: `Invalid seed URL: ${String(err)}`,\n    };\n  }\n\n  const discovered = new Set<string>([input.url]);\n  const queued = new Set<string>([input.url]);\n  const queue: Array<{ url: string; depth: number }> = [{ url: input.url, depth: 0 }];\n  let sitemapFound = false;\n\n  // Phase 1: Sitemap discovery (best-effort, errors are non-fatal)\n  try {\n    const sitemapUrls = await discoverSitemapUrls(origin, fetchFn);\n    if (sitemapUrls.length > 0) {\n      sitemapFound = true;\n      for (const sitemapUrl of sitemapUrls) {\n        if (discovered.size >= maxPages) break;\n        if (matchesPatterns(sitemapUrl, input.include_patterns, input.exclude_patterns)) {\n          discovered.add(sitemapUrl);\n          // Also queue sitemap URLs for BFS traversal so their links are explored\n          if (!queued.has(sitemapUrl)) {\n            queued.add(sitemapUrl);\n            queue.push({ url: sitemapUrl, depth: 0 });\n          }\n        }\n      }\n    }\n  } catch (err) {\n    log.debug('Sitemap discovery failed, continuing with BFS only', { error: String(err) });\n  }\n\n  // Phase 2: BFS link traversal\n  let seedError: string | undefined;\n\n  while (queue.length > 0 && discovered.size < maxPages) {\n    const current = queue.shift()!;\n\n    if (current.depth > maxDepth) continue;\n\n    try {\n      const { html } = await fetchFn(current.url);\n      const links = extractLinks(html, origin);\n\n      for (const link of links) {\n        if (discovered.size >= maxPages) break;\n        if (discovered.has(link)) continue;\n        if (!matchesPatterns(link, input.include_patterns, input.exclude_patterns)) continue;\n\n        discovered.add(link);\n\n        // Only queue for further traversal if we haven't hit max depth\n        if (current.depth + 1 <= maxDepth && !queued.has(link)) {\n          queued.add(link);\n          queue.push({ url: link, depth: current.depth + 1 });\n        }\n      }\n    } catch (err) {\n      if (current.url === input.url && current.depth === 0) {\n        seedError = String(err);\n        log.warn('Seed URL fetch failed during map', { url: current.url, error: String(err) });\n      } else {\n        log.debug('Child page fetch failed during map, skipping', { url: current.url, error: String(err) });\n      }\n    }\n  }\n\n  const urls = Array.from(discovered);\n\n  return {\n    urls,\n    total_found: urls.length,\n    sitemap_found: sitemapFound,\n    ...(seedError !== undefined ? { error: seedError } : {}),\n  };\n}\n\nasync function discoverSitemapUrls(origin: string, fetchFn: LightFetchFn): Promise<string[]> {\n  const sitemapLocations: string[] = [];\n\n  // Try robots.txt first for Sitemap directives\n  try {\n    const { html: robotsTxt } = await fetchFn(`${origin}/robots.txt`);\n    const fromRobots = extractSitemapUrlFromRobots(robotsTxt);\n    sitemapLocations.push(...fromRobots);\n  } catch {\n    log.debug('robots.txt not found, trying default sitemap location');\n  }\n\n  // Fallback to default /sitemap.xml\n  if (sitemapLocations.length === 0) {\n    sitemapLocations.push(`${origin}/sitemap.xml`);\n  }\n\n  const allUrls: string[] = [];\n\n  for (const sitemapUrl of sitemapLocations) {\n    try {\n      const { html: sitemapXml } = await fetchFn(sitemapUrl);\n\n      // Check if it's a sitemap index (contains sub-sitemaps)\n      const indexUrls = parseSitemapIndex(sitemapXml);\n      if (indexUrls.length > 0) {\n        for (const subUrl of indexUrls) {\n          try {\n            const { html: subXml } = await fetchFn(subUrl);\n            allUrls.push(...parseSitemap(subXml));\n          } catch {\n            log.debug('Failed to fetch sub-sitemap', { url: subUrl });\n          }\n        }\n      } else {\n        allUrls.push(...parseSitemap(sitemapXml));\n      }\n    } catch {\n      log.debug('Sitemap fetch failed', { url: sitemapUrl });\n    }\n  }\n\n  return allUrls;\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,uBAAuB;AAChC,SAAS,cAAc,mBAAmB,mCAAmC;AAC7E,SAAS,oBAAoB;AAG7B,MAAM,MAAM,aAAa,OAAO;AAYhC,MAAM,oBAAoB,CAAC,eAAe,WAAW,QAAQ,SAAS,SAAS,MAAM;AAE9E,SAAS,aAAa,MAAc,QAA0B;AACnE,MAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,EAAG,QAAO,CAAC;AAEnC,MAAI;AACF,UAAM,EAAE,UAAU,IAAI,IAAI,UAAU,IAAI;AACxC,UAAM,UAAU,IAAI,iBAAiB,SAAS;AAC9C,UAAM,OAAO,oBAAI,IAAY;AAC7B,UAAM,QAAkB,CAAC;AACzB,UAAM,eAAe,IAAI,IAAI,MAAM,EAAE;AAErC,eAAW,UAAU,MAAM,KAAK,OAAO,GAAG;AACxC,YAAM,OAAO,OAAO,aAAa,MAAM;AACvC,UAAI,CAAC,KAAM;AAEX,YAAM,UAAU,KAAK,KAAK;AAC1B,UAAI,CAAC,QAAS;AAGd,UAAI,QAAQ,WAAW,GAAG,EAAG;AAG7B,UAAI,kBAAkB,KAAK,CAAC,MAAM,QAAQ,YAAY,EAAE,WAAW,CAAC,CAAC,EAAG;AAExE,UAAI;AACF,cAAM,WAAW,IAAI,IAAI,SAAS,MAAM;AAGxC,YAAI,SAAS,WAAW,aAAc;AAGtC,iBAAS,OAAO;AAChB,cAAM,aAAa,SAAS;AAE5B,YAAI,CAAC,KAAK,IAAI,UAAU,GAAG;AACzB,eAAK,IAAI,UAAU;AACnB,gBAAM,KAAK,UAAU;AAAA,QACvB;AAAA,MACF,QAAQ;AACN,YAAI,MAAM,yBAAyB,EAAE,MAAM,SAAS,OAAO,CAAC;AAAA,MAC9D;AAAA,IACF;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,MAAM,4CAA4C,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAC5E,WAAO,CAAC;AAAA,EACV;AACF;AAEA,eAAsB,QAAQ,OAAiB,SAA2C;AACxF,QAAM,WAAW,MAAM,aAAa;AACpC,QAAM,WAAW,MAAM,aAAa;AAEpC,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM,GAAG,EAAE;AAAA,EAC9B,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,MAAM,CAAC;AAAA,MACP,aAAa;AAAA,MACb,eAAe;AAAA,MACf,OAAO,qBAAqB,OAAO,GAAG,CAAC;AAAA,IACzC;AAAA,EACF;AAEA,QAAM,aAAa,oBAAI,IAAY,CAAC,MAAM,GAAG,CAAC;AAC9C,QAAM,SAAS,oBAAI,IAAY,CAAC,MAAM,GAAG,CAAC;AAC1C,QAAM,QAA+C,CAAC,EAAE,KAAK,MAAM,KAAK,OAAO,EAAE,CAAC;AAClF,MAAI,eAAe;AAGnB,MAAI;AACF,UAAM,cAAc,MAAM,oBAAoB,QAAQ,OAAO;AAC7D,QAAI,YAAY,SAAS,GAAG;AAC1B,qBAAe;AACf,iBAAW,cAAc,aAAa;AACpC,YAAI,WAAW,QAAQ,SAAU;AACjC,YAAI,gBAAgB,YAAY,MAAM,kBAAkB,MAAM,gBAAgB,GAAG;AAC/E,qBAAW,IAAI,UAAU;AAEzB,cAAI,CAAC,OAAO,IAAI,UAAU,GAAG;AAC3B,mBAAO,IAAI,UAAU;AACrB,kBAAM,KAAK,EAAE,KAAK,YAAY,OAAO,EAAE,CAAC;AAAA,UAC1C;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,MAAM,sDAAsD,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,EACxF;AAGA,MAAI;AAEJ,SAAO,MAAM,SAAS,KAAK,WAAW,OAAO,UAAU;AACrD,UAAM,UAAU,MAAM,MAAM;AAE5B,QAAI,QAAQ,QAAQ,SAAU;AAE9B,QAAI;AACF,YAAM,EAAE,KAAK,IAAI,MAAM,QAAQ,QAAQ,GAAG;AAC1C,YAAM,QAAQ,aAAa,MAAM,MAAM;AAEvC,iBAAW,QAAQ,OAAO;AACxB,YAAI,WAAW,QAAQ,SAAU;AACjC,YAAI,WAAW,IAAI,IAAI,EAAG;AAC1B,YAAI,CAAC,gBAAgB,MAAM,MAAM,kBAAkB,MAAM,gBAAgB,EAAG;AAE5E,mBAAW,IAAI,IAAI;AAGnB,YAAI,QAAQ,QAAQ,KAAK,YAAY,CAAC,OAAO,IAAI,IAAI,GAAG;AACtD,iBAAO,IAAI,IAAI;AACf,gBAAM,KAAK,EAAE,KAAK,MAAM,OAAO,QAAQ,QAAQ,EAAE,CAAC;AAAA,QACpD;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AACZ,UAAI,QAAQ,QAAQ,MAAM,OAAO,QAAQ,UAAU,GAAG;AACpD,oBAAY,OAAO,GAAG;AACtB,YAAI,KAAK,oCAAoC,EAAE,KAAK,QAAQ,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,MACvF,OAAO;AACL,YAAI,MAAM,gDAAgD,EAAE,KAAK,QAAQ,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,MACpG;AAAA,IACF;AAAA,EACF;AAEA,QAAM,OAAO,MAAM,KAAK,UAAU;AAElC,SAAO;AAAA,IACL;AAAA,IACA,aAAa,KAAK;AAAA,IAClB,eAAe;AAAA,IACf,GAAI,cAAc,SAAY,EAAE,OAAO,UAAU,IAAI,CAAC;AAAA,EACxD;AACF;AAEA,eAAe,oBAAoB,QAAgB,SAA0C;AAC3F,QAAM,mBAA6B,CAAC;AAGpC,MAAI;AACF,UAAM,EAAE,MAAM,UAAU,IAAI,MAAM,QAAQ,GAAG,MAAM,aAAa;AAChE,UAAM,aAAa,4BAA4B,SAAS;AACxD,qBAAiB,KAAK,GAAG,UAAU;AAAA,EACrC,QAAQ;AACN,QAAI,MAAM,uDAAuD;AAAA,EACnE;AAGA,MAAI,iBAAiB,WAAW,GAAG;AACjC,qBAAiB,KAAK,GAAG,MAAM,cAAc;AAAA,EAC/C;AAEA,QAAM,UAAoB,CAAC;AAE3B,aAAW,cAAc,kBAAkB;AACzC,QAAI;AACF,YAAM,EAAE,MAAM,WAAW,IAAI,MAAM,QAAQ,UAAU;AAGrD,YAAM,YAAY,kBAAkB,UAAU;AAC9C,UAAI,UAAU,SAAS,GAAG;AACxB,mBAAW,UAAU,WAAW;AAC9B,cAAI;AACF,kBAAM,EAAE,MAAM,OAAO,IAAI,MAAM,QAAQ,MAAM;AAC7C,oBAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAAA,UACtC,QAAQ;AACN,gBAAI,MAAM,+BAA+B,EAAE,KAAK,OAAO,CAAC;AAAA,UAC1D;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,KAAK,GAAG,aAAa,UAAU,CAAC;AAAA,MAC1C;AAAA,IACF,QAAQ;AACN,UAAI,MAAM,wBAAwB,EAAE,KAAK,WAAW,CAAC;AAAA,IACvD;AAAA,EACF;AAEA,SAAO;AACT;","names":[]}

package/dist/crawl/rate-limiter.js

@@ -1,72 +1,69 @@
-import { isPrivateUrl } from './url-utils.js';
-import { getConfig } from '../config.js';
-export class RateLimiter {
-    domains = new Map();
-    robotsDelays = new Map();
-    setRobotsCrawlDelay(domain, delaySeconds) {
-        this.robotsDelays.set(domain, delaySeconds * 1000);
+import { isPrivateUrl } from "./url-utils.js";
+import { getConfig } from "../config.js";
+class RateLimiter {
+  domains = /* @__PURE__ */ new Map();
+  robotsDelays = /* @__PURE__ */ new Map();
+  setRobotsCrawlDelay(domain, delaySeconds) {
+    this.robotsDelays.set(domain, delaySeconds * 1e3);
+  }
+  async acquire(url) {
+    const domain = new URL(url).hostname;
+    const state = this.getOrCreateState(url, domain);
+    if (state.activeCount < state.maxConcurrency) {
+      const elapsed = Date.now() - state.lastRequestTime;
+      const remaining = state.delayMs - elapsed;
+      if (remaining > 0 && state.lastRequestTime > 0) {
+        await new Promise((r) => setTimeout(r, remaining));
+      }
+      return this.startRequest(state);
     }
-    async acquire(url) {
-        const domain = new URL(url).hostname;
-        const state = this.getOrCreateState(url, domain);
-        if (state.activeCount < state.maxConcurrency) {
-            // Enforce delay even when under concurrency limit
-            const elapsed = Date.now() - state.lastRequestTime;
-            const remaining = state.delayMs - elapsed;
-            if (remaining > 0 && state.lastRequestTime > 0) {
-                await new Promise((r) => setTimeout(r, remaining));
-            }
-            return this.startRequest(state);
-        }
-        // Wait in queue
-        return new Promise((resolve) => {
-            state.queue.push(() => resolve(this.startRequest(state)));
-        });
+    return new Promise((resolve) => {
+      state.queue.push(() => resolve(this.startRequest(state)));
+    });
+  }
+  getOrCreateState(url, domain) {
+    if (!this.domains.has(domain)) {
+      const config = getConfig();
+      const isPrivate = isPrivateUrl(url);
+      const configDelay = isPrivate ? config.crawlPrivateDelayMs : config.crawlDelayMs;
+      const robotsDelay2 = this.robotsDelays.get(domain) ?? 0;
+      const effectiveDelay = Math.max(configDelay, robotsDelay2);
+      this.domains.set(domain, {
+        activeCount: 0,
+        lastRequestTime: 0,
+        queue: [],
+        maxConcurrency: isPrivate ? config.crawlPrivateConcurrency : config.crawlConcurrency,
+        delayMs: effectiveDelay
+      });
     }
-    getOrCreateState(url, domain) {
-        if (!this.domains.has(domain)) {
-            const config = getConfig();
-            const isPrivate = isPrivateUrl(url);
-            const configDelay = isPrivate ? config.crawlPrivateDelayMs : config.crawlDelayMs;
-            // Use robots.txt delay if it's higher than configured delay
-            const robotsDelay = this.robotsDelays.get(domain) ?? 0;
-            const effectiveDelay = Math.max(configDelay, robotsDelay);
-            this.domains.set(domain, {
-                activeCount: 0,
-                lastRequestTime: 0,
-                queue: [],
-                maxConcurrency: isPrivate ? config.crawlPrivateConcurrency : config.crawlConcurrency,
-                delayMs: effectiveDelay,
-            });
-        }
-        const state = this.domains.get(domain);
-        // Update delay if robots delay was set after state creation
-        const robotsDelay = this.robotsDelays.get(domain);
-        if (robotsDelay !== undefined && robotsDelay > state.delayMs) {
-            state.delayMs = robotsDelay;
-        }
-        return state;
+    const state = this.domains.get(domain);
+    const robotsDelay = this.robotsDelays.get(domain);
+    if (robotsDelay !== void 0 && robotsDelay > state.delayMs) {
+      state.delayMs = robotsDelay;
     }
-    startRequest(state) {
-        state.activeCount++;
-        state.lastRequestTime = Date.now();
-        return () => {
-            state.activeCount--;
-            this.processQueue(state);
-        };
-    }
-    processQueue(state) {
-        if (state.queue.length === 0 || state.activeCount >= state.maxConcurrency)
-            return;
-        const next = state.queue.shift();
-        const elapsed = Date.now() - state.lastRequestTime;
-        const remaining = state.delayMs - elapsed;
-        if (remaining <= 0) {
-            next();
-        }
-        else {
-            setTimeout(next, remaining);
-        }
+    return state;
+  }
+  startRequest(state) {
+    state.activeCount++;
+    state.lastRequestTime = Date.now();
+    return () => {
+      state.activeCount--;
+      this.processQueue(state);
+    };
+  }
+  processQueue(state) {
+    if (state.queue.length === 0 || state.activeCount >= state.maxConcurrency) return;
+    const next = state.queue.shift();
+    const elapsed = Date.now() - state.lastRequestTime;
+    const remaining = state.delayMs - elapsed;
+    if (remaining <= 0) {
+      next();
+    } else {
+      setTimeout(next, remaining);
     }
+  }
 }
+export {
+  RateLimiter
+};
 //# sourceMappingURL=rate-limiter.js.map

package/dist/crawl/rate-limiter.js.map

@@ -1 +1 @@
-{"version":3,"file":"rate-limiter.js","sourceRoot":"","sources":["../../src/crawl/rate-limiter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAUzC,MAAM,OAAO,WAAW;IACd,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IACzC,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEjD,mBAAmB,CAAC,MAAc,EAAE,YAAoB;QACtD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAW;QACvB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAEjD,IAAI,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,cAAc,EAAE,CAAC;YAC7C,kDAAkD;YAClD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,eAAe,CAAC;YACnD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;YAC1C,IAAI,SAAS,GAAG,CAAC,IAAI,KAAK,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;gBAC/C,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;QAED,gBAAgB;QAChB,OAAO,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,EAAE;YACzC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,gBAAgB,CAAC,GAAW,EAAE,MAAc;QAClD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YACpC,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;YAEjF,4DAA4D;YAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACvD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YAE1D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;gBACvB,WAAW,EAAE,CAAC;gBACd,eAAe,EAAE,CAAC;gBAClB,KAAK,EAAE,EAAE;gBACT,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,uBAAuB,CAAC,CAAC,CAAC,MAAM,CAAC,gBAAgB;gBACpF,OAAO,EAAE,cAAc;aACxB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC;QACxC,4DAA4D;QAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;YAC7D,KAAK,CAAC,OAAO,GAAG,WAAW,CAAC;QAC9B,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,YAAY,CAAC,KAAkB;QACrC,KAAK,CAAC,WAAW,EAAE,CAAC;QACpB,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEnC,OAAO,GAAG,EAAE;YACV,KAAK,CAAC,WAAW,EAAE,CAAC;YACpB,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,KAAkB;QACrC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,cAAc;YAAE,OAAO;QAElF,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,eAAe,CAAC;QACnD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;QAE1C,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YACnB,IAAI,EAAE,CAAC;QACT,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;CACF"}
+{"version":3,"sources":["../../src/crawl/rate-limiter.ts"],"sourcesContent":["import { isPrivateUrl } from './url-utils.js';\nimport { getConfig } from '../config.js';\n\ninterface DomainState {\n  activeCount: number;\n  lastRequestTime: number;\n  queue: Array<() => void>;\n  maxConcurrency: number;\n  delayMs: number;\n}\n\nexport class RateLimiter {\n  private domains = new Map<string, DomainState>();\n  private robotsDelays = new Map<string, number>();\n\n  setRobotsCrawlDelay(domain: string, delaySeconds: number): void {\n    this.robotsDelays.set(domain, delaySeconds * 1000);\n  }\n\n  async acquire(url: string): Promise<() => void> {\n    const domain = new URL(url).hostname;\n    const state = this.getOrCreateState(url, domain);\n\n    if (state.activeCount < state.maxConcurrency) {\n      // Enforce delay even when under concurrency limit\n      const elapsed = Date.now() - state.lastRequestTime;\n      const remaining = state.delayMs - elapsed;\n      if (remaining > 0 && state.lastRequestTime > 0) {\n        await new Promise<void>((r) => setTimeout(r, remaining));\n      }\n      return this.startRequest(state);\n    }\n\n    // Wait in queue\n    return new Promise<() => void>((resolve) => {\n      state.queue.push(() => resolve(this.startRequest(state)));\n    });\n  }\n\n  private getOrCreateState(url: string, domain: string): DomainState {\n    if (!this.domains.has(domain)) {\n      const config = getConfig();\n      const isPrivate = isPrivateUrl(url);\n      const configDelay = isPrivate ? config.crawlPrivateDelayMs : config.crawlDelayMs;\n\n      // Use robots.txt delay if it's higher than configured delay\n      const robotsDelay = this.robotsDelays.get(domain) ?? 0;\n      const effectiveDelay = Math.max(configDelay, robotsDelay);\n\n      this.domains.set(domain, {\n        activeCount: 0,\n        lastRequestTime: 0,\n        queue: [],\n        maxConcurrency: isPrivate ? config.crawlPrivateConcurrency : config.crawlConcurrency,\n        delayMs: effectiveDelay,\n      });\n    }\n\n    const state = this.domains.get(domain)!;\n    // Update delay if robots delay was set after state creation\n    const robotsDelay = this.robotsDelays.get(domain);\n    if (robotsDelay !== undefined && robotsDelay > state.delayMs) {\n      state.delayMs = robotsDelay;\n    }\n\n    return state;\n  }\n\n  private startRequest(state: DomainState): () => void {\n    state.activeCount++;\n    state.lastRequestTime = Date.now();\n\n    return () => {\n      state.activeCount--;\n      this.processQueue(state);\n    };\n  }\n\n  private processQueue(state: DomainState): void {\n    if (state.queue.length === 0 || state.activeCount >= state.maxConcurrency) return;\n\n    const next = state.queue.shift()!;\n    const elapsed = Date.now() - state.lastRequestTime;\n    const remaining = state.delayMs - elapsed;\n\n    if (remaining <= 0) {\n      next();\n    } else {\n      setTimeout(next, remaining);\n    }\n  }\n}\n"],"mappings":"AAAA,SAAS,oBAAoB;AAC7B,SAAS,iBAAiB;AAUnB,MAAM,YAAY;AAAA,EACf,UAAU,oBAAI,IAAyB;AAAA,EACvC,eAAe,oBAAI,IAAoB;AAAA,EAE/C,oBAAoB,QAAgB,cAA4B;AAC9D,SAAK,aAAa,IAAI,QAAQ,eAAe,GAAI;AAAA,EACnD;AAAA,EAEA,MAAM,QAAQ,KAAkC;AAC9C,UAAM,SAAS,IAAI,IAAI,GAAG,EAAE;AAC5B,UAAM,QAAQ,KAAK,iBAAiB,KAAK,MAAM;AAE/C,QAAI,MAAM,cAAc,MAAM,gBAAgB;AAE5C,YAAM,UAAU,KAAK,IAAI,IAAI,MAAM;AACnC,YAAM,YAAY,MAAM,UAAU;AAClC,UAAI,YAAY,KAAK,MAAM,kBAAkB,GAAG;AAC9C,cAAM,IAAI,QAAc,CAAC,MAAM,WAAW,GAAG,SAAS,CAAC;AAAA,MACzD;AACA,aAAO,KAAK,aAAa,KAAK;AAAA,IAChC;AAGA,WAAO,IAAI,QAAoB,CAAC,YAAY;AAC1C,YAAM,MAAM,KAAK,MAAM,QAAQ,KAAK,aAAa,KAAK,CAAC,CAAC;AAAA,IAC1D,CAAC;AAAA,EACH;AAAA,EAEQ,iBAAiB,KAAa,QAA6B;AACjE,QAAI,CAAC,KAAK,QAAQ,IAAI,MAAM,GAAG;AAC7B,YAAM,SAAS,UAAU;AACzB,YAAM,YAAY,aAAa,GAAG;AAClC,YAAM,cAAc,YAAY,OAAO,sBAAsB,OAAO;AAGpE,YAAMA,eAAc,KAAK,aAAa,IAAI,MAAM,KAAK;AACrD,YAAM,iBAAiB,KAAK,IAAI,aAAaA,YAAW;AAExD,WAAK,QAAQ,IAAI,QAAQ;AAAA,QACvB,aAAa;AAAA,QACb,iBAAiB;AAAA,QACjB,OAAO,CAAC;AAAA,QACR,gBAAgB,YAAY,OAAO,0BAA0B,OAAO;AAAA,QACpE,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AAEA,UAAM,QAAQ,KAAK,QAAQ,IAAI,MAAM;AAErC,UAAM,cAAc,KAAK,aAAa,IAAI,MAAM;AAChD,QAAI,gBAAgB,UAAa,cAAc,MAAM,SAAS;AAC5D,YAAM,UAAU;AAAA,IAClB;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,aAAa,OAAgC;AACnD,UAAM;AACN,UAAM,kBAAkB,KAAK,IAAI;AAEjC,WAAO,MAAM;AACX,YAAM;AACN,WAAK,aAAa,KAAK;AAAA,IACzB;AAAA,EACF;AAAA,EAEQ,aAAa,OAA0B;AAC7C,QAAI,MAAM,MAAM,WAAW,KAAK,MAAM,eAAe,MAAM,eAAgB;AAE3E,UAAM,OAAO,MAAM,MAAM,MAAM;AAC/B,UAAM,UAAU,KAAK,IAAI,IAAI,MAAM;AACnC,UAAM,YAAY,MAAM,UAAU;AAElC,QAAI,aAAa,GAAG;AAClB,WAAK;AAAA,IACP,OAAO;AACL,iBAAW,MAAM,SAAS;AAAA,IAC5B;AAAA,EACF;AACF;","names":["robotsDelay"]}