hanzio 1.2.0 → 2.1.0

package/dist/src/sitemap/index.js

@@ -1,552 +1,224 @@
-// src/p-queue/index.ts
-class TimeoutError extends Error {
-  constructor(message) {
-    super(message);
-    this.name = "TimeoutError";
-  }
-}
-
-class PriorityQueue {
-  #queue = [];
-  enqueue(run, options = {}) {
-    const element = {
-      priority: options.priority ?? 0,
-      id: options.id,
-      run
-    };
-    const index = this.#queue.findIndex((item) => item.priority < element.priority);
-    if (index === -1) {
-      this.#queue.push(element);
-      return;
-    }
-    this.#queue.splice(index, 0, element);
-  }
-  dequeue() {
-    return this.#queue.shift()?.run;
-  }
-  filter(options) {
-    return this.#queue.filter((item) => Object.entries(options).every(([key, value]) => item[key] === value)).map((item) => item.run);
-  }
-  setPriority(id, priority) {
-    const index = this.#queue.findIndex((item) => item.id === id);
-    if (index === -1) {
-      throw new ReferenceError(`No promise function with the id "${id}" exists in the queue.`);
-    }
-    const [item] = this.#queue.splice(index, 1);
-    this.enqueue(item.run, { ...item, priority });
-  }
-  remove(id) {
-    const index = this.#queue.findIndex((item) => item.id === id);
-    if (index !== -1) {
-      this.#queue.splice(index, 1);
-    }
-  }
-  get size() {
-    return this.#queue.length;
-  }
-}
+import {
+  PQueue2
+} from "../../index-pwt0dz63.js";
+import {
+  escapeRegExp2
+} from "../../index-kj1phgrx.js";
+import {
+  normalizeDomain2
+} from "../../index-mnkcfyx8.js";
 
-class PQueue {
-  #listeners = new Map;
-  #queue;
-  #queueClass;
-  #pending = 0;
-  #concurrency;
-  #isPaused;
-  #intervalCap;
-  #interval;
-  #carryoverIntervalCount;
-  #strict;
-  #intervalCount = 0;
-  #intervalEnd = 0;
-  #intervalTimer;
-  #strictTimestamps = [];
-  #rateLimited = false;
-  #idCounter = 1n;
-  #runningTasks = new Map;
-  timeout;
-  constructor(options = {}) {
-    const {
-      concurrency = Number.POSITIVE_INFINITY,
-      autoStart = true,
-      queueClass = PriorityQueue,
-      intervalCap = Number.POSITIVE_INFINITY,
-      interval = 0,
-      carryoverIntervalCount,
-      carryoverConcurrencyCount,
-      strict = false,
-      timeout
-    } = options;
-    if (!(typeof concurrency === "number" && concurrency >= 1)) {
-      throw new TypeError("Expected `concurrency` to be a number from 1 and up");
-    }
-    if (!(typeof intervalCap === "number" && intervalCap >= 1)) {
-      throw new TypeError("Expected `intervalCap` to be a number from 1 and up");
-    }
-    if (!(typeof interval === "number" && Number.isFinite(interval) && interval >= 0)) {
-      throw new TypeError("Expected `interval` to be a finite number >= 0");
-    }
-    if (strict && interval === 0) {
-      throw new TypeError("The `strict` option requires a non-zero `interval`");
-    }
-    if (strict && intervalCap === Number.POSITIVE_INFINITY) {
-      throw new TypeError("The `strict` option requires a finite `intervalCap`");
-    }
-    if (timeout !== undefined && !(Number.isFinite(timeout) && timeout > 0)) {
-      throw new TypeError("Expected `timeout` to be a positive finite number");
-    }
-    this.#queueClass = queueClass;
-    this.#queue = new queueClass;
-    this.#concurrency = concurrency;
-    this.#isPaused = !autoStart;
-    this.#intervalCap = intervalCap;
-    this.#interval = interval;
-    this.#carryoverIntervalCount = carryoverIntervalCount ?? carryoverConcurrencyCount ?? false;
-    this.#strict = strict;
-    this.timeout = timeout;
-  }
-  on(event, listener) {
-    const listeners = this.#listeners.get(event) ?? new Set;
-    listeners.add(listener);
-    this.#listeners.set(event, listeners);
-    return this;
-  }
-  off(event, listener) {
-    this.#listeners.get(event)?.delete(listener);
-    return this;
-  }
-  emit(event, ...arguments_) {
-    const listeners = this.#listeners.get(event);
-    if (!listeners || listeners.size === 0) {
-      return false;
-    }
-    for (const listener of [...listeners]) {
-      listener(...arguments_);
-    }
+// src/sitemap/index.ts
+var COMMON_LOCATIONS = [
+  "/sitemap.xml",
+  "/sitemap_index.xml",
+  "/sitemap/",
+  "/sitemaps.xml",
+  "/sitemap/sitemap.xml"
+];
+var XML_ENTITIES = {
+  amp: "&",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  apos: "'"
+};
+var decodeXml = (value) => value.replace(/&(#[xX][\da-fA-F]+|#\d+|amp|lt|gt|quot|apos);/g, (_, entity) => entity[0] !== "#" ? XML_ENTITIES[entity] : String.fromCodePoint(entity[1] === "x" || entity[1] === "X" ? Number.parseInt(entity.slice(2), 16) : Number(entity.slice(1))));
+var matchesFilter = (filter, url) => {
+  if (filter === undefined)
     return true;
+  if (typeof filter === "string")
+    return url.includes(filter);
+  if (typeof filter === "function")
+    return filter(url);
+  return url.search(filter) !== -1;
+};
+var defaultOnError = (url, error) => console.warn(`Failed to fetch ${url}:`, error instanceof Error ? error.message : "Unknown error");
+var readBody = async (response) => {
+  const bytes = new Uint8Array(await response.arrayBuffer());
+  if (bytes[0] !== 31 || bytes[1] !== 139) {
+    return new TextDecoder().decode(bytes);
   }
-  get concurrency() {
-    return this.#concurrency;
-  }
-  set concurrency(newConcurrency) {
-    if (!(typeof newConcurrency === "number" && newConcurrency >= 1)) {
-      throw new TypeError("Expected `concurrency` to be a number from 1 and up");
-    }
-    this.#concurrency = newConcurrency;
-    this.#processQueue();
-  }
-  get size() {
-    return this.#queue.size;
-  }
-  sizeBy(options) {
-    return this.#queue.filter(options).length;
-  }
-  get pending() {
-    return this.#pending;
-  }
-  get isPaused() {
-    return this.#isPaused;
-  }
-  get isRateLimited() {
-    this.#updateRateLimitState();
-    return this.#rateLimited;
-  }
-  get isSaturated() {
-    return this.#pending >= this.#concurrency && this.#queue.size > 0 || this.isRateLimited && this.#queue.size > 0;
-  }
-  get runningTasks() {
-    return [...this.#runningTasks.values()].map((task) => ({ ...task }));
-  }
-  setPriority(id, priority) {
-    if (typeof priority !== "number" || !Number.isFinite(priority)) {
-      throw new TypeError("Expected `priority` to be a finite number");
-    }
-    this.#queue.setPriority(id, priority);
-  }
-  async add(function_, options = {}) {
-    const taskOptions = {
-      timeout: this.timeout,
-      ...options,
-      id: options.id ?? (this.#idCounter++).toString()
-    };
-    return new Promise((resolve, reject) => {
-      const taskSymbol = Symbol(taskOptions.id);
-      let didRun = false;
-      let queuedAbortHandler;
-      let runningAbortHandler;
-      const cleanupQueuedAbort = () => {
-        if (queuedAbortHandler) {
-          taskOptions.signal?.removeEventListener("abort", queuedAbortHandler);
-          queuedAbortHandler = undefined;
-        }
-      };
-      const run = async () => {
-        didRun = true;
-        cleanupQueuedAbort();
-        this.#pending++;
-        this.#runningTasks.set(taskSymbol, {
-          id: taskOptions.id,
-          priority: taskOptions.priority ?? 0,
-          startTime: Date.now(),
-          timeout: taskOptions.timeout
-        });
-        try {
-          taskOptions.signal?.throwIfAborted();
-          const operation = Promise.resolve(function_({ signal: taskOptions.signal }));
-          const result = await this.#withTimeoutAndAbort(operation, taskOptions, (listener) => {
-            runningAbortHandler = listener;
-          });
-          resolve(result);
-          this.emit("completed", result);
-        } catch (error) {
-          reject(error);
-          this.emit("error", error);
-        } finally {
-          if (runningAbortHandler) {
-            taskOptions.signal?.removeEventListener("abort", runningAbortHandler);
-          }
-          this.#runningTasks.delete(taskSymbol);
-          queueMicrotask(() => {
-            this.#next();
-          });
-        }
-      };
-      if (taskOptions.signal?.aborted) {
-        reject(taskOptions.signal.reason);
-        return;
-      }
-      if (taskOptions.signal) {
-        queuedAbortHandler = () => {
-          if (didRun) {
-            return;
-          }
-          this.#queue.remove?.(taskOptions.id);
-          reject(taskOptions.signal?.reason);
-          this.emit("next");
-          this.#tryToStartAnother();
-        };
-        taskOptions.signal.addEventListener("abort", queuedAbortHandler, {
-          once: true
-        });
-      }
-      this.#queue.enqueue(run, taskOptions);
-      this.emit("add");
-      this.#processQueue();
-    });
-  }
-  async addAll(functions, options) {
-    return Promise.all(functions.map(async (function_) => this.add(function_, options)));
-  }
-  start() {
-    if (!this.#isPaused) {
-      return this;
-    }
-    this.#isPaused = false;
-    this.#processQueue();
-    return this;
-  }
-  pause() {
-    this.#isPaused = true;
-  }
-  clear() {
-    this.#queue = new this.#queueClass;
-    this.#clearIntervalTimer();
-    this.#updateRateLimitState();
-    this.emit("empty");
-    if (this.#pending === 0) {
-      this.emit("idle");
-    }
-    this.emit("next");
-  }
-  async onEmpty() {
-    if (this.#queue.size === 0) {
-      return;
-    }
-    await this.#onEvent("empty");
-  }
-  async onIdle() {
-    if (this.#queue.size === 0 && this.#pending === 0) {
-      return;
-    }
-    await this.#onEvent("idle");
-  }
-  async onPendingZero() {
-    if (this.#pending === 0) {
-      return;
-    }
-    await this.#onEvent("pendingZero");
-  }
-  async onRateLimit() {
-    if (this.isRateLimited) {
-      return;
-    }
-    await this.#onEvent("rateLimit");
-  }
-  async onRateLimitCleared() {
-    if (!this.isRateLimited) {
-      return;
-    }
-    await this.#onEvent("rateLimitCleared");
-  }
-  onError() {
-    return new Promise((_resolve, reject) => {
-      const listener = (error) => {
-        this.off("error", listener);
-        reject(error);
-      };
-      this.on("error", listener);
-    });
-  }
-  async onSizeLessThan(limit) {
-    if (this.#queue.size < limit) {
-      return;
-    }
-    await this.#onEvent("next", () => this.#queue.size < limit);
-  }
-  async#onEvent(event, filter) {
-    return new Promise((resolve) => {
-      const listener = () => {
-        if (filter && !filter()) {
-          return;
-        }
-        this.off(event, listener);
-        resolve();
-      };
-      this.on(event, listener);
-    });
-  }
-  async#withTimeoutAndAbort(operation, options, setAbortListener) {
-    const races = [operation];
-    let timeoutId;
-    if (options.timeout) {
-      races.push(new Promise((_resolve, reject) => {
-        timeoutId = setTimeout(() => {
-          reject(new TimeoutError(`Task timed out after ${options.timeout}ms`));
-        }, options.timeout);
-      }));
-    }
-    if (options.signal) {
-      races.push(new Promise((_resolve, reject) => {
-        const listener = () => {
-          reject(options.signal?.reason);
-        };
-        setAbortListener(listener);
-        options.signal?.addEventListener("abort", listener, { once: true });
-      }));
-    }
-    try {
-      return await Promise.race(races);
-    } finally {
-      if (timeoutId) {
-        clearTimeout(timeoutId);
+  return new Response(new Blob([bytes]).stream().pipeThrough(new DecompressionStream("gzip"))).text();
+};
+var parseSitemap = (xml) => ({
+  isIndex: /<sitemapindex[\s>]/i.test(xml),
+  entries: [...xml.matchAll(/<(url|sitemap)[\s>][\s\S]*?<\/\1>/gi)].flatMap(([block]) => {
+    const loc = block.match(/<loc>\s*(?:<!\[CDATA\[([\s\S]*?)\]\]>|([^<]*?))\s*<\/loc>/i);
+    if (!loc)
+      return [];
+    const lastmod = block.match(/<lastmod>\s*([^<]*?)\s*<\/lastmod>/i)?.[1];
+    return [
+      {
+        loc: (loc[1] ?? decodeXml(loc[2])).trim(),
+        ...lastmod ? { lastmod } : {}
       }
-    }
-  }
-  #next() {
-    this.#pending--;
-    if (this.#pending === 0) {
-      this.emit("pendingZero");
-    }
-    this.#processQueue();
-    this.emit("next");
-  }
-  #processQueue() {
-    while (this.#tryToStartAnother()) {}
-    this.#updateRateLimitState();
-  }
-  #tryToStartAnother() {
-    if (this.#queue.size === 0) {
-      this.#clearIntervalTimer();
-      this.emit("empty");
-      if (this.#pending === 0) {
-        this.emit("idle");
+    ];
+  })
+});
+var productToken = (userAgent) => userAgent.trim() === "*" ? "*" : /^[a-z_-]+/i.exec(userAgent.trim())?.[0].toLowerCase() ?? "";
+function parseRobotsTxt(body, userAgent = "*") {
+  const groups = [];
+  const sitemaps = new Set;
+  let group;
+  let inRules = false;
+  for (const line of body.split(/\r\n|\r|\n/)) {
+    const match = /^\s*([a-z-]+)\s*:\s*(.*?)\s*$/i.exec(line.replace(/#.*/, ""));
+    if (!match)
+      continue;
+    const field = match[1].toLowerCase();
+    const value = match[2];
+    if (field === "sitemap") {
+      if (value)
+        sitemaps.add(value);
+    } else if (field === "user-agent") {
+      if (!group || inRules) {
+        group = { agents: [], allow: [], disallow: [] };
+        groups.push(group);
+        inRules = false;
       }
-      return false;
-    }
-    if (this.#isPaused || this.#pending >= this.#concurrency || !this.#doesIntervalAllowAnother()) {
-      return false;
-    }
-    const job = this.#queue.dequeue();
-    if (!job) {
-      return false;
-    }
-    this.#consumeIntervalSlot();
-    this.emit("active");
-    job();
+      group.agents.push(productToken(value));
+    } else if (group && (field === "allow" || field === "disallow")) {
+      inRules = true;
+      if (value)
+        group[field].push(value);
+    } else if (group && field === "crawl-delay") {
+      inRules = true;
+      const seconds = Number(value);
+      if (value && seconds >= 0)
+        group.crawlDelaySec ??= seconds;
+    }
+  }
+  const token = productToken(userAgent);
+  const named = groups.filter((g) => g.agents.includes(token));
+  const selected = named.length > 0 ? named : groups.filter((g) => g.agents.includes("*"));
+  const crawlDelaySec = selected.find((g) => g.crawlDelaySec !== undefined)?.crawlDelaySec;
+  return {
+    allow: selected.flatMap((g) => g.allow),
+    disallow: selected.flatMap((g) => g.disallow),
+    sitemaps: [...sitemaps],
+    ...crawlDelaySec === undefined ? {} : { crawlDelaySec }
+  };
+}
+var patternCache = new Map;
+var matchesRobotsPattern = (pattern, path) => {
+  let regex = patternCache.get(pattern);
+  if (!regex) {
+    const anchored = pattern.endsWith("$");
+    const source = (anchored ? pattern.slice(0, -1) : pattern).split("*").map(escapeRegExp2).join(".*");
+    regex = new RegExp(`^${source}${anchored ? "$" : ""}`);
+    if (patternCache.size >= 1000)
+      patternCache.clear();
+    patternCache.set(pattern, regex);
+  }
+  return regex.test(path);
+};
+function isAllowedByRobots(rules, url) {
+  let path;
+  try {
+    const parsed = new URL(url, "http://localhost");
+    path = parsed.pathname + parsed.search;
+  } catch {
     return true;
   }
-  #doesIntervalAllowAnother() {
-    if (this.#intervalCap === Number.POSITIVE_INFINITY || this.#interval === 0) {
-      return true;
-    }
-    const now = Date.now();
-    if (this.#strict) {
-      this.#strictTimestamps = this.#strictTimestamps.filter((timestamp) => now - timestamp < this.#interval);
-      if (this.#strictTimestamps.length < this.#intervalCap) {
-        return true;
+  if (path === "/robots.txt")
+    return true;
+  let best;
+  for (const [allow, patterns] of [
+    [true, rules.allow],
+    [false, rules.disallow]
+  ]) {
+    for (const pattern of patterns) {
+      if (pattern.length > (best?.length ?? -1) && matchesRobotsPattern(pattern, path)) {
+        best = { allow, length: pattern.length };
       }
-      this.#scheduleIntervalResume(this.#interval - (now - this.#strictTimestamps[0]));
-      return false;
-    }
-    if (this.#intervalEnd === 0 || now >= this.#intervalEnd) {
-      this.#intervalCount = this.#carryoverIntervalCount ? this.#pending : 0;
-      this.#intervalEnd = now + this.#interval;
-    }
-    if (this.#intervalCount < this.#intervalCap) {
-      return true;
-    }
-    this.#scheduleIntervalResume(this.#intervalEnd - now);
-    return false;
-  }
-  #consumeIntervalSlot() {
-    if (this.#intervalCap === Number.POSITIVE_INFINITY || this.#interval === 0) {
-      return;
-    }
-    if (this.#strict) {
-      this.#strictTimestamps.push(Date.now());
-      return;
-    }
-    this.#intervalCount++;
-  }
-  #scheduleIntervalResume(delay) {
-    if (this.#intervalTimer) {
-      return;
-    }
-    this.#intervalTimer = setTimeout(() => {
-      this.#intervalTimer = undefined;
-      this.#processQueue();
-    }, Math.max(0, delay));
-  }
-  #clearIntervalTimer() {
-    if (this.#intervalTimer) {
-      clearTimeout(this.#intervalTimer);
-      this.#intervalTimer = undefined;
-    }
-  }
-  #updateRateLimitState() {
-    const previous = this.#rateLimited;
-    const isRateLimited = this.#queue.size > 0 && this.#intervalCap !== Number.POSITIVE_INFINITY && this.#interval > 0 && !this.#doesIntervalAllowAnother();
-    this.#rateLimited = isRateLimited;
-    if (previous !== isRateLimited) {
-      this.emit(isRateLimited ? "rateLimit" : "rateLimitCleared");
     }
   }
+  return best?.allow ?? true;
 }
-
-// src/sitemap/index.ts
-var createTimeoutController = (timeout) => {
-  const controller = new AbortController;
-  setTimeout(() => controller.abort(), timeout);
-  return controller;
-};
-var discoverSitemaps = async (domain) => {
-  const normalizedDomain = domain.replace(/\/$/, "");
-  const commonLocations = [
-    "/sitemap.xml",
-    "/sitemap_index.xml",
-    "/sitemap/",
-    "/sitemaps.xml",
-    "/sitemap/sitemap.xml"
-  ];
-  const sitemaps = new Set;
-  const requestOptions = {
-    headers: {
-      "User-Agent": "SitemapCrawler/1.0"
-    },
-    signal: createTimeoutController(5000).signal
-  };
+var fetchRobots = async (origin, request, userAgent, onError) => {
+  const robotsUrl = `${origin}/robots.txt`;
   try {
-    const robotsTxt = await fetch(`${normalizedDomain}/robots.txt`, {
-      ...requestOptions
-    });
-    if (robotsTxt.ok) {
-      const text = await robotsTxt.text();
-      const sitemapMatches = text.match(/Sitemap: (.*)/gi);
-      if (sitemapMatches) {
-        sitemapMatches.forEach((match) => {
-          const url = match.replace("Sitemap:", "").trim();
-          sitemaps.add(url);
-        });
-      }
-      if (sitemaps.size > 0) {
-        return [...sitemaps];
-      }
-    }
+    const response = await request(robotsUrl);
+    if (response.ok)
+      return parseRobotsTxt(await response.text(), userAgent);
   } catch (error) {
-    console.warn("Could not fetch robots.txt:", error instanceof Error ? error.message : "Unknown error");
+    onError(robotsUrl, error);
   }
-  for (const location of commonLocations) {
+  return;
+};
+var probeSitemapLocations = async (origin, request) => {
+  for (const location of COMMON_LOCATIONS) {
     try {
-      const response = await fetch(`${normalizedDomain}${location}`, {
-        ...requestOptions,
-        method: "HEAD"
-      });
-      if (response.ok) {
-        return [`${normalizedDomain}${location}`];
-      }
-    } catch (_error) {}
+      const response = await request(`${origin}${location}`, "HEAD");
+      if (response.ok)
+        return [`${origin}${location}`];
+    } catch {}
   }
   return [];
 };
-var getDomainSitemap = async (domain, options = {}) => {
+var getDomainSitemapEntries = async (domain, options = {}) => {
   const {
     filterIndexes,
+    filterUrls,
     concurrency = 10,
     maxDepth = 5,
-    timeout = 1e4
+    maxUrls = Number.POSITIVE_INFINITY,
+    timeout = 1e4,
+    userAgent = "SitemapCrawler/1.0",
+    fetch: fetchFn = globalThis.fetch,
+    onError = defaultOnError,
+    respectRobots = false
   } = options;
-  let normalizedDomain = domain.replace(/\/$/, "");
-  if (!normalizedDomain.startsWith("http")) {
-    normalizedDomain = `https://${normalizedDomain}`;
-  }
-  const sitemapUrls = await discoverSitemaps(normalizedDomain);
+  const request = (url, method = "GET") => fetchFn(url, {
+    method,
+    headers: { "User-Agent": userAgent },
+    signal: AbortSignal.timeout(timeout)
+  });
+  const origin = normalizeDomain2(domain);
+  const robots = await fetchRobots(origin, request, userAgent, onError);
+  const sitemapUrls = robots?.sitemaps.length ? robots.sitemaps : await probeSitemapLocations(origin, request);
+  const keepUrl = (url) => matchesFilter(filterUrls, url) && (!respectRobots || !robots || isAllowedByRobots(robots, url));
   if (sitemapUrls.length === 0) {
     throw new Error("No sitemaps found for the domain");
   }
-  const queue = new PQueue({ concurrency });
-  const result = {
-    links: new Set,
-    visited: new Set
-  };
-  const locRegex = /<loc>([^<]+)<\/loc>/g;
-  const isXmlSitemap = (url) => url.endsWith(".xml") || url.toLowerCase().includes("sitemap");
+  const queue = new PQueue2({ concurrency });
+  const entries = new Map;
+  const visited = new Set;
   const processSitemap = async (sitemapUrl, depth = 0) => {
-    if (depth > maxDepth || result.visited.has(sitemapUrl)) {
+    if (depth > maxDepth || visited.has(sitemapUrl) || entries.size >= maxUrls) {
       return;
     }
-    result.visited.add(sitemapUrl);
+    visited.add(sitemapUrl);
     try {
-      const controller = createTimeoutController(timeout);
-      const response = await fetch(sitemapUrl, {
-        headers: {
-          "User-Agent": "SitemapCrawler/1.0"
-        },
-        signal: controller.signal
-      });
-      if (!response.ok)
+      const response = await request(sitemapUrl);
+      if (!response.ok) {
         throw new Error(`HTTP error! status: ${response.status}`);
-      const data = await response.text();
-      const matches = [...data.matchAll(locRegex)].map((match) => match[1].trim()).filter((link) => {
-        if (filterIndexes && isXmlSitemap(link)) {
-          return link.includes(filterIndexes);
-        }
-        return true;
-      });
-      for (const link of matches) {
-        if (isXmlSitemap(link)) {
-          queue.add(() => processSitemap(link, depth + 1));
-        } else {
-          result.links.add(link);
+      }
+      const { isIndex, entries: found } = parseSitemap(await readBody(response));
+      for (const entry of found) {
+        if (isIndex) {
+          if (matchesFilter(filterIndexes, entry.loc)) {
+            queue.add(() => processSitemap(entry.loc, depth + 1));
+          }
+        } else if (entries.size >= maxUrls) {
+          return;
+        } else if (keepUrl(entry.loc)) {
+          entries.set(entry.loc, entries.get(entry.loc) ?? entry);
         }
       }
     } catch (error) {
-      console.warn(`Failed to process sitemap ${sitemapUrl}:`, error instanceof Error ? error.message : "Unknown error");
+      onError(sitemapUrl, error);
     }
   };
   await Promise.all(sitemapUrls.map((sitemapUrl) => queue.add(() => processSitemap(sitemapUrl))));
   await queue.onIdle();
-  return [...result.links];
+  return [...entries.values()];
 };
+var getDomainSitemap = async (domain, options = {}) => (await getDomainSitemapEntries(domain, options)).map((entry) => entry.loc);
 export {
-  getDomainSitemap
+  getDomainSitemap,
+  getDomainSitemapEntries,
+  isAllowedByRobots,
+  parseRobotsTxt
 };