hanzio 1.0.2 → 1.0.6

package/dist/src/sitemap/index.js

@@ -0,0 +1,552 @@
+// src/p-queue/index.ts
+class TimeoutError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "TimeoutError";
+  }
+}
+
+class PriorityQueue {
+  #queue = [];
+  enqueue(run, options = {}) {
+    const element = {
+      priority: options.priority ?? 0,
+      id: options.id,
+      run
+    };
+    const index = this.#queue.findIndex((item) => item.priority < element.priority);
+    if (index === -1) {
+      this.#queue.push(element);
+      return;
+    }
+    this.#queue.splice(index, 0, element);
+  }
+  dequeue() {
+    return this.#queue.shift()?.run;
+  }
+  filter(options) {
+    return this.#queue.filter((item) => Object.entries(options).every(([key, value]) => item[key] === value)).map((item) => item.run);
+  }
+  setPriority(id, priority) {
+    const index = this.#queue.findIndex((item2) => item2.id === id);
+    if (index === -1) {
+      throw new ReferenceError(`No promise function with the id "${id}" exists in the queue.`);
+    }
+    const [item] = this.#queue.splice(index, 1);
+    this.enqueue(item.run, { ...item, priority });
+  }
+  remove(id) {
+    const index = this.#queue.findIndex((item) => item.id === id);
+    if (index !== -1) {
+      this.#queue.splice(index, 1);
+    }
+  }
+  get size() {
+    return this.#queue.length;
+  }
+}
+
+class PQueue {
+  #listeners = new Map;
+  #queue;
+  #queueClass;
+  #pending = 0;
+  #concurrency;
+  #isPaused;
+  #intervalCap;
+  #interval;
+  #carryoverIntervalCount;
+  #strict;
+  #intervalCount = 0;
+  #intervalEnd = 0;
+  #intervalTimer;
+  #strictTimestamps = [];
+  #rateLimited = false;
+  #idCounter = 1n;
+  #runningTasks = new Map;
+  timeout;
+  constructor(options = {}) {
+    const {
+      concurrency = Number.POSITIVE_INFINITY,
+      autoStart = true,
+      queueClass = PriorityQueue,
+      intervalCap = Number.POSITIVE_INFINITY,
+      interval = 0,
+      carryoverIntervalCount,
+      carryoverConcurrencyCount,
+      strict = false,
+      timeout
+    } = options;
+    if (!(typeof concurrency === "number" && concurrency >= 1)) {
+      throw new TypeError("Expected `concurrency` to be a number from 1 and up");
+    }
+    if (!(typeof intervalCap === "number" && intervalCap >= 1)) {
+      throw new TypeError("Expected `intervalCap` to be a number from 1 and up");
+    }
+    if (!(typeof interval === "number" && Number.isFinite(interval) && interval >= 0)) {
+      throw new TypeError("Expected `interval` to be a finite number >= 0");
+    }
+    if (strict && interval === 0) {
+      throw new TypeError("The `strict` option requires a non-zero `interval`");
+    }
+    if (strict && intervalCap === Number.POSITIVE_INFINITY) {
+      throw new TypeError("The `strict` option requires a finite `intervalCap`");
+    }
+    if (timeout !== undefined && !(Number.isFinite(timeout) && timeout > 0)) {
+      throw new TypeError("Expected `timeout` to be a positive finite number");
+    }
+    this.#queueClass = queueClass;
+    this.#queue = new queueClass;
+    this.#concurrency = concurrency;
+    this.#isPaused = !autoStart;
+    this.#intervalCap = intervalCap;
+    this.#interval = interval;
+    this.#carryoverIntervalCount = carryoverIntervalCount ?? carryoverConcurrencyCount ?? false;
+    this.#strict = strict;
+    this.timeout = timeout;
+  }
+  on(event, listener) {
+    const listeners = this.#listeners.get(event) ?? new Set;
+    listeners.add(listener);
+    this.#listeners.set(event, listeners);
+    return this;
+  }
+  off(event, listener) {
+    this.#listeners.get(event)?.delete(listener);
+    return this;
+  }
+  emit(event, ...arguments_) {
+    const listeners = this.#listeners.get(event);
+    if (!listeners || listeners.size === 0) {
+      return false;
+    }
+    for (const listener of [...listeners]) {
+      listener(...arguments_);
+    }
+    return true;
+  }
+  get concurrency() {
+    return this.#concurrency;
+  }
+  set concurrency(newConcurrency) {
+    if (!(typeof newConcurrency === "number" && newConcurrency >= 1)) {
+      throw new TypeError("Expected `concurrency` to be a number from 1 and up");
+    }
+    this.#concurrency = newConcurrency;
+    this.#processQueue();
+  }
+  get size() {
+    return this.#queue.size;
+  }
+  sizeBy(options) {
+    return this.#queue.filter(options).length;
+  }
+  get pending() {
+    return this.#pending;
+  }
+  get isPaused() {
+    return this.#isPaused;
+  }
+  get isRateLimited() {
+    this.#updateRateLimitState();
+    return this.#rateLimited;
+  }
+  get isSaturated() {
+    return this.#pending >= this.#concurrency && this.#queue.size > 0 || this.isRateLimited && this.#queue.size > 0;
+  }
+  get runningTasks() {
+    return [...this.#runningTasks.values()].map((task) => ({ ...task }));
+  }
+  setPriority(id, priority) {
+    if (typeof priority !== "number" || !Number.isFinite(priority)) {
+      throw new TypeError("Expected `priority` to be a finite number");
+    }
+    this.#queue.setPriority(id, priority);
+  }
+  async add(function_, options = {}) {
+    const taskOptions = {
+      timeout: this.timeout,
+      ...options,
+      id: options.id ?? (this.#idCounter++).toString()
+    };
+    return new Promise((resolve, reject) => {
+      const taskSymbol = Symbol(taskOptions.id);
+      let didRun = false;
+      let queuedAbortHandler;
+      let runningAbortHandler;
+      const cleanupQueuedAbort = () => {
+        if (queuedAbortHandler) {
+          taskOptions.signal?.removeEventListener("abort", queuedAbortHandler);
+          queuedAbortHandler = undefined;
+        }
+      };
+      const run = async () => {
+        didRun = true;
+        cleanupQueuedAbort();
+        this.#pending++;
+        this.#runningTasks.set(taskSymbol, {
+          id: taskOptions.id,
+          priority: taskOptions.priority ?? 0,
+          startTime: Date.now(),
+          timeout: taskOptions.timeout
+        });
+        try {
+          taskOptions.signal?.throwIfAborted();
+          const operation = Promise.resolve(function_({ signal: taskOptions.signal }));
+          const result = await this.#withTimeoutAndAbort(operation, taskOptions, (listener) => {
+            runningAbortHandler = listener;
+          });
+          resolve(result);
+          this.emit("completed", result);
+        } catch (error) {
+          reject(error);
+          this.emit("error", error);
+        } finally {
+          if (runningAbortHandler) {
+            taskOptions.signal?.removeEventListener("abort", runningAbortHandler);
+          }
+          this.#runningTasks.delete(taskSymbol);
+          queueMicrotask(() => {
+            this.#next();
+          });
+        }
+      };
+      if (taskOptions.signal?.aborted) {
+        reject(taskOptions.signal.reason);
+        return;
+      }
+      if (taskOptions.signal) {
+        queuedAbortHandler = () => {
+          if (didRun) {
+            return;
+          }
+          this.#queue.remove?.(taskOptions.id);
+          reject(taskOptions.signal?.reason);
+          this.emit("next");
+          this.#tryToStartAnother();
+        };
+        taskOptions.signal.addEventListener("abort", queuedAbortHandler, {
+          once: true
+        });
+      }
+      this.#queue.enqueue(run, taskOptions);
+      this.emit("add");
+      this.#processQueue();
+    });
+  }
+  async addAll(functions, options) {
+    return Promise.all(functions.map(async (function_) => this.add(function_, options)));
+  }
+  start() {
+    if (!this.#isPaused) {
+      return this;
+    }
+    this.#isPaused = false;
+    this.#processQueue();
+    return this;
+  }
+  pause() {
+    this.#isPaused = true;
+  }
+  clear() {
+    this.#queue = new this.#queueClass;
+    this.#clearIntervalTimer();
+    this.#updateRateLimitState();
+    this.emit("empty");
+    if (this.#pending === 0) {
+      this.emit("idle");
+    }
+    this.emit("next");
+  }
+  async onEmpty() {
+    if (this.#queue.size === 0) {
+      return;
+    }
+    await this.#onEvent("empty");
+  }
+  async onIdle() {
+    if (this.#queue.size === 0 && this.#pending === 0) {
+      return;
+    }
+    await this.#onEvent("idle");
+  }
+  async onPendingZero() {
+    if (this.#pending === 0) {
+      return;
+    }
+    await this.#onEvent("pendingZero");
+  }
+  async onRateLimit() {
+    if (this.isRateLimited) {
+      return;
+    }
+    await this.#onEvent("rateLimit");
+  }
+  async onRateLimitCleared() {
+    if (!this.isRateLimited) {
+      return;
+    }
+    await this.#onEvent("rateLimitCleared");
+  }
+  onError() {
+    return new Promise((_resolve, reject) => {
+      const listener = (error) => {
+        this.off("error", listener);
+        reject(error);
+      };
+      this.on("error", listener);
+    });
+  }
+  async onSizeLessThan(limit) {
+    if (this.#queue.size < limit) {
+      return;
+    }
+    await this.#onEvent("next", () => this.#queue.size < limit);
+  }
+  async#onEvent(event, filter) {
+    return new Promise((resolve) => {
+      const listener = () => {
+        if (filter && !filter()) {
+          return;
+        }
+        this.off(event, listener);
+        resolve();
+      };
+      this.on(event, listener);
+    });
+  }
+  async#withTimeoutAndAbort(operation, options, setAbortListener) {
+    const races = [operation];
+    let timeoutId;
+    if (options.timeout) {
+      races.push(new Promise((_resolve, reject) => {
+        timeoutId = setTimeout(() => {
+          reject(new TimeoutError(`Task timed out after ${options.timeout}ms`));
+        }, options.timeout);
+      }));
+    }
+    if (options.signal) {
+      races.push(new Promise((_resolve, reject) => {
+        const listener = () => {
+          reject(options.signal?.reason);
+        };
+        setAbortListener(listener);
+        options.signal?.addEventListener("abort", listener, { once: true });
+      }));
+    }
+    try {
+      return await Promise.race(races);
+    } finally {
+      if (timeoutId) {
+        clearTimeout(timeoutId);
+      }
+    }
+  }
+  #next() {
+    this.#pending--;
+    if (this.#pending === 0) {
+      this.emit("pendingZero");
+    }
+    this.#processQueue();
+    this.emit("next");
+  }
+  #processQueue() {
+    while (this.#tryToStartAnother()) {}
+    this.#updateRateLimitState();
+  }
+  #tryToStartAnother() {
+    if (this.#queue.size === 0) {
+      this.#clearIntervalTimer();
+      this.emit("empty");
+      if (this.#pending === 0) {
+        this.emit("idle");
+      }
+      return false;
+    }
+    if (this.#isPaused || this.#pending >= this.#concurrency || !this.#doesIntervalAllowAnother()) {
+      return false;
+    }
+    const job = this.#queue.dequeue();
+    if (!job) {
+      return false;
+    }
+    this.#consumeIntervalSlot();
+    this.emit("active");
+    job();
+    return true;
+  }
+  #doesIntervalAllowAnother() {
+    if (this.#intervalCap === Number.POSITIVE_INFINITY || this.#interval === 0) {
+      return true;
+    }
+    const now = Date.now();
+    if (this.#strict) {
+      this.#strictTimestamps = this.#strictTimestamps.filter((timestamp) => now - timestamp < this.#interval);
+      if (this.#strictTimestamps.length < this.#intervalCap) {
+        return true;
+      }
+      this.#scheduleIntervalResume(this.#interval - (now - this.#strictTimestamps[0]));
+      return false;
+    }
+    if (this.#intervalEnd === 0 || now >= this.#intervalEnd) {
+      this.#intervalCount = this.#carryoverIntervalCount ? this.#pending : 0;
+      this.#intervalEnd = now + this.#interval;
+    }
+    if (this.#intervalCount < this.#intervalCap) {
+      return true;
+    }
+    this.#scheduleIntervalResume(this.#intervalEnd - now);
+    return false;
+  }
+  #consumeIntervalSlot() {
+    if (this.#intervalCap === Number.POSITIVE_INFINITY || this.#interval === 0) {
+      return;
+    }
+    if (this.#strict) {
+      this.#strictTimestamps.push(Date.now());
+      return;
+    }
+    this.#intervalCount++;
+  }
+  #scheduleIntervalResume(delay) {
+    if (this.#intervalTimer) {
+      return;
+    }
+    this.#intervalTimer = setTimeout(() => {
+      this.#intervalTimer = undefined;
+      this.#processQueue();
+    }, Math.max(0, delay));
+  }
+  #clearIntervalTimer() {
+    if (this.#intervalTimer) {
+      clearTimeout(this.#intervalTimer);
+      this.#intervalTimer = undefined;
+    }
+  }
+  #updateRateLimitState() {
+    const previous = this.#rateLimited;
+    const isRateLimited = this.#queue.size > 0 && this.#intervalCap !== Number.POSITIVE_INFINITY && this.#interval > 0 && !this.#doesIntervalAllowAnother();
+    this.#rateLimited = isRateLimited;
+    if (previous !== isRateLimited) {
+      this.emit(isRateLimited ? "rateLimit" : "rateLimitCleared");
+    }
+  }
+}
+
+// src/sitemap/index.ts
+var createTimeoutController = (timeout) => {
+  const controller = new AbortController;
+  setTimeout(() => controller.abort(), timeout);
+  return controller;
+};
+var discoverSitemaps = async (domain) => {
+  const normalizedDomain = domain.replace(/\/$/, "");
+  const commonLocations = [
+    "/sitemap.xml",
+    "/sitemap_index.xml",
+    "/sitemap/",
+    "/sitemaps.xml",
+    "/sitemap/sitemap.xml"
+  ];
+  const sitemaps = new Set;
+  const requestOptions = {
+    headers: {
+      "User-Agent": "SitemapCrawler/1.0"
+    },
+    signal: createTimeoutController(5000).signal
+  };
+  try {
+    const robotsTxt = await fetch(`${normalizedDomain}/robots.txt`, {
+      ...requestOptions
+    });
+    if (robotsTxt.ok) {
+      const text = await robotsTxt.text();
+      const sitemapMatches = text.match(/Sitemap: (.*)/gi);
+      if (sitemapMatches) {
+        sitemapMatches.forEach((match) => {
+          const url = match.replace("Sitemap:", "").trim();
+          sitemaps.add(url);
+        });
+      }
+      if (sitemaps.size > 0) {
+        return [...sitemaps];
+      }
+    }
+  } catch (error) {
+    console.warn("Could not fetch robots.txt:", error instanceof Error ? error.message : "Unknown error");
+  }
+  for (const location of commonLocations) {
+    try {
+      const response = await fetch(`${normalizedDomain}${location}`, {
+        ...requestOptions,
+        method: "HEAD"
+      });
+      if (response.ok) {
+        return [`${normalizedDomain}${location}`];
+      }
+    } catch (_error) {}
+  }
+  return [];
+};
+var getDomainSitemap = async (domain, options = {}) => {
+  const {
+    filterIndexes,
+    concurrency = 10,
+    maxDepth = 5,
+    timeout = 1e4
+  } = options;
+  let normalizedDomain = domain.replace(/\/$/, "");
+  if (!normalizedDomain.startsWith("http")) {
+    normalizedDomain = `https://${normalizedDomain}`;
+  }
+  const sitemapUrls = await discoverSitemaps(normalizedDomain);
+  if (sitemapUrls.length === 0) {
+    throw new Error("No sitemaps found for the domain");
+  }
+  const queue = new PQueue({ concurrency });
+  const result = {
+    links: new Set,
+    visited: new Set
+  };
+  const locRegex = /<loc>([^<]+)<\/loc>/g;
+  const isXmlSitemap = (url) => url.endsWith(".xml") || url.toLowerCase().includes("sitemap");
+  const processSitemap = async (sitemapUrl, depth = 0) => {
+    if (depth > maxDepth || result.visited.has(sitemapUrl)) {
+      return;
+    }
+    result.visited.add(sitemapUrl);
+    try {
+      const controller = createTimeoutController(timeout);
+      const response = await fetch(sitemapUrl, {
+        headers: {
+          "User-Agent": "SitemapCrawler/1.0"
+        },
+        signal: controller.signal
+      });
+      if (!response.ok)
+        throw new Error(`HTTP error! status: ${response.status}`);
+      const data = await response.text();
+      const matches = [...data.matchAll(locRegex)].map((match) => match[1].trim()).filter((link) => {
+        if (filterIndexes && isXmlSitemap(link)) {
+          return link.includes(filterIndexes);
+        }
+        return true;
+      });
+      for (const link of matches) {
+        if (isXmlSitemap(link)) {
+          queue.add(() => processSitemap(link, depth + 1));
+        } else {
+          result.links.add(link);
+        }
+      }
+    } catch (error) {
+      console.warn(`Failed to process sitemap ${sitemapUrl}:`, error instanceof Error ? error.message : "Unknown error");
+    }
+  };
+  await Promise.all(sitemapUrls.map((sitemapUrl) => queue.add(() => processSitemap(sitemapUrl))));
+  await queue.onIdle();
+  return [...result.links];
+};
+export {
+  getDomainSitemap
+};