@djangocfg/seo 2.1.50

package/dist/crawler/index.d.ts

@@ -0,0 +1,88 @@
+import { CrawlerConfig, CrawlResult, SeoIssue } from '../types/index.js';
+
+/**
+ * @djangocfg/seo - Site Crawler
+ * Internal site crawler for SEO analysis
+ */
+
+declare class SiteCrawler {
+    private config;
+    private baseUrl;
+    private visited;
+    private queue;
+    private results;
+    private limit;
+    constructor(siteUrl: string, config?: CrawlerConfig);
+    /**
+     * Start crawling the site
+     */
+    crawl(): Promise<CrawlResult[]>;
+    /**
+     * Crawl a single page
+     */
+    private crawlPage;
+    /**
+     * Parse HTML and extract SEO-relevant data
+     */
+    private parseHtml;
+    /**
+     * Normalize URL for deduplication
+     */
+    private normalizeUrl;
+    /**
+     * Check if URL should be excluded
+     */
+    private shouldExclude;
+}
+/**
+ * Analyze crawl results for SEO issues
+ */
+declare function analyzeCrawlResults(results: CrawlResult[]): SeoIssue[];
+
+/**
+ * @djangocfg/seo - Robots.txt Parser
+ * Parse and analyze robots.txt files
+ */
+
+interface RobotsAnalysis {
+    exists: boolean;
+    content?: string;
+    sitemaps: string[];
+    allowedPaths: string[];
+    disallowedPaths: string[];
+    crawlDelay?: number;
+    issues: SeoIssue[];
+}
+/**
+ * Fetch and parse robots.txt for a site
+ */
+declare function analyzeRobotsTxt(siteUrl: string): Promise<RobotsAnalysis>;
+/**
+ * Check if a URL is allowed by robots.txt
+ */
+declare function isUrlAllowed(siteUrl: string, url: string, userAgent?: string): Promise<boolean>;
+
+/**
+ * @djangocfg/seo - Sitemap Validator
+ * Validate XML sitemaps
+ */
+
+interface SitemapAnalysis {
+    url: string;
+    exists: boolean;
+    type: 'sitemap' | 'sitemap-index' | 'unknown';
+    urls: string[];
+    childSitemaps: string[];
+    lastmod?: string;
+    issues: SeoIssue[];
+}
+/**
+ * Analyze a sitemap URL
+ */
+declare function analyzeSitemap(sitemapUrl: string): Promise<SitemapAnalysis>;
+/**
+ * Recursively analyze a sitemap and all its children
+ */
+declare function analyzeAllSitemaps(sitemapUrl: string, maxDepth?: number): Promise<SitemapAnalysis[]>;
+
+export { type RobotsAnalysis, SiteCrawler, type SitemapAnalysis, analyzeAllSitemaps, analyzeCrawlResults, analyzeRobotsTxt, analyzeSitemap, isUrlAllowed };

package/dist/crawler/index.mjs

@@ -0,0 +1,610 @@
+import { load } from 'cheerio';
+import pLimit from 'p-limit';
+import consola from 'consola';
+import robotsParser from 'robots-parser';
+
+// src/crawler/crawler.ts
+var DEFAULT_CONFIG = {
+  maxPages: 100,
+  maxDepth: 3,
+  concurrency: 5,
+  timeout: 3e4,
+  userAgent: "DjangoCFG-SEO-Crawler/1.0 (+https://djangocfg.com/bot)",
+  respectRobotsTxt: true,
+  includePatterns: [],
+  excludePatterns: [
+    "/api/",
+    "/admin/",
+    "/_next/",
+    "/static/",
+    ".pdf",
+    ".jpg",
+    ".png",
+    ".gif",
+    ".svg",
+    ".css",
+    ".js"
+  ]
+};
+var SiteCrawler = class {
+  config;
+  baseUrl;
+  visited = /* @__PURE__ */ new Set();
+  queue = [];
+  results = [];
+  limit;
+  constructor(siteUrl, config) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+    this.baseUrl = new URL(siteUrl);
+    this.limit = pLimit(this.config.concurrency);
+  }
+  /**
+   * Start crawling the site
+   */
+  async crawl() {
+    consola.info(`Starting crawl of ${this.baseUrl.origin}`);
+    consola.info(`Config: maxPages=${this.config.maxPages}, maxDepth=${this.config.maxDepth}`);
+    this.queue.push({ url: this.baseUrl.href, depth: 0 });
+    while (this.queue.length > 0 && this.results.length < this.config.maxPages) {
+      const batch = this.queue.splice(0, this.config.concurrency);
+      const promises = batch.map(
+        ({ url, depth }) => this.limit(() => this.crawlPage(url, depth))
+      );
+      await Promise.all(promises);
+    }
+    consola.success(`Crawl complete. Crawled ${this.results.length} pages.`);
+    return this.results;
+  }
+  /**
+   * Crawl a single page
+   */
+  async crawlPage(url, depth) {
+    const normalizedUrl = this.normalizeUrl(url);
+    if (this.visited.has(normalizedUrl)) return;
+    if (this.shouldExclude(normalizedUrl)) return;
+    this.visited.add(normalizedUrl);
+    const startTime = Date.now();
+    const result = {
+      url: normalizedUrl,
+      statusCode: 0,
+      links: { internal: [], external: [] },
+      images: [],
+      loadTime: 0,
+      errors: [],
+      warnings: [],
+      crawledAt: (/* @__PURE__ */ new Date()).toISOString()
+    };
+    try {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
+      const response = await fetch(normalizedUrl, {
+        headers: {
+          "User-Agent": this.config.userAgent,
+          Accept: "text/html,application/xhtml+xml"
+        },
+        signal: controller.signal,
+        redirect: "follow"
+      });
+      result.ttfb = Date.now() - startTime;
+      clearTimeout(timeoutId);
+      result.statusCode = response.status;
+      result.contentType = response.headers.get("content-type") || void 0;
+      result.contentLength = Number(response.headers.get("content-length")) || void 0;
+      if (response.ok && result.contentType?.includes("text/html")) {
+        const html = await response.text();
+        this.parseHtml(html, result, normalizedUrl, depth);
+      } else if (!response.ok) {
+        result.errors.push(`HTTP ${response.status}: ${response.statusText}`);
+      }
+    } catch (error) {
+      if (error instanceof Error) {
+        if (error.name === "AbortError") {
+          result.errors.push("Request timeout");
+        } else {
+          result.errors.push(error.message);
+        }
+      }
+    }
+    result.loadTime = Date.now() - startTime;
+    this.results.push(result);
+    consola.debug(`Crawled: ${normalizedUrl} (${result.statusCode}) - ${result.loadTime}ms`);
+  }
+  /**
+   * Parse HTML and extract SEO-relevant data
+   */
+  parseHtml(html, result, pageUrl, depth) {
+    const $ = load(html);
+    result.title = $("title").first().text().trim() || void 0;
+    if (!result.title) {
+      result.warnings.push("Missing title tag");
+    } else if (result.title.length > 60) {
+      result.warnings.push(`Title too long (${result.title.length} chars, recommended: <60)`);
+    }
+    result.metaDescription = $('meta[name="description"]').attr("content")?.trim() || void 0;
+    if (!result.metaDescription) {
+      result.warnings.push("Missing meta description");
+    } else if (result.metaDescription.length > 160) {
+      result.warnings.push(
+        `Meta description too long (${result.metaDescription.length} chars, recommended: <160)`
+      );
+    }
+    result.metaRobots = $('meta[name="robots"]').attr("content")?.trim() || void 0;
+    const xRobots = $('meta[http-equiv="X-Robots-Tag"]').attr("content")?.trim();
+    if (xRobots) {
+      result.metaRobots = result.metaRobots ? `${result.metaRobots}, ${xRobots}` : xRobots;
+    }
+    result.canonicalUrl = $('link[rel="canonical"]').attr("href")?.trim() || void 0;
+    if (!result.canonicalUrl) {
+      result.warnings.push("Missing canonical tag");
+    }
+    result.h1 = $("h1").map((_, el) => $(el).text().trim()).get();
+    result.h2 = $("h2").map((_, el) => $(el).text().trim()).get();
+    if (result.h1.length === 0) {
+      result.warnings.push("Missing H1 tag");
+    } else if (result.h1.length > 1) {
+      result.warnings.push(`Multiple H1 tags (${result.h1.length})`);
+    }
+    $("a[href]").each((_, el) => {
+      const href = $(el).attr("href");
+      if (!href) return;
+      try {
+        const linkUrl = new URL(href, pageUrl);
+        if (linkUrl.hostname === this.baseUrl.hostname) {
+          const internalUrl = this.normalizeUrl(linkUrl.href);
+          result.links.internal.push(internalUrl);
+          if (depth < this.config.maxDepth && !this.visited.has(internalUrl)) {
+            this.queue.push({ url: internalUrl, depth: depth + 1 });
+          }
+        } else {
+          result.links.external.push(linkUrl.href);
+        }
+      } catch {
+      }
+    });
+    $("img").each((_, el) => {
+      const src = $(el).attr("src");
+      const alt = $(el).attr("alt");
+      if (src) {
+        result.images.push({
+          src,
+          alt,
+          hasAlt: alt !== void 0 && alt.trim().length > 0
+        });
+      }
+    });
+    const imagesWithoutAlt = result.images.filter((img) => !img.hasAlt);
+    if (imagesWithoutAlt.length > 0) {
+      result.warnings.push(`${imagesWithoutAlt.length} images without alt text`);
+    }
+  }
+  /**
+   * Normalize URL for deduplication
+   */
+  normalizeUrl(url) {
+    try {
+      const parsed = new URL(url, this.baseUrl.href);
+      parsed.hash = "";
+      let pathname = parsed.pathname;
+      if (pathname.endsWith("/") && pathname !== "/") {
+        pathname = pathname.slice(0, -1);
+      }
+      parsed.pathname = pathname;
+      return parsed.href;
+    } catch {
+      return url;
+    }
+  }
+  /**
+   * Check if URL should be excluded
+   */
+  shouldExclude(url) {
+    if (this.config.includePatterns.length > 0) {
+      const included = this.config.includePatterns.some(
+        (pattern) => url.includes(pattern)
+      );
+      if (!included) return true;
+    }
+    return this.config.excludePatterns.some((pattern) => url.includes(pattern));
+  }
+};
+function analyzeCrawlResults(results) {
+  const issues = [];
+  for (const result of results) {
+    if (result.statusCode >= 400) {
+      issues.push({
+        id: `http-error-${hash(result.url)}`,
+        url: result.url,
+        category: "technical",
+        severity: result.statusCode >= 500 ? "critical" : "error",
+        title: `HTTP ${result.statusCode} error`,
+        description: `Page returns ${result.statusCode} status code.`,
+        recommendation: result.statusCode === 404 ? "Either restore the content or set up a redirect." : "Fix the server error and ensure the page is accessible.",
+        detectedAt: result.crawledAt,
+        metadata: { statusCode: result.statusCode }
+      });
+    }
+    if (!result.title && result.statusCode === 200) {
+      issues.push({
+        id: `missing-title-${hash(result.url)}`,
+        url: result.url,
+        category: "content",
+        severity: "error",
+        title: "Missing title tag",
+        description: "This page does not have a title tag.",
+        recommendation: "Add a unique, descriptive title tag (50-60 characters).",
+        detectedAt: result.crawledAt
+      });
+    }
+    if (!result.metaDescription && result.statusCode === 200) {
+      issues.push({
+        id: `missing-meta-desc-${hash(result.url)}`,
+        url: result.url,
+        category: "content",
+        severity: "warning",
+        title: "Missing meta description",
+        description: "This page does not have a meta description.",
+        recommendation: "Add a unique meta description (120-160 characters).",
+        detectedAt: result.crawledAt
+      });
+    }
+    if (result.h1 && result.h1.length === 0 && result.statusCode === 200) {
+      issues.push({
+        id: `missing-h1-${hash(result.url)}`,
+        url: result.url,
+        category: "content",
+        severity: "warning",
+        title: "Missing H1 heading",
+        description: "This page does not have an H1 heading.",
+        recommendation: "Add a single H1 heading that describes the page content.",
+        detectedAt: result.crawledAt
+      });
+    }
+    if (result.h1 && result.h1.length > 1) {
+      issues.push({
+        id: `multiple-h1-${hash(result.url)}`,
+        url: result.url,
+        category: "content",
+        severity: "warning",
+        title: "Multiple H1 headings",
+        description: `This page has ${result.h1.length} H1 headings.`,
+        recommendation: "Use only one H1 heading per page.",
+        detectedAt: result.crawledAt,
+        metadata: { h1Count: result.h1.length }
+      });
+    }
+    const imagesWithoutAlt = result.images.filter((img) => !img.hasAlt);
+    if (imagesWithoutAlt.length > 0) {
+      issues.push({
+        id: `images-no-alt-${hash(result.url)}`,
+        url: result.url,
+        category: "content",
+        severity: "info",
+        title: "Images without alt text",
+        description: `${imagesWithoutAlt.length} images are missing alt text.`,
+        recommendation: "Add descriptive alt text to all images for accessibility and SEO.",
+        detectedAt: result.crawledAt,
+        metadata: { count: imagesWithoutAlt.length }
+      });
+    }
+    if (result.loadTime > 3e3) {
+      issues.push({
+        id: `slow-page-${hash(result.url)}`,
+        url: result.url,
+        category: "performance",
+        severity: result.loadTime > 5e3 ? "error" : "warning",
+        title: "Slow page load time",
+        description: `Page took ${result.loadTime}ms to load.`,
+        recommendation: "Optimize page load time. Target under 3 seconds.",
+        detectedAt: result.crawledAt,
+        metadata: { loadTime: result.loadTime }
+      });
+    }
+    if (result.ttfb && result.ttfb > 800) {
+      issues.push({
+        id: `slow-ttfb-${hash(result.url)}`,
+        url: result.url,
+        category: "performance",
+        severity: result.ttfb > 1500 ? "error" : "warning",
+        title: "Slow Time to First Byte",
+        description: `TTFB is ${result.ttfb}ms. Server responded slowly.`,
+        recommendation: "Optimize server response. Target TTFB under 800ms. Consider CDN, caching, or server upgrades.",
+        detectedAt: result.crawledAt,
+        metadata: { ttfb: result.ttfb }
+      });
+    }
+    if (result.metaRobots?.includes("noindex")) {
+      issues.push({
+        id: `noindex-${hash(result.url)}`,
+        url: result.url,
+        category: "indexing",
+        severity: "info",
+        title: "Page marked as noindex",
+        description: "This page has a noindex directive.",
+        recommendation: "Verify this is intentional. Remove noindex if the page should be indexed.",
+        detectedAt: result.crawledAt,
+        metadata: { metaRobots: result.metaRobots }
+      });
+    }
+  }
+  return issues;
+}
+function hash(str) {
+  let hash3 = 0;
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i);
+    hash3 = (hash3 << 5) - hash3 + char;
+    hash3 = hash3 & hash3;
+  }
+  return Math.abs(hash3).toString(36);
+}
+async function analyzeRobotsTxt(siteUrl) {
+  const robotsUrl = new URL("/robots.txt", siteUrl).href;
+  const analysis = {
+    exists: false,
+    sitemaps: [],
+    allowedPaths: [],
+    disallowedPaths: [],
+    issues: []
+  };
+  try {
+    const response = await fetch(robotsUrl);
+    if (!response.ok) {
+      analysis.issues.push({
+        id: "missing-robots-txt",
+        url: robotsUrl,
+        category: "technical",
+        severity: "warning",
+        title: "Missing robots.txt",
+        description: `No robots.txt file found (HTTP ${response.status}).`,
+        recommendation: "Create a robots.txt file to control crawler access.",
+        detectedAt: (/* @__PURE__ */ new Date()).toISOString()
+      });
+      return analysis;
+    }
+    analysis.exists = true;
+    analysis.content = await response.text();
+    const robots = robotsParser(robotsUrl, analysis.content);
+    analysis.sitemaps = robots.getSitemaps();
+    if (analysis.sitemaps.length === 0) {
+      analysis.issues.push({
+        id: "no-sitemap-in-robots",
+        url: robotsUrl,
+        category: "technical",
+        severity: "info",
+        title: "No sitemap in robots.txt",
+        description: "No sitemap URL is declared in robots.txt.",
+        recommendation: "Add a Sitemap directive pointing to your XML sitemap.",
+        detectedAt: (/* @__PURE__ */ new Date()).toISOString()
+      });
+    }
+    const lines = analysis.content.split("\n");
+    let currentUserAgent = "*";
+    for (const line of lines) {
+      const trimmed = line.trim().toLowerCase();
+      if (trimmed.startsWith("user-agent:")) {
+        currentUserAgent = trimmed.replace("user-agent:", "").trim();
+      } else if (trimmed.startsWith("disallow:")) {
+        const path = line.trim().replace(/disallow:/i, "").trim();
+        if (path) {
+          analysis.disallowedPaths.push(path);
+        }
+      } else if (trimmed.startsWith("allow:")) {
+        const path = line.trim().replace(/allow:/i, "").trim();
+        if (path) {
+          analysis.allowedPaths.push(path);
+        }
+      } else if (trimmed.startsWith("crawl-delay:")) {
+        const delay = parseInt(trimmed.replace("crawl-delay:", "").trim(), 10);
+        if (!isNaN(delay)) {
+          analysis.crawlDelay = delay;
+        }
+      }
+    }
+    const importantPaths = ["/", "/sitemap.xml"];
+    for (const path of importantPaths) {
+      if (!robots.isAllowed(new URL(path, siteUrl).href, "Googlebot")) {
+        analysis.issues.push({
+          id: `blocked-important-path-${path.replace(/\//g, "-")}`,
+          url: siteUrl,
+          category: "crawling",
+          severity: "error",
+          title: `Important path blocked: ${path}`,
+          description: `The path ${path} is blocked in robots.txt.`,
+          recommendation: `Ensure ${path} is accessible to search engines.`,
+          detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          metadata: { path }
+        });
+      }
+    }
+    if (analysis.disallowedPaths.includes("/")) {
+      analysis.issues.push({
+        id: "all-blocked",
+        url: robotsUrl,
+        category: "crawling",
+        severity: "critical",
+        title: "Entire site blocked",
+        description: "robots.txt blocks access to the entire site (Disallow: /).",
+        recommendation: "Remove or modify this rule if you want your site to be indexed.",
+        detectedAt: (/* @__PURE__ */ new Date()).toISOString()
+      });
+    }
+    consola.debug(`Analyzed robots.txt: ${analysis.disallowedPaths.length} disallow rules`);
+  } catch (error) {
+    consola.error("Failed to fetch robots.txt:", error);
+    analysis.issues.push({
+      id: "robots-txt-error",
+      url: robotsUrl,
+      category: "technical",
+      severity: "warning",
+      title: "Failed to fetch robots.txt",
+      description: `Error fetching robots.txt: ${error instanceof Error ? error.message : "Unknown error"}`,
+      recommendation: "Ensure robots.txt is accessible.",
+      detectedAt: (/* @__PURE__ */ new Date()).toISOString()
+    });
+  }
+  return analysis;
+}
+async function isUrlAllowed(siteUrl, url, userAgent = "Googlebot") {
+  const robotsUrl = new URL("/robots.txt", siteUrl).href;
+  try {
+    const response = await fetch(robotsUrl);
+    if (!response.ok) return true;
+    const content = await response.text();
+    const robots = robotsParser(robotsUrl, content);
+    return robots.isAllowed(url, userAgent) ?? true;
+  } catch {
+    return true;
+  }
+}
+async function analyzeSitemap(sitemapUrl) {
+  const analysis = {
+    url: sitemapUrl,
+    exists: false,
+    type: "unknown",
+    urls: [],
+    childSitemaps: [],
+    issues: []
+  };
+  try {
+    const response = await fetch(sitemapUrl, {
+      headers: {
+        Accept: "application/xml, text/xml, */*"
+      }
+    });
+    if (!response.ok) {
+      analysis.issues.push({
+        id: `sitemap-not-found-${hash2(sitemapUrl)}`,
+        url: sitemapUrl,
+        category: "technical",
+        severity: "error",
+        title: "Sitemap not accessible",
+        description: `Sitemap returned HTTP ${response.status}.`,
+        recommendation: "Ensure the sitemap URL is correct and accessible.",
+        detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        metadata: { statusCode: response.status }
+      });
+      return analysis;
+    }
+    analysis.exists = true;
+    const content = await response.text();
+    const contentType = response.headers.get("content-type") || "";
+    if (!contentType.includes("xml") && !content.trim().startsWith("<?xml")) {
+      analysis.issues.push({
+        id: `sitemap-not-xml-${hash2(sitemapUrl)}`,
+        url: sitemapUrl,
+        category: "technical",
+        severity: "warning",
+        title: "Sitemap is not XML",
+        description: "The sitemap does not have an XML content type.",
+        recommendation: "Ensure sitemap is served with Content-Type: application/xml.",
+        detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        metadata: { contentType }
+      });
+    }
+    const $ = load(content, { xmlMode: true });
+    const sitemapIndex = $("sitemapindex");
+    if (sitemapIndex.length > 0) {
+      analysis.type = "sitemap-index";
+      $("sitemap").each((_, el) => {
+        const loc = $("loc", el).text().trim();
+        if (loc) {
+          analysis.childSitemaps.push(loc);
+        }
+      });
+      consola.debug(`Sitemap index contains ${analysis.childSitemaps.length} sitemaps`);
+    } else {
+      analysis.type = "sitemap";
+      $("url").each((_, el) => {
+        const loc = $("loc", el).text().trim();
+        if (loc) {
+          analysis.urls.push(loc);
+        }
+      });
+      const lastmod = $("url lastmod").first().text().trim();
+      if (lastmod) {
+        analysis.lastmod = lastmod;
+      }
+      consola.debug(`Sitemap contains ${analysis.urls.length} URLs`);
+    }
+    if (analysis.type === "sitemap" && analysis.urls.length === 0) {
+      analysis.issues.push({
+        id: `sitemap-empty-${hash2(sitemapUrl)}`,
+        url: sitemapUrl,
+        category: "technical",
+        severity: "warning",
+        title: "Sitemap is empty",
+        description: "The sitemap contains no URLs.",
+        recommendation: "Add URLs to your sitemap or remove it if not needed.",
+        detectedAt: (/* @__PURE__ */ new Date()).toISOString()
+      });
+    }
+    if (analysis.urls.length > 5e4) {
+      analysis.issues.push({
+        id: `sitemap-too-large-${hash2(sitemapUrl)}`,
+        url: sitemapUrl,
+        category: "technical",
+        severity: "error",
+        title: "Sitemap exceeds URL limit",
+        description: `Sitemap contains ${analysis.urls.length} URLs. Maximum is 50,000.`,
+        recommendation: "Split the sitemap into multiple files using a sitemap index.",
+        detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        metadata: { urlCount: analysis.urls.length }
+      });
+    }
+    const sizeInMB = new Blob([content]).size / (1024 * 1024);
+    if (sizeInMB > 50) {
+      analysis.issues.push({
+        id: `sitemap-too-large-size-${hash2(sitemapUrl)}`,
+        url: sitemapUrl,
+        category: "technical",
+        severity: "error",
+        title: "Sitemap exceeds size limit",
+        description: `Sitemap is ${sizeInMB.toFixed(2)}MB. Maximum is 50MB.`,
+        recommendation: "Split the sitemap or compress it.",
+        detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        metadata: { sizeMB: sizeInMB }
+      });
+    }
+  } catch (error) {
+    consola.error("Failed to analyze sitemap:", error);
+    analysis.issues.push({
+      id: `sitemap-error-${hash2(sitemapUrl)}`,
+      url: sitemapUrl,
+      category: "technical",
+      severity: "error",
+      title: "Failed to parse sitemap",
+      description: `Error: ${error instanceof Error ? error.message : "Unknown error"}`,
+      recommendation: "Check sitemap validity using Google Search Console.",
+      detectedAt: (/* @__PURE__ */ new Date()).toISOString()
+    });
+  }
+  return analysis;
+}
+async function analyzeAllSitemaps(sitemapUrl, maxDepth = 3) {
+  const results = [];
+  const visited = /* @__PURE__ */ new Set();
+  async function analyze(url, depth) {
+    if (depth > maxDepth || visited.has(url)) return;
+    visited.add(url);
+    const analysis = await analyzeSitemap(url);
+    results.push(analysis);
+    for (const childUrl of analysis.childSitemaps) {
+      await analyze(childUrl, depth + 1);
+    }
+  }
+  await analyze(sitemapUrl, 0);
+  return results;
+}
+function hash2(str) {
+  let hash3 = 0;
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i);
+    hash3 = (hash3 << 5) - hash3 + char;
+    hash3 = hash3 & hash3;
+  }
+  return Math.abs(hash3).toString(36);
+}
+
+export { SiteCrawler, analyzeAllSitemaps, analyzeCrawlResults, analyzeRobotsTxt, analyzeSitemap, isUrlAllowed };
+//# sourceMappingURL=index.mjs.map
+//# sourceMappingURL=index.mjs.map