aeorank 1.3.0 → 1.4.0

package/dist/index.cjs

@@ -3,6 +3,9 @@ var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
 var __export = (target, all) => {
   for (var name in all)
     __defProp(target, name, { get: all[name], enumerable: true });
@@ -17,6 +20,303 @@ var __copyProps = (to, from, except, desc) => {
 };
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
+// src/full-site-crawler.ts
+var full_site_crawler_exports = {};
+__export(full_site_crawler_exports, {
+  crawlFullSite: () => crawlFullSite,
+  extractAllUrlsFromSitemap: () => extractAllUrlsFromSitemap,
+  extractInternalLinks: () => extractInternalLinks,
+  inferCategory: () => inferCategory,
+  isDisallowedByRobots: () => isDisallowedByRobots,
+  parseRobotsTxt: () => parseRobotsTxt
+});
+function parseRobotsTxt(robotsText) {
+  const lines = robotsText.split("\n");
+  const rules = { disallow: [], allow: [] };
+  let inRelevantSection = false;
+  for (const rawLine of lines) {
+    const line = rawLine.trim();
+    if (!line || line.startsWith("#")) continue;
+    const uaMatch = line.match(/^user-agent:\s*(.+)/i);
+    if (uaMatch) {
+      const agent = uaMatch[1].trim().toLowerCase();
+      inRelevantSection = agent === "*" || agent === "aeo-visibility-bot";
+      continue;
+    }
+    if (!inRelevantSection) continue;
+    const disallowMatch = line.match(/^disallow:\s*(.*)/i);
+    if (disallowMatch) {
+      const path = disallowMatch[1].trim();
+      if (path) rules.disallow.push(path);
+      continue;
+    }
+    const allowMatch = line.match(/^allow:\s*(.*)/i);
+    if (allowMatch) {
+      const path = allowMatch[1].trim();
+      if (path) rules.allow.push(path);
+    }
+  }
+  return rules;
+}
+function isDisallowedByRobots(urlPath, rules) {
+  let longestAllow = 0;
+  let longestDisallow = 0;
+  for (const pattern of rules.allow) {
+    if (urlPath.startsWith(pattern) && pattern.length > longestAllow) {
+      longestAllow = pattern.length;
+    }
+  }
+  for (const pattern of rules.disallow) {
+    if (urlPath.startsWith(pattern) && pattern.length > longestDisallow) {
+      longestDisallow = pattern.length;
+    }
+  }
+  if (longestAllow === 0 && longestDisallow === 0) return false;
+  return longestDisallow > longestAllow;
+}
+async function fetchPage2(url, timeoutMs = 1e4) {
+  try {
+    const res = await fetch(url, {
+      signal: AbortSignal.timeout(timeoutMs),
+      headers: { "User-Agent": "AEO-Visibility-Bot/1.0" },
+      redirect: "follow"
+    });
+    if (res.status !== 200) return null;
+    const text = await res.text();
+    if (text.length < 200) return null;
+    return { text: text.slice(0, 5e5), status: res.status, finalUrl: res.url };
+  } catch {
+    return null;
+  }
+}
+async function fetchSitemapXml(url, timeoutMs = 1e4) {
+  try {
+    const res = await fetch(url, {
+      signal: AbortSignal.timeout(timeoutMs),
+      headers: { "User-Agent": "AEO-Visibility-Bot/1.0" },
+      redirect: "follow"
+    });
+    if (res.status !== 200) return null;
+    return await res.text();
+  } catch {
+    return null;
+  }
+}
+async function extractAllUrlsFromSitemap(sitemapText, domain, timeoutMs = 1e4) {
+  const cleanDomain = domain.replace(/^www\./, "").toLowerCase();
+  const urls = /* @__PURE__ */ new Set();
+  const subSitemapLocs = sitemapText.match(/<sitemap>[\s\S]*?<loc>([^<]+)<\/loc>[\s\S]*?<\/sitemap>/gi) || [];
+  if (subSitemapLocs.length > 0) {
+    const subUrls = [];
+    for (const block of subSitemapLocs) {
+      const locMatch = block.match(/<loc>([^<]+)<\/loc>/i);
+      if (locMatch) subUrls.push(locMatch[1].trim());
+    }
+    const fetches = subUrls.slice(0, 10).map((u) => fetchSitemapXml(u, timeoutMs));
+    const results = await Promise.all(fetches);
+    for (const text of results) {
+      if (text) {
+        extractLocsFromXml(text, cleanDomain, urls);
+      }
+    }
+  }
+  extractLocsFromXml(sitemapText, cleanDomain, urls);
+  return Array.from(urls);
+}
+function extractLocsFromXml(xml, cleanDomain, urls) {
+  const locMatches = xml.match(/<url>[\s\S]*?<loc>([^<]+)<\/loc>[\s\S]*?<\/url>/gi) || [];
+  for (const block of locMatches) {
+    const locMatch = block.match(/<loc>([^<]+)<\/loc>/i);
+    if (!locMatch) continue;
+    const url = locMatch[1].trim();
+    try {
+      const parsed = new URL(url);
+      const urlDomain = parsed.hostname.replace(/^www\./, "").toLowerCase();
+      if (urlDomain !== cleanDomain) continue;
+      if (RESOURCE_EXTENSIONS.test(parsed.pathname)) continue;
+      urls.add(url);
+    } catch {
+      continue;
+    }
+  }
+}
+function extractInternalLinks(html, domain) {
+  const cleanDomain = domain.replace(/^www\./, "").toLowerCase();
+  const hrefMatches = html.match(/href="([^"]*)"/gi) || [];
+  const urls = /* @__PURE__ */ new Set();
+  for (const match of hrefMatches) {
+    const href = match.match(/href="([^"]*)"/i)?.[1];
+    if (!href || !href.trim()) continue;
+    let fullUrl;
+    if (href.startsWith("//")) {
+      fullUrl = `https:${href}`;
+    } else if (href.startsWith("/")) {
+      if (href === "/" || href.startsWith("/#")) continue;
+      fullUrl = `https://${domain}${href}`;
+    } else if (href.startsWith("http")) {
+      fullUrl = href;
+    } else if (href.startsWith("#") || href.startsWith("?") || href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) {
+      continue;
+    } else {
+      fullUrl = `https://${domain}/${href}`;
+    }
+    try {
+      const parsed = new URL(fullUrl);
+      const linkDomain = parsed.hostname.replace(/^www\./, "").toLowerCase();
+      if (linkDomain !== cleanDomain) continue;
+      parsed.hash = "";
+      const path = parsed.pathname;
+      if (path === "/" || path === "") continue;
+      if (RESOURCE_EXTENSIONS.test(path)) continue;
+      if (SKIP_PATH_PATTERNS.test(path)) continue;
+      const normalized = parsed.origin + path.replace(/\/+$/, "") + parsed.search;
+      urls.add(normalized);
+    } catch {
+      continue;
+    }
+  }
+  return Array.from(urls);
+}
+function inferCategory(url) {
+  try {
+    const path = new URL(url).pathname;
+    for (const [pattern, category] of CATEGORY_PATTERNS) {
+      if (pattern.test(path)) return category;
+    }
+  } catch {
+  }
+  return "content";
+}
+async function crawlFullSite(siteData, options) {
+  const startTime = Date.now();
+  const maxPages = options?.maxPages ?? 200;
+  const timeoutMs = options?.timeoutMs ?? 1e4;
+  const concurrency = options?.concurrency ?? 5;
+  const respectRobots = options?.respectRobots ?? true;
+  const pages = [];
+  const discoveredUrls = /* @__PURE__ */ new Set();
+  const fetchedUrls = /* @__PURE__ */ new Set();
+  const skippedUrls = /* @__PURE__ */ new Set();
+  const visited = /* @__PURE__ */ new Set();
+  let robotsRules = { disallow: [], allow: [] };
+  if (respectRobots && siteData.robotsTxt?.text) {
+    robotsRules = parseRobotsTxt(siteData.robotsTxt.text);
+  }
+  const baseUrl = `${siteData.protocol}://${siteData.domain}`;
+  visited.add(normalizeUrl(baseUrl));
+  visited.add(normalizeUrl(baseUrl + "/"));
+  if (siteData.blogSample) {
+    for (const page of siteData.blogSample) {
+      if (page.finalUrl) visited.add(normalizeUrl(page.finalUrl));
+    }
+  }
+  const queue = [];
+  if (siteData.sitemapXml?.text) {
+    const sitemapUrls = await extractAllUrlsFromSitemap(
+      siteData.sitemapXml.text,
+      siteData.domain,
+      timeoutMs
+    );
+    for (const url of sitemapUrls) {
+      const norm = normalizeUrl(url);
+      if (!visited.has(norm)) {
+        discoveredUrls.add(url);
+        if (!queue.includes(url)) queue.push(url);
+      }
+    }
+  }
+  if (siteData.homepage?.text) {
+    const homeLinks = extractInternalLinks(siteData.homepage.text, siteData.domain);
+    for (const url of homeLinks) {
+      const norm = normalizeUrl(url);
+      if (!visited.has(norm) && !discoveredUrls.has(url)) {
+        discoveredUrls.add(url);
+        if (!queue.includes(url)) queue.push(url);
+      }
+    }
+  }
+  while (queue.length > 0 && fetchedUrls.size < maxPages) {
+    const batchSize = Math.min(concurrency, maxPages - fetchedUrls.size, queue.length);
+    const batch = [];
+    while (batch.length < batchSize && queue.length > 0) {
+      const url = queue.shift();
+      const norm = normalizeUrl(url);
+      if (visited.has(norm)) continue;
+      visited.add(norm);
+      if (respectRobots) {
+        try {
+          const path = new URL(url).pathname;
+          if (isDisallowedByRobots(path, robotsRules)) {
+            skippedUrls.add(url);
+            continue;
+          }
+        } catch {
+          continue;
+        }
+      }
+      batch.push(url);
+    }
+    if (batch.length === 0) continue;
+    const results = await Promise.all(batch.map((url) => fetchPage2(url, timeoutMs)));
+    for (let i = 0; i < results.length; i++) {
+      const result = results[i];
+      const url = batch[i];
+      fetchedUrls.add(url);
+      if (!result) continue;
+      result.category = inferCategory(url);
+      pages.push(result);
+      const newLinks = extractInternalLinks(result.text, siteData.domain);
+      for (const link of newLinks) {
+        const norm = normalizeUrl(link);
+        if (!visited.has(norm) && !discoveredUrls.has(link)) {
+          discoveredUrls.add(link);
+          queue.push(link);
+        }
+      }
+    }
+  }
+  for (const url of queue) {
+    if (!fetchedUrls.has(url)) {
+      skippedUrls.add(url);
+    }
+  }
+  return {
+    pages,
+    discoveredUrls: Array.from(discoveredUrls),
+    fetchedUrls: Array.from(fetchedUrls),
+    skippedUrls: Array.from(skippedUrls),
+    elapsed: Math.round((Date.now() - startTime) / 100) / 10
+  };
+}
+function normalizeUrl(url) {
+  try {
+    const parsed = new URL(url);
+    return (parsed.origin + parsed.pathname.replace(/\/+$/, "") + parsed.search).toLowerCase();
+  } catch {
+    return url.toLowerCase();
+  }
+}
+var RESOURCE_EXTENSIONS, SKIP_PATH_PATTERNS, CATEGORY_PATTERNS;
+var init_full_site_crawler = __esm({
+  "src/full-site-crawler.ts"() {
+    "use strict";
+    RESOURCE_EXTENSIONS = /\.(js|css|png|jpg|jpeg|gif|svg|ico|pdf|xml|txt|woff|woff2|ttf|eot|mp4|mp3|webp|avif|zip|gz|tar|json)$/i;
+    SKIP_PATH_PATTERNS = /^\/(api|wp-admin|wp-json|static|assets|_next|auth|login|signup|cart|checkout|admin|feed|xmlrpc)\b/i;
+    CATEGORY_PATTERNS = [
+      [/\/(blog|articles?|posts?|news|insights|guides)\b/i, "blog"],
+      [/\/(about|about-us|company|who-we-are)\b/i, "about"],
+      [/\/(pricing|plans|packages)\b/i, "pricing"],
+      [/\/(services?|features?|solutions?|products?|what-we-do|offerings?)\b/i, "services"],
+      [/\/(contact|contact-us|get-in-touch)\b/i, "contact"],
+      [/\/(team|our-team|authors?|people|leadership|staff)\b/i, "team"],
+      [/\/(resources?|resource-center|library|downloads?)\b/i, "resources"],
+      [/\/(docs?|documentation|help|help-center|support|knowledge-base)\b/i, "docs"],
+      [/\/(case-stud\w*|cases|customers?|success-stor\w*|testimonials?)\b/i, "cases"],
+      [/\/(faq|frequently-asked|questions)\b/i, "faq"]
+    ];
+  }
+});
+
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
@@ -30,8 +330,11 @@ __export(index_exports, {
   calculateOverallScore: () => calculateOverallScore,
   classifyRendering: () => classifyRendering,
   compare: () => compare,
+  crawlFullSite: () => crawlFullSite,
   detectParkedDomain: () => detectParkedDomain,
+  extractAllUrlsFromSitemap: () => extractAllUrlsFromSitemap,
   extractContentPagesFromSitemap: () => extractContentPagesFromSitemap,
+  extractInternalLinks: () => extractInternalLinks,
   extractNavLinks: () => extractNavLinks,
   extractRawDataSummary: () => extractRawDataSummary,
   fetchMultiPageData: () => fetchMultiPageData,
@@ -42,6 +345,7 @@ __export(index_exports, {
   generateOpportunities: () => generateOpportunities,
   generatePitchNumbers: () => generatePitchNumbers,
   generateVerdict: () => generateVerdict,
+  inferCategory: () => inferCategory,
   isSpaShell: () => isSpaShell,
   prefetchSiteData: () => prefetchSiteData,
   scoreToStatus: () => scoreToStatus
@@ -1803,7 +2107,11 @@ function extractRawDataSummary(data) {
       const d = new Date(m[1]);
       if (isNaN(d.getTime())) return null;
       return Math.floor((Date.now() - d.getTime()) / (1e3 * 60 * 60 * 24));
-    })()
+    })(),
+    // Full-crawl stats
+    crawl_discovered: data.crawlStats?.discovered ?? 0,
+    crawl_fetched: data.crawlStats?.fetched ?? 0,
+    crawl_skipped: data.crawlStats?.skipped ?? 0
   };
 }
 function auditSiteFromData(data) {
@@ -2799,7 +3107,20 @@ async function audit(domain, options) {
       }
     }
   }
-  if (!options?.noMultiPage) {
+  if (options?.fullCrawl) {
+    const { crawlFullSite: crawlFullSite2 } = await Promise.resolve().then(() => (init_full_site_crawler(), full_site_crawler_exports));
+    const crawlResult = await crawlFullSite2(siteData, {
+      maxPages: options.maxPages ?? 200,
+      concurrency: options.concurrency ?? 5
+    });
+    siteData.blogSample = crawlResult.pages;
+    siteData.crawlStats = {
+      discovered: crawlResult.discoveredUrls.length,
+      fetched: crawlResult.fetchedUrls.length,
+      skipped: crawlResult.skippedUrls.length,
+      elapsed: crawlResult.elapsed
+    };
+  } else if (!options?.noMultiPage) {
     await fetchMultiPageData(siteData);
   }
   const results = auditSiteFromData(siteData);
@@ -2832,6 +3153,9 @@ async function audit(domain, options) {
   };
 }
 
+// src/index.ts
+init_full_site_crawler();
+
 // src/html-report.ts
 function scoreColor(score) {
   if (score <= 40) return "#F44336";
@@ -3166,8 +3490,11 @@ async function compare(domainA, domainB, options) {
   calculateOverallScore,
   classifyRendering,
   compare,
+  crawlFullSite,
   detectParkedDomain,
+  extractAllUrlsFromSitemap,
   extractContentPagesFromSitemap,
+  extractInternalLinks,
   extractNavLinks,
   extractRawDataSummary,
   fetchMultiPageData,
@@ -3178,6 +3505,7 @@ async function compare(domainA, domainB, options) {
   generateOpportunities,
   generatePitchNumbers,
   generateVerdict,
+  inferCategory,
   isSpaShell,
   prefetchSiteData,
   scoreToStatus