aeorank 2.1.0 → 2.2.0

package/dist/cli.js

@@ -2737,6 +2737,38 @@ function extractNavLinks(html, domain) {
   }
   return Array.from(paths);
 }
+function extractAllInternalLinks(html, domain, limit = 30) {
+  const cleanDomain = domain.replace(/^www\./, "").toLowerCase();
+  const hrefMatches = html.match(/href="([^"#]*)"/gi) || [];
+  const paths = /* @__PURE__ */ new Set();
+  for (const match of hrefMatches) {
+    const href = match.match(/href="([^"#]*)"/i)?.[1];
+    if (!href) continue;
+    let path;
+    if (href.startsWith("/")) {
+      path = href;
+    } else if (href.startsWith("http")) {
+      try {
+        const url = new URL(href);
+        const linkDomain = url.hostname.replace(/^www\./, "").toLowerCase();
+        if (linkDomain !== cleanDomain) continue;
+        path = url.pathname;
+      } catch {
+        continue;
+      }
+    } else {
+      continue;
+    }
+    path = path.replace(/\/+$/, "") || "/";
+    if (path === "/") continue;
+    if (path.includes("#") || path.includes("?")) continue;
+    if (/\.(js|css|png|jpg|jpeg|gif|svg|ico|pdf|xml|txt|zip|woff|woff2|ttf|eot|mp4|webm|mp3)$/i.test(path)) continue;
+    if (/^\/(api|wp-admin|wp-includes|wp-json|static|assets|_next|auth|login|signup|sign-up|register|cart|checkout|account|admin|cdn-cgi|feed|rss)\b/i.test(path)) continue;
+    if (path.startsWith("mailto:") || path.startsWith("tel:")) continue;
+    paths.add(path);
+  }
+  return Array.from(paths).sort((a, b) => a.split("/").length - b.split("/").length || a.localeCompare(b)).slice(0, limit);
+}
 function extractContentPagesFromSitemap(sitemapText, domain, limit = 6) {
   const urlBlocks = sitemapText.match(/<url>([\s\S]*?)<\/url>/gi) || [];
   const cleanDomain = domain.replace(/^www\./, "").toLowerCase();
@@ -2804,6 +2836,16 @@ async function fetchMultiPageData(siteData, options) {
       if (!existingUrls.has(url)) urlsToFetch.set(url, "content");
     }
   }
+  const hasBlogSample = (siteData.blogSample?.length ?? 0) > 3;
+  if (!hasBlogSample) {
+    const allPaths = extractAllInternalLinks(siteData.homepage.text, siteData.domain, 30);
+    for (const path of allPaths) {
+      const url = `${baseUrl}${path}`;
+      if (!existingUrls.has(url) && !urlsToFetch.has(url)) {
+        urlsToFetch.set(url, "content");
+      }
+    }
+  }
   const entries = Array.from(urlsToFetch.entries());
   if (entries.length === 0) return 0;
   const results = await Promise.all(entries.map(([url]) => fetchPage(url, timeoutMs)));