aeorank 2.3.1 → 2.3.2

package/dist/index.js

@@ -185,16 +185,20 @@ async function prefetchSiteData(domain) {
       }
     }
   }
-  let blogSample = [];
-  if (sitemapXml && sitemapXml.status === 200) {
-    let sitemapForBlog = sitemapXml.text;
-    const subSitemapUrl = extractSubSitemapUrl(sitemapForBlog);
-    if (subSitemapUrl) {
-      const subSitemap = await fetchText(subSitemapUrl);
-      if (subSitemap && subSitemap.status === 200) {
-        sitemapForBlog = subSitemap.text;
+  if (sitemapXml && sitemapXml.status === 200 && sitemapXml.text.includes("<sitemapindex")) {
+    const subUrls = extractAllSubSitemapUrls(sitemapXml.text, 5);
+    if (subUrls.length > 0) {
+      const subResults = await Promise.all(subUrls.map((u) => fetchText(u)));
+      for (const sub of subResults) {
+        if (sub && sub.status === 200) {
+          sitemapXml.text += "\n" + sub.text;
+        }
       }
     }
+  }
+  let blogSample = [];
+  if (sitemapXml && sitemapXml.status === 200) {
+    const sitemapForBlog = sitemapXml.text;
     const blogUrls = extractBlogUrlsFromSitemap(sitemapForBlog, domain, 50);
     if (blogUrls.length > 0) {
       const fetched = await Promise.all(blogUrls.map((url) => fetchText(url)));
@@ -1239,16 +1243,16 @@ function extractBlogUrlsFromSitemap(sitemapText, domain, limit = 50) {
   });
   return candidates.slice(0, limit).map((c) => c.url);
 }
-function extractSubSitemapUrl(sitemapText) {
-  if (!sitemapText.includes("<sitemapindex")) return null;
+function extractAllSubSitemapUrls(sitemapText, limit = 5) {
+  if (!sitemapText.includes("<sitemapindex")) return [];
   const sitemapLocs = sitemapText.match(/<sitemap>[\s\S]*?<loc>([^<]+)<\/loc>[\s\S]*?<\/sitemap>/gi) || [];
-  if (sitemapLocs.length === 0) return null;
   const urls = sitemapLocs.map((block) => {
     const match = block.match(/<loc>([^<]+)<\/loc>/i);
     return match ? match[1].trim() : "";
   }).filter(Boolean);
-  const preferred = urls.find((u) => /post|blog|article/i.test(u));
-  return preferred || urls[0] || null;
+  const preferred = urls.filter((u) => /post|blog|article|page/i.test(u));
+  const rest = urls.filter((u) => !preferred.includes(u));
+  return [...preferred, ...rest].slice(0, limit);
 }
 function checkContentVelocity(data) {
   const findings = [];