@rog0x/mcp-seo-tools 1.0.0

package/src/tools/sitemap-parser.ts

@@ -0,0 +1,230 @@
+import * as cheerio from "cheerio";
+
+export interface SitemapUrl {
+  loc: string;
+  lastmod: string | null;
+  changefreq: string | null;
+  priority: string | null;
+}
+
+export interface SitemapIndex {
+  loc: string;
+  lastmod: string | null;
+}
+
+export interface SitemapAnalysis {
+  url: string;
+  type: "urlset" | "sitemapindex" | "not_found";
+  urls: SitemapUrl[];
+  sitemapIndexEntries: SitemapIndex[];
+  totalUrls: number;
+  stats: {
+    withLastmod: number;
+    withChangefreq: number;
+    withPriority: number;
+    uniqueHosts: string[];
+    oldestLastmod: string | null;
+    newestLastmod: string | null;
+  };
+  issues: string[];
+  recommendations: string[];
+}
+
+async function fetchSitemap(url: string): Promise<string> {
+  const response = await fetch(url, {
+    headers: { "User-Agent": "MCPSEOTools/1.0 (Sitemap Parser)" },
+    redirect: "follow",
+    signal: AbortSignal.timeout(20000),
+  });
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} fetching sitemap at ${url}`);
+  }
+  return response.text();
+}
+
+function resolveSitemapUrl(pageUrl: string): string {
+  try {
+    const parsed = new URL(pageUrl);
+    // If the URL already ends with sitemap.xml or similar, use it directly
+    if (parsed.pathname.includes("sitemap")) return pageUrl;
+    // Otherwise, try the standard location
+    return `${parsed.protocol}//${parsed.host}/sitemap.xml`;
+  } catch {
+    return pageUrl;
+  }
+}
+
+export async function parseSitemap(url: string): Promise<SitemapAnalysis> {
+  const sitemapUrl = resolveSitemapUrl(url);
+  let xml: string;
+
+  try {
+    xml = await fetchSitemap(sitemapUrl);
+  } catch (err: any) {
+    // Try robots.txt for sitemap location
+    try {
+      const robotsUrl = `${new URL(sitemapUrl).protocol}//${new URL(sitemapUrl).host}/robots.txt`;
+      const robotsResp = await fetch(robotsUrl, {
+        headers: { "User-Agent": "MCPSEOTools/1.0 (Sitemap Parser)" },
+        signal: AbortSignal.timeout(10000),
+      });
+      const robotsTxt = await robotsResp.text();
+      const sitemapMatch = robotsTxt.match(/Sitemap:\s*(.+)/i);
+      if (sitemapMatch) {
+        xml = await fetchSitemap(sitemapMatch[1].trim());
+      } else {
+        return {
+          url: sitemapUrl,
+          type: "not_found",
+          urls: [],
+          sitemapIndexEntries: [],
+          totalUrls: 0,
+          stats: { withLastmod: 0, withChangefreq: 0, withPriority: 0, uniqueHosts: [], oldestLastmod: null, newestLastmod: null },
+          issues: [`No sitemap found at ${sitemapUrl} and no Sitemap directive in robots.txt.`],
+          recommendations: ["Create a sitemap.xml and submit it to search engines via Google Search Console."],
+        };
+      }
+    } catch {
+      return {
+        url: sitemapUrl,
+        type: "not_found",
+        urls: [],
+        sitemapIndexEntries: [],
+        totalUrls: 0,
+        stats: { withLastmod: 0, withChangefreq: 0, withPriority: 0, uniqueHosts: [], oldestLastmod: null, newestLastmod: null },
+        issues: [`Failed to fetch sitemap: ${err.message}`],
+        recommendations: ["Ensure a sitemap.xml exists at the root of your domain."],
+      };
+    }
+  }
+
+  const $ = cheerio.load(xml, { xml: true });
+
+  const issues: string[] = [];
+  const recommendations: string[] = [];
+
+  // Check if it's a sitemap index
+  const sitemapIndexEntries: SitemapIndex[] = [];
+  $("sitemapindex > sitemap").each((_, el) => {
+    sitemapIndexEntries.push({
+      loc: $(el).find("loc").text().trim(),
+      lastmod: $(el).find("lastmod").text().trim() || null,
+    });
+  });
+
+  if (sitemapIndexEntries.length > 0) {
+    if (sitemapIndexEntries.length > 500) {
+      issues.push(`Sitemap index has ${sitemapIndexEntries.length} entries. Google supports up to 500 sitemaps per index.`);
+    }
+    const withoutLastmod = sitemapIndexEntries.filter((s) => !s.lastmod).length;
+    if (withoutLastmod > 0) {
+      recommendations.push(`${withoutLastmod} sitemap index entries lack lastmod dates. Adding them helps search engines prioritize crawling.`);
+    }
+
+    return {
+      url: sitemapUrl,
+      type: "sitemapindex",
+      urls: [],
+      sitemapIndexEntries,
+      totalUrls: sitemapIndexEntries.length,
+      stats: { withLastmod: sitemapIndexEntries.length - (sitemapIndexEntries.filter((s) => !s.lastmod).length), withChangefreq: 0, withPriority: 0, uniqueHosts: [], oldestLastmod: null, newestLastmod: null },
+      issues,
+      recommendations,
+    };
+  }
+
+  // Parse URL set
+  const urls: SitemapUrl[] = [];
+  $("urlset > url").each((_, el) => {
+    urls.push({
+      loc: $(el).find("loc").text().trim(),
+      lastmod: $(el).find("lastmod").text().trim() || null,
+      changefreq: $(el).find("changefreq").text().trim() || null,
+      priority: $(el).find("priority").text().trim() || null,
+    });
+  });
+
+  // Stats
+  const withLastmod = urls.filter((u) => u.lastmod).length;
+  const withChangefreq = urls.filter((u) => u.changefreq).length;
+  const withPriority = urls.filter((u) => u.priority).length;
+
+  const hosts = new Set<string>();
+  for (const u of urls) {
+    try {
+      hosts.add(new URL(u.loc).hostname);
+    } catch { /* skip */ }
+  }
+
+  const lastmods = urls
+    .map((u) => u.lastmod)
+    .filter((d): d is string => d !== null)
+    .sort();
+  const oldestLastmod = lastmods.length > 0 ? lastmods[0] : null;
+  const newestLastmod = lastmods.length > 0 ? lastmods[lastmods.length - 1] : null;
+
+  // Analysis
+  if (urls.length === 0) {
+    issues.push("Sitemap contains no URLs.");
+  }
+
+  if (urls.length > 50000) {
+    issues.push(`Sitemap has ${urls.length} URLs. Maximum allowed per sitemap file is 50,000. Split into multiple sitemaps.`);
+  }
+
+  if (withLastmod === 0 && urls.length > 0) {
+    recommendations.push("No lastmod dates found. Adding lastmod helps search engines identify updated content.");
+  } else if (withLastmod < urls.length * 0.5) {
+    recommendations.push(`Only ${withLastmod} of ${urls.length} URLs have lastmod dates. Add dates to all entries.`);
+  }
+
+  // Check for stale lastmod
+  if (newestLastmod) {
+    const newestDate = new Date(newestLastmod);
+    const sixMonthsAgo = new Date();
+    sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6);
+    if (newestDate < sixMonthsAgo) {
+      issues.push(`Most recent lastmod is ${newestLastmod}. The sitemap appears outdated. Update it regularly.`);
+    }
+  }
+
+  // Check for duplicate URLs
+  const locSet = new Set<string>();
+  let duplicates = 0;
+  for (const u of urls) {
+    if (locSet.has(u.loc)) duplicates++;
+    locSet.add(u.loc);
+  }
+  if (duplicates > 0) {
+    issues.push(`${duplicates} duplicate URL(s) found. Remove duplicates to avoid crawl budget waste.`);
+  }
+
+  // Check for non-canonical patterns
+  const mixedProtocol = urls.some((u) => u.loc.startsWith("http://")) && urls.some((u) => u.loc.startsWith("https://"));
+  if (mixedProtocol) {
+    issues.push("Sitemap contains both HTTP and HTTPS URLs. Use only HTTPS URLs.");
+  }
+
+  const mixedTrailingSlash = urls.some((u) => u.loc.endsWith("/")) && urls.some((u) => !u.loc.endsWith("/") && !u.loc.match(/\.\w{2,5}$/));
+  if (mixedTrailingSlash) {
+    recommendations.push("Inconsistent trailing slashes in URLs. Standardize to one pattern for cleaner crawling.");
+  }
+
+  return {
+    url: sitemapUrl,
+    type: "urlset",
+    urls: urls.slice(0, 200), // Cap output to avoid overwhelming responses
+    sitemapIndexEntries: [],
+    totalUrls: urls.length,
+    stats: {
+      withLastmod,
+      withChangefreq,
+      withPriority,
+      uniqueHosts: [...hosts],
+      oldestLastmod,
+      newestLastmod,
+    },
+    issues,
+    recommendations,
+  };
+}

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "outDir": "dist",
+    "rootDir": "src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "resolveJsonModule": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}