@automatelab/ai-seo-mcp 0.1.0

package/dist/tools/check-robots.js

@@ -0,0 +1,142 @@
+// Tool: check_robots
+// Parses a domain's robots.txt and reports per-crawler allow/disallow posture.
+import { z } from "zod";
+import { fetchRobotsTxt, checkCrawlerStatus, CRAWLERS } from "../lib/robots.js";
+export const checkRobotsInputSchema = z.object({
+    domain: z.string().min(3),
+});
+function normalizeDomain(domain) {
+    let d = domain.trim();
+    if (d.startsWith("http://") || d.startsWith("https://")) {
+        try {
+            const url = new URL(d);
+            return url.hostname;
+        }
+        catch {
+            return d;
+        }
+    }
+    return d.replace(/\/$/, "");
+}
+export async function checkRobots(input) {
+    const hostname = normalizeDomain(input.domain);
+    const robotsUrl = `https://${hostname}/robots.txt`;
+    const fetched_at = new Date().toISOString();
+    const findings = [];
+    const robotsText = await fetchRobotsTxt(robotsUrl);
+    if (!robotsText) {
+        findings.push({
+            severity: "warning",
+            category: "robots",
+            where: robotsUrl,
+            message: "robots.txt not found (404 or empty response).",
+            fix: "Create a robots.txt file at the domain root. All AI crawlers default to allow-all when it is absent.",
+            estimated_impact: "medium",
+        });
+    }
+    // Evaluate each crawler category
+    const training_crawlers = {};
+    for (const agent of CRAWLERS.training) {
+        training_crawlers[agent] = checkCrawlerStatus(robotsText, robotsUrl, agent);
+    }
+    const search_crawlers = {};
+    for (const agent of CRAWLERS.search) {
+        search_crawlers[agent] = checkCrawlerStatus(robotsText, robotsUrl, agent);
+    }
+    const user_triggered = {};
+    for (const agent of CRAWLERS.user_triggered) {
+        user_triggered[agent] = checkCrawlerStatus(robotsText, robotsUrl, agent);
+    }
+    // Emit findings for key conditions
+    for (const agent of CRAWLERS.training) {
+        if (training_crawlers[agent] !== "disallowed") {
+            findings.push({
+                severity: "warning",
+                category: "robots",
+                where: `robots.txt User-agent: ${agent}`,
+                message: `${agent} is not disallowed. Your content may be harvested for model training.`,
+                fix: `Add:\nUser-agent: ${agent}\nDisallow: /`,
+                estimated_impact: "medium",
+            });
+        }
+    }
+    // OAI-SearchBot should be explicitly allowed
+    if (search_crawlers["OAI-SearchBot"] === "not-mentioned") {
+        findings.push({
+            severity: "info",
+            category: "robots",
+            where: "robots.txt User-agent: OAI-SearchBot",
+            message: "OAI-SearchBot is not mentioned. Explicit Allow: / increases ChatGPT search citation probability.",
+            fix: "Add:\nUser-agent: OAI-SearchBot\nAllow: /",
+        });
+    }
+    // PerplexityBot blocked warning
+    if (search_crawlers["PerplexityBot"] === "disallowed") {
+        findings.push({
+            severity: "warning",
+            category: "robots",
+            where: "robots.txt User-agent: PerplexityBot",
+            message: "PerplexityBot is blocked. This prevents citations in Perplexity AI answers.",
+            fix: "Remove the PerplexityBot Disallow directive or add Allow: /.",
+            estimated_impact: "high",
+        });
+    }
+    // GPTBot blocked but OAI-SearchBot not explicitly allowed
+    if (training_crawlers["GPTBot"] === "disallowed" &&
+        search_crawlers["OAI-SearchBot"] !== "allowed") {
+        findings.push({
+            severity: "warning",
+            category: "robots",
+            where: "robots.txt",
+            message: "GPTBot is blocked (training) but OAI-SearchBot is not explicitly allowed (search). These are independently controllable.",
+            fix: "Add:\nUser-agent: OAI-SearchBot\nAllow: /\nto preserve ChatGPT search citations.",
+            estimated_impact: "high",
+        });
+    }
+    // Robots-token-only notes
+    for (const token of CRAWLERS.robots_token_only) {
+        const lowerText = (robotsText ?? "").toLowerCase();
+        if (lowerText.includes(token.toLowerCase())) {
+            findings.push({
+                severity: "info",
+                category: "robots",
+                where: `robots.txt User-agent: ${token}`,
+                message: `${token} is a training-opt-out token, not an HTTP crawler. It does not affect search crawling.`,
+                fix: "No action needed - this token correctly controls training data opt-out.",
+            });
+        }
+    }
+    // Google-Agent always present as info
+    findings.push({
+        severity: "info",
+        category: "robots",
+        where: "robots.txt User-agent: Google-Agent",
+        message: "Google-Agent (Mariner) ignores robots.txt per Google's documentation. You cannot opt out via robots.txt.",
+        fix: "No robots.txt action possible for Google-Agent. Use noindex meta tags if you need page-level exclusion.",
+    });
+    // Determine recommended posture
+    const allTrainingBlocked = CRAWLERS.training.every((a) => training_crawlers[a] === "disallowed");
+    const allSearchAllowedOrUnmentioned = CRAWLERS.search.every((a) => search_crawlers[a] !== "disallowed");
+    const allTrainingAllowed = CRAWLERS.training.every((a) => training_crawlers[a] !== "disallowed");
+    const allSearchBlocked = CRAWLERS.search.every((a) => search_crawlers[a] === "disallowed");
+    let recommended_posture = "custom";
+    if (allTrainingBlocked && allSearchAllowedOrUnmentioned) {
+        recommended_posture = "block_training_allow_search";
+    }
+    else if (allTrainingAllowed && !allSearchBlocked) {
+        recommended_posture = "allow_all";
+    }
+    else if (allTrainingBlocked && allSearchBlocked) {
+        recommended_posture = "block_all";
+    }
+    return {
+        robots_url: robotsUrl,
+        fetched_at,
+        training_crawlers,
+        search_crawlers,
+        user_triggered,
+        findings,
+        recommended_posture,
+    };
+}
+//# sourceMappingURL=check-robots.js.map

package/dist/tools/check-robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"check-robots.js","sourceRoot":"","sources":["../../src/tools/check-robots.ts"],"names":[],"mappings":"AAAA,qBAAqB;AACrB,+EAA+E;AAE/E,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,QAAQ,EAAsB,MAAM,kBAAkB,CAAC;AAGpG,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;CAC1B,CAAC,CAAC;AAcH,SAAS,eAAe,CAAC,MAAc;IACrC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QACxD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;YACvB,OAAO,GAAG,CAAC,QAAQ,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IACD,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,KAAuB;IACvD,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,WAAW,QAAQ,aAAa,CAAC;IACnD,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5C,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC;IAEnD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,QAAQ,CAAC,IAAI,CAAC;YACZ,QAAQ,EAAE,SAAS;YACnB,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,+CAA+C;YACxD,GAAG,EAAE,sGAAsG;YAC3G,gBAAgB,EAAE,QAAQ;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,iCAAiC;IACjC,MAAM,iBAAiB,GAAkC,EAAE,CAAC;IAC5D,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACtC,iBAAiB,CAAC,KAAK,CAAC,GAAG,kBAAkB,CAAC,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9E,CAAC;IAED,MAAM,eAAe,GAAkC,EAAE,CAAC;IAC1D,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,eAAe,CAAC,KAAK,CAAC,GAAG,kBAAkB,CAAC,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,cAAc,GAAkC,EAAE,CAAC;IACzD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;QAC5C,cAAc,CAAC,KAAK,CAAC,GAAG,kBAAkB,CAAC,UAAU,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAC3E,CAAC;IAED,mCAAmC;IACnC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,iBAAiB,CAAC,KAAK,CAAC,KAAK,YAAY,EAAE,CAAC;YAC9C,QAAQ,CAAC,IAAI,CAAC;gBACZ,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,QAAQ;gBAClB,KAAK,EAAE,0BAA0B,KAAK,EAAE;gBACxC,OAAO,EAAE,GAAG,KAAK,uEAAuE;gBACxF,GAAG,EAAE,qBAAqB,KAAK,eAAe;gBAC9C,gBAAgB,EAAE,QAAQ;aAC3B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,IAAI,eAAe,CAAC,eAAe,CAAC,KAAK,eAAe,EAAE,CAAC;QACzD,QAAQ,CAAC,IAAI,CAAC;YACZ,QAAQ,EAAE,MAAM;YAChB,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,sCAAsC;YAC7C,OAAO,EAAE,kGAAkG;YAC3G,GAAG,EAAE,2CAA2C;SACjD,CAAC,CAAC;IACL,CAAC;IAED,gCAAgC;IAChC,IAAI,eAAe,CAAC,eAAe,CAAC,KAAK,YAAY,EAAE,CAAC;QACtD,QAAQ,CAAC,IAAI,CAAC;YACZ,QAAQ,EAAE,SAAS;YACnB,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,sCAAsC;YAC7C,OAAO,EAAE,6EAA6E;YACtF,GAAG,EAAE,8DAA8D;YACnE,gBAAgB,EAAE,MAAM;SACzB,CAAC,CAAC;IACL,CAAC;IAED,0DAA0D;IAC1D,IACE,iBAAiB,CAAC,QAAQ,CAAC,KAAK,YAAY;QAC5C,eAAe,CAAC,eAAe,CAAC,KAAK,SAAS,EAC9C,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,QAAQ,EAAE,SAAS;YACnB,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,YAAY;YACnB,OAAO,EACL,0HAA0H;YAC5H,GAAG,EAAE,kFAAkF;YACvF,gBAAgB,EAAE,MAAM;SACzB,CAAC,CAAC;IACL,CAAC;IAED,0BAA0B;IAC1B,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/C,MAAM,SAAS,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACnD,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAC5C,QAAQ,CAAC,IAAI,CAAC;gBACZ,QAAQ,EAAE,MAAM;gBAChB,QAAQ,EAAE,QAAQ;gBAClB,KAAK,EAAE,0BAA0B,KAAK,EAAE;gBACxC,OAAO,EAAE,GAAG,KAAK,wFAAwF;gBACzG,GAAG,EAAE,yEAAyE;aAC/E,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,QAAQ,CAAC,IAAI,CAAC;QACZ,QAAQ,EAAE,MAAM;QAChB,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,qCAAqC;QAC5C,OAAO,EAAE,0GAA0G;QACnH,GAAG,EAAE,yGAAyG;KAC/G,CAAC,CAAC;IAEH,gCAAgC;IAChC,MAAM,kBAAkB,GAAG,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAChD,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,KAAK,YAAY,CAC7C,CAAC;IACF,MAAM,6BAA6B,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CACzD,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,YAAY,CAC3C,CAAC;IACF,MAAM,kBAAkB,GAAG,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAChD,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,KAAK,YAAY,CAC7C,CAAC;IACF,MAAM,gBAAgB,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAC5C,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,YAAY,CAC3C,CAAC;IAEF,IAAI,mBAAmB,GAAwC,QAAQ,CAAC;IACxE,IAAI,kBAAkB,IAAI,6BAA6B,EAAE,CAAC;QACxD,mBAAmB,GAAG,6BAA6B,CAAC;IACtD,CAAC;SAAM,IAAI,kBAAkB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACnD,mBAAmB,GAAG,WAAW,CAAC;IACpC,CAAC;SAAM,IAAI,kBAAkB,IAAI,gBAAgB,EAAE,CAAC;QAClD,mBAAmB,GAAG,WAAW,CAAC;IACpC,CAAC;IAED,OAAO;QACL,UAAU,EAAE,SAAS;QACrB,UAAU;QACV,iBAAiB;QACjB,eAAe;QACf,cAAc;QACd,QAAQ;QACR,mBAAmB;KACpB,CAAC;AACJ,CAAC"}

package/dist/tools/check-sitemap.d.ts

@@ -0,0 +1,27 @@
+import { z } from "zod";
+import { type HostDelayMap } from "../lib/fetch.js";
+import type { Finding } from "../types.js";
+export declare const checkSitemapInputSchema: z.ZodObject<{
+    domain: z.ZodString;
+    max_urls_to_check: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+}, "strip", z.ZodTypeAny, {
+    domain: string;
+    max_urls_to_check: number;
+}, {
+    domain: string;
+    max_urls_to_check?: number | undefined;
+}>;
+export type CheckSitemapInput = z.infer<typeof checkSitemapInputSchema>;
+export interface SitemapResult {
+    sitemap_url: string | null;
+    status: "found" | "missing" | "error";
+    total_urls: number;
+    urls_with_lastmod: number;
+    stale_urls: number;
+    has_image_sitemap: boolean;
+    has_video_sitemap: boolean;
+    sitemap_index: boolean;
+    findings: Finding[];
+}
+export declare function checkSitemap(input: CheckSitemapInput, hostDelays?: HostDelayMap, robotsCache?: Map<string, string>): Promise<SitemapResult>;
+//# sourceMappingURL=check-sitemap.d.ts.map

package/dist/tools/check-sitemap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"check-sitemap.d.ts","sourceRoot":"","sources":["../../src/tools/check-sitemap.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAe,KAAK,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAGjE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAE3C,eAAO,MAAM,uBAAuB;;;;;;;;;EAGlC,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,OAAO,GAAG,SAAS,GAAG,OAAO,CAAC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,OAAO,CAAC;IAC3B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAsFD,wBAAsB,YAAY,CAChC,KAAK,EAAE,iBAAiB,EACxB,UAAU,CAAC,EAAE,YAAY,EACzB,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAChC,OAAO,CAAC,aAAa,CAAC,CA0IxB"}

package/dist/tools/check-sitemap.js

@@ -0,0 +1,214 @@
+// Tool: check_sitemap
+// Validates a domain's XML sitemap for presence, accessibility, and AI-search attributes.
+import { z } from "zod";
+import { politeFetch } from "../lib/fetch.js";
+import { fetchRobotsTxt } from "../lib/robots.js";
+import { XMLParser } from "fast-xml-parser";
+export const checkSitemapInputSchema = z.object({
+    domain: z.string().min(3),
+    max_urls_to_check: z.number().int().min(1).max(500).optional().default(100),
+});
+function normalizeDomain(domain) {
+    if (domain.startsWith("http://") || domain.startsWith("https://")) {
+        try {
+            return new URL(domain).hostname;
+        }
+        catch {
+            return domain;
+        }
+    }
+    return domain.replace(/\/$/, "");
+}
+function isStale(lastmod) {
+    try {
+        const d = new Date(lastmod).getTime();
+        const ageDays = (Date.now() - d) / (1000 * 60 * 60 * 24);
+        return ageDays > 90;
+    }
+    catch {
+        return false;
+    }
+}
+function parseSitemapXml(xml, sitemapUrl) {
+    const parser = new XMLParser({ ignoreAttributes: false });
+    let parsed;
+    try {
+        parsed = parser.parse(xml);
+    }
+    catch {
+        return { urls: [], isSitemapIndex: false, childSitemaps: [] };
+    }
+    // Sitemap index
+    const sitemapIndex = parsed["sitemapindex"];
+    if (sitemapIndex) {
+        const sitemaps = sitemapIndex["sitemap"];
+        const sitemapList = Array.isArray(sitemaps)
+            ? sitemaps
+            : sitemaps
+                ? [sitemaps]
+                : [];
+        const childSitemaps = sitemapList
+            .map((s) => String(s["loc"] ?? ""))
+            .filter(Boolean);
+        return { urls: [], isSitemapIndex: true, childSitemaps };
+    }
+    // Regular sitemap
+    const urlset = parsed["urlset"];
+    if (!urlset)
+        return { urls: [], isSitemapIndex: false, childSitemaps: [] };
+    const urlEntries = urlset["url"];
+    const urlList = Array.isArray(urlEntries)
+        ? urlEntries
+        : urlEntries
+            ? [urlEntries]
+            : [];
+    const urls = urlList.map((u) => ({
+        loc: String(u["loc"] ?? ""),
+        lastmod: u["lastmod"] ? String(u["lastmod"]) : undefined,
+        priority: u["priority"] ? String(u["priority"]) : undefined,
+        changefreq: u["changefreq"] ? String(u["changefreq"]) : undefined,
+    }));
+    // Check for image/video namespaces in raw XML
+    const hasImage = xml.includes("image:") || xml.includes("xmlns:image");
+    const hasVideo = xml.includes("video:") || xml.includes("xmlns:video");
+    return {
+        urls,
+        isSitemapIndex: false,
+        childSitemaps: [],
+    };
+}
+export async function checkSitemap(input, hostDelays, robotsCache) {
+    const hostname = normalizeDomain(input.domain);
+    const findings = [];
+    let sitemapUrl = null;
+    let allUrls = [];
+    let isSitemapIndex = false;
+    let hasImageSitemap = false;
+    let hasVideoSitemap = false;
+    // Try to discover sitemap
+    const candidates = [
+        `https://${hostname}/sitemap.xml`,
+        `https://${hostname}/sitemap_index.xml`,
+    ];
+    // Check robots.txt for Sitemap: directive
+    const robotsText = await fetchRobotsTxt(`https://${hostname}/robots.txt`);
+    if (robotsText) {
+        const sitemapMatch = robotsText.match(/^Sitemap:\s*(.+)$/im);
+        if (sitemapMatch && sitemapMatch[1]) {
+            const robotsSitemapUrl = sitemapMatch[1].trim();
+            if (!candidates.includes(robotsSitemapUrl)) {
+                candidates.unshift(robotsSitemapUrl); // try first
+            }
+        }
+    }
+    let sitemapXml = null;
+    for (const candidate of candidates) {
+        try {
+            const res = await politeFetch(candidate, {
+                respectRobots: false, // sitemaps are always allowed
+                hostDelays,
+                robotsCache,
+            });
+            if (res.statusCode === 200) {
+                sitemapUrl = candidate;
+                sitemapXml = res.body;
+                break;
+            }
+        }
+        catch {
+            // try next candidate
+        }
+    }
+    if (!sitemapXml || !sitemapUrl) {
+        findings.push({
+            severity: "critical",
+            category: "sitemap",
+            where: `https://${hostname}/sitemap.xml`,
+            message: "No sitemap found at standard locations.",
+            fix: "Create a sitemap.xml and declare it in robots.txt with 'Sitemap: https://yourdomain.com/sitemap.xml'.",
+            estimated_impact: "medium",
+        });
+        return {
+            sitemap_url: null,
+            status: "missing",
+            total_urls: 0,
+            urls_with_lastmod: 0,
+            stale_urls: 0,
+            has_image_sitemap: false,
+            has_video_sitemap: false,
+            sitemap_index: false,
+            findings,
+        };
+    }
+    hasImageSitemap = sitemapXml.includes("image:") || sitemapXml.includes("xmlns:image");
+    hasVideoSitemap = sitemapXml.includes("video:") || sitemapXml.includes("xmlns:video");
+    const { urls, isSitemapIndex: isIndex, childSitemaps } = parseSitemapXml(sitemapXml, sitemapUrl);
+    isSitemapIndex = isIndex;
+    allUrls = urls;
+    // Load child sitemaps (up to 3 levels, capped at max_urls_to_check)
+    if (isIndex && childSitemaps.length > 0) {
+        let loaded = 0;
+        for (const childUrl of childSitemaps) {
+            if (allUrls.length >= input.max_urls_to_check)
+                break;
+            if (loaded >= 3)
+                break; // max 3 child sitemaps for now
+            try {
+                const childRes = await politeFetch(childUrl, {
+                    respectRobots: false,
+                    hostDelays,
+                    robotsCache,
+                });
+                const { urls: childUrls } = parseSitemapXml(childRes.body, childUrl);
+                if (childRes.body.includes("image:"))
+                    hasImageSitemap = true;
+                if (childRes.body.includes("video:"))
+                    hasVideoSitemap = true;
+                allUrls.push(...childUrls);
+                loaded++;
+            }
+            catch {
+                // skip failed child sitemaps
+            }
+        }
+    }
+    // Cap at max_urls_to_check
+    const checked = allUrls.slice(0, input.max_urls_to_check);
+    const total_urls = checked.length;
+    const urls_with_lastmod = checked.filter((u) => !!u.lastmod).length;
+    const stale_urls = checked.filter((u) => u.lastmod && isStale(u.lastmod)).length;
+    // Findings
+    const missingLastmod = total_urls - urls_with_lastmod;
+    if (total_urls > 0 && missingLastmod / total_urls > 0.5) {
+        findings.push({
+            severity: "warning",
+            category: "sitemap",
+            where: "sitemap.xml",
+            message: `${Math.round((missingLastmod / total_urls) * 100)}% of URLs are missing lastmod dates - reduces freshness signals.`,
+            fix: "Add <lastmod> to all <url> entries. Use ISO 8601 format (YYYY-MM-DD).",
+            estimated_impact: "medium",
+        });
+    }
+    if (total_urls > 0 && stale_urls / total_urls > 0.5) {
+        findings.push({
+            severity: "warning",
+            category: "sitemap",
+            where: "sitemap.xml",
+            message: `${Math.round((stale_urls / total_urls) * 100)}% of URLs have lastmod older than 90 days.`,
+            fix: "Update lastmod dates when content changes. Stale content signals reduce AI citation probability.",
+            estimated_impact: "medium",
+        });
+    }
+    return {
+        sitemap_url: sitemapUrl,
+        status: "found",
+        total_urls,
+        urls_with_lastmod,
+        stale_urls,
+        has_image_sitemap: hasImageSitemap,
+        has_video_sitemap: hasVideoSitemap,
+        sitemap_index: isSitemapIndex,
+        findings,
+    };
+}
+//# sourceMappingURL=check-sitemap.js.map

package/dist/tools/check-sitemap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"check-sitemap.js","sourceRoot":"","sources":["../../src/tools/check-sitemap.ts"],"names":[],"mappings":"AAAA,sBAAsB;AACtB,0FAA0F;AAE1F,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,WAAW,EAAqB,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;CAC5E,CAAC,CAAC;AAgBH,SAAS,eAAe,CAAC,MAAc;IACrC,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAClE,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;QAClC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,MAAM,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACnC,CAAC;AASD,SAAS,OAAO,CAAC,OAAe;IAC9B,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACzD,OAAO,OAAO,GAAG,EAAE,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CACtB,GAAW,EACX,UAAkB;IAElB,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1D,IAAI,MAA+B,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC;IAChE,CAAC;IAED,gBAAgB;IAChB,MAAM,YAAY,GAAG,MAAM,CAAC,cAAc,CAAwC,CAAC;IACnF,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,WAAW,GAAmC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YACzE,CAAC,CAAE,QAA2C;YAC9C,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,CAAC,QAAmC,CAAC;gBACvC,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,aAAa,GAAG,WAAW;aAC9B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aAClC,MAAM,CAAC,OAAO,CAAC,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;IAC3D,CAAC;IAED,kBAAkB;IAClB,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAwC,CAAC;IACvE,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC;IAE3E,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,OAAO,GAAmC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;QACvE,CAAC,CAAE,UAA6C;QAChD,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,CAAC,UAAqC,CAAC;YACzC,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,IAAI,GAAiB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7C,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC3B,OAAO,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QACxD,QAAQ,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;QAC3D,UAAU,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;KAClE,CAAC,CAAC,CAAC;IAEJ,8CAA8C;IAC9C,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAEvE,OAAO;QACL,IAAI;QACJ,cAAc,EAAE,KAAK;QACrB,aAAa,EAAE,EAAE;KAClB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAwB,EACxB,UAAyB,EACzB,WAAiC;IAEjC,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,IAAI,UAAU,GAAkB,IAAI,CAAC;IACrC,IAAI,OAAO,GAAiB,EAAE,CAAC;IAC/B,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,IAAI,eAAe,GAAG,KAAK,CAAC;IAE5B,0BAA0B;IAC1B,MAAM,UAAU,GAAG;QACjB,WAAW,QAAQ,cAAc;QACjC,WAAW,QAAQ,oBAAoB;KACxC,CAAC;IAEF,0CAA0C;IAC1C,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,WAAW,QAAQ,aAAa,CAAC,CAAC;IAC1E,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,YAAY,GAAG,UAAU,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAC7D,IAAI,YAAY,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,MAAM,gBAAgB,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAChD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBAC3C,UAAU,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,YAAY;YACpD,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,UAAU,GAAkB,IAAI,CAAC;IACrC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE;gBACvC,aAAa,EAAE,KAAK,EAAE,8BAA8B;gBACpD,UAAU;gBACV,WAAW;aACZ,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBAC3B,UAAU,GAAG,SAAS,CAAC;gBACvB,UAAU,GAAG,GAAG,CAAC,IAAI,CAAC;gBACtB,MAAM;YACR,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;IACH,CAAC;IAED,IAAI,CAAC,UAAU,IAAI,CAAC,UAAU,EAAE,CAAC;QAC/B,QAAQ,CAAC,IAAI,CAAC;YACZ,QAAQ,EAAE,UAAU;YACpB,QAAQ,EAAE,SAAS;YACnB,KAAK,EAAE,WAAW,QAAQ,cAAc;YACxC,OAAO,EAAE,yCAAyC;YAClD,GAAG,EAAE,uGAAuG;YAC5G,gBAAgB,EAAE,QAAQ;SAC3B,CAAC,CAAC;QACH,OAAO;YACL,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,CAAC;YACb,iBAAiB,EAAE,CAAC;YACpB,UAAU,EAAE,CAAC;YACb,iBAAiB,EAAE,KAAK;YACxB,iBAAiB,EAAE,KAAK;YACxB,aAAa,EAAE,KAAK;YACpB,QAAQ;SACT,CAAC;IACJ,CAAC;IAED,eAAe,GAAG,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IACtF,eAAe,GAAG,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAEtF,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,eAAe,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IACjG,cAAc,GAAG,OAAO,CAAC;IACzB,OAAO,GAAG,IAAI,CAAC;IAEf,oEAAoE;IACpE,IAAI,OAAO,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;YACrC,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,iBAAiB;gBAAE,MAAM;YACrD,IAAI,MAAM,IAAI,CAAC;gBAAE,MAAM,CAAC,+BAA+B;YACvD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE;oBAC3C,aAAa,EAAE,KAAK;oBACpB,UAAU;oBACV,WAAW;iBACZ,CAAC,CAAC;gBACH,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBACrE,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBAAE,eAAe,GAAG,IAAI,CAAC;gBAC7D,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBAAE,eAAe,GAAG,IAAI,CAAC;gBAC7D,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC3B,MAAM,EAAE,CAAC;YACX,CAAC;YAAC,MAAM,CAAC;gBACP,6BAA6B;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAED,2BAA2B;IAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAClC,MAAM,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;IAEjF,WAAW;IACX,MAAM,cAAc,GAAG,UAAU,GAAG,iBAAiB,CAAC;IACtD,IAAI,UAAU,GAAG,CAAC,IAAI,cAAc,GAAG,UAAU,GAAG,GAAG,EAAE,CAAC;QACxD,QAAQ,CAAC,IAAI,CAAC;YACZ,QAAQ,EAAE,SAAS;YACnB,QAAQ,EAAE,SAAS;YACnB,KAAK,EAAE,aAAa;YACpB,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,kEAAkE;YAC7H,GAAG,EAAE,uEAAuE;YAC5E,gBAAgB,EAAE,QAAQ;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,GAAG,UAAU,GAAG,GAAG,EAAE,CAAC;QACpD,QAAQ,CAAC,IAAI,CAAC;YACZ,QAAQ,EAAE,SAAS;YACnB,QAAQ,EAAE,SAAS;YACnB,KAAK,EAAE,aAAa;YACpB,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,4CAA4C;YACnG,GAAG,EAAE,kGAAkG;YACvG,gBAAgB,EAAE,QAAQ;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,WAAW,EAAE,UAAU;QACvB,MAAM,EAAE,OAAO;QACf,UAAU;QACV,iBAAiB;QACjB,UAAU;QACV,iBAAiB,EAAE,eAAe;QAClC,iBAAiB,EAAE,eAAe;QAClC,aAAa,EAAE,cAAc;QAC7B,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/tools/check-technical.d.ts

@@ -0,0 +1,50 @@
+import { z } from "zod";
+import { type HostDelayMap } from "../lib/fetch.js";
+import type { Finding } from "../types.js";
+export declare const checkTechnicalInputSchema: z.ZodObject<{
+    url: z.ZodString;
+    respect_robots: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+}, "strip", z.ZodTypeAny, {
+    url: string;
+    respect_robots: boolean;
+}, {
+    url: string;
+    respect_robots?: boolean | undefined;
+}>;
+export type CheckTechnicalInput = z.infer<typeof checkTechnicalInputSchema>;
+export interface TechnicalResult {
+    url: string;
+    https: boolean;
+    canonical: {
+        present: boolean;
+        value: string | null;
+        self_referential: boolean;
+        cross_domain: boolean;
+    };
+    noindex: boolean;
+    noindex_header: boolean;
+    og_tags: {
+        title: boolean;
+        description: boolean;
+        image: boolean;
+        url: boolean;
+        type: boolean;
+    };
+    twitter_card: {
+        present: boolean;
+        card_type: string | null;
+    };
+    hreflang: {
+        present: boolean;
+        count: number;
+        x_default: boolean;
+    };
+    title_og_match: boolean;
+    meta_description: {
+        present: boolean;
+        length: number;
+    };
+    findings: Finding[];
+}
+export declare function checkTechnical(input: CheckTechnicalInput, hostDelays?: HostDelayMap, robotsCache?: Map<string, string>): Promise<TechnicalResult>;
+//# sourceMappingURL=check-technical.d.ts.map

package/dist/tools/check-technical.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"check-technical.d.ts","sourceRoot":"","sources":["../../src/tools/check-technical.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAA+B,KAAK,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAEjF,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAE3C,eAAO,MAAM,yBAAyB;;;;;;;;;EAGpC,CAAC;AAEH,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAE5E,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,EAAE;QACT,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,gBAAgB,EAAE,OAAO,CAAC;QAC1B,YAAY,EAAE,OAAO,CAAC;KACvB,CAAC;IACF,OAAO,EAAE,OAAO,CAAC;IACjB,cAAc,EAAE,OAAO,CAAC;IACxB,OAAO,EAAE;QACP,KAAK,EAAE,OAAO,CAAC;QACf,WAAW,EAAE,OAAO,CAAC;QACrB,KAAK,EAAE,OAAO,CAAC;QACf,GAAG,EAAE,OAAO,CAAC;QACb,IAAI,EAAE,OAAO,CAAC;KACf,CAAC;IACF,YAAY,EAAE;QACZ,OAAO,EAAE,OAAO,CAAC;QACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;KAC1B,CAAC;IACF,QAAQ,EAAE;QACR,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,OAAO,CAAC;KACpB,CAAC;IACF,cAAc,EAAE,OAAO,CAAC;IACxB,gBAAgB,EAAE;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAED,wBAAsB,cAAc,CAClC,KAAK,EAAE,mBAAmB,EAC1B,UAAU,CAAC,EAAE,YAAY,EACzB,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAChC,OAAO,CAAC,eAAe,CAAC,CAgN1B"}