@djangocfg/seo 2.1.50 → 2.1.52

package/dist/cli.mjs

@@ -1003,6 +1003,22 @@ async function analyzeRobotsTxt(siteUrl) {
     }
     analysis.exists = true;
     analysis.content = await response.text();
+    if (analysis.content.includes("content-signal") || analysis.content.includes("Content-Signal") || analysis.content.includes("ai-input") || analysis.content.includes("ai-train")) {
+      analysis.issues.push({
+        id: "cloudflare-managed-robots",
+        url: robotsUrl,
+        category: "technical",
+        severity: "warning",
+        title: "Cloudflare managed robots.txt detected",
+        description: `Your robots.txt is being overwritten by Cloudflare's "Content Signals Policy". Your app/robots.ts file is not being served.`,
+        recommendation: 'Disable in Cloudflare Dashboard: Security \u2192 Settings \u2192 "Manage your robots.txt" \u2192 Set to "Off".',
+        detectedAt: (/* @__PURE__ */ new Date()).toISOString(),
+        metadata: {
+          cloudflareFeature: "Managed robots.txt",
+          docsUrl: "https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/"
+        }
+      });
+    }
     const robots = robotsParser(robotsUrl, analysis.content);
     analysis.sitemaps = robots.getSitemaps();
     if (analysis.sitemaps.length === 0) {
@@ -2097,6 +2113,65 @@ function generateClaudeContext(report) {
   lines.push("- Parallel `@folder` - skipped");
   lines.push("- Private `_folder` - skipped");
   lines.push("");
+  lines.push("## SEO Files (Next.js App Router)");
+  lines.push("");
+  lines.push("**Required files in `app/`:**");
+  lines.push("");
+  lines.push("### sitemap.xml/route.ts");
+  lines.push("```typescript");
+  lines.push("import { createSitemapHandler } from '@djangocfg/nextjs/sitemap';");
+  lines.push("");
+  lines.push('export const dynamic = "force-static";');
+  lines.push("export const { GET } = createSitemapHandler({");
+  lines.push("  siteUrl,");
+  lines.push("  staticPages: [");
+  lines.push('    { loc: "/", priority: 1.0, changefreq: "daily" },');
+  lines.push('    { loc: "/about", priority: 0.8 },');
+  lines.push("  ],");
+  lines.push("  dynamicPages: async () => fetchPagesFromAPI(),");
+  lines.push("});");
+  lines.push("```");
+  lines.push("");
+  lines.push("### robots.ts");
+  lines.push("```typescript");
+  lines.push("import type { MetadataRoute } from 'next';");
+  lines.push("");
+  lines.push("export default function robots(): MetadataRoute.Robots {");
+  lines.push("  return {");
+  lines.push('    rules: { userAgent: "*", allow: "/" },');
+  lines.push("    sitemap: `${siteUrl}/sitemap.xml`,");
+  lines.push("  };");
+  lines.push("}");
+  lines.push("```");
+  lines.push("");
+  lines.push("### Cloudflare Override");
+  lines.push("");
+  lines.push('If robots.txt shows "Content-Signal" or "ai-train" \u2014 Cloudflare is overriding your file.');
+  lines.push('**Fix:** Dashboard \u2192 Security \u2192 Settings \u2192 "Manage your robots.txt" \u2192 Set to "Off"');
+  lines.push("");
+  lines.push("### Declarative Routes with SEO");
+  lines.push("");
+  lines.push("Create `app/_routes/` with SEO metadata for sitemap:");
+  lines.push("```typescript");
+  lines.push("import { defineRoute } from '@djangocfg/nextjs/navigation';");
+  lines.push("");
+  lines.push("export const home = defineRoute('/', {");
+  lines.push("  label: 'Home',");
+  lines.push("  protected: false,");
+  lines.push("  priority: 1.0,        // Sitemap priority 0.0-1.0");
+  lines.push("  changefreq: 'daily',  // always|hourly|daily|weekly|monthly|yearly|never");
+  lines.push("  noindex: false,       // Exclude from sitemap");
+  lines.push("});");
+  lines.push("");
+  lines.push("export const staticRoutes = [home, about, contact];");
+  lines.push("```");
+  lines.push("");
+  lines.push("Then in `sitemap.xml/route.ts`:");
+  lines.push("```typescript");
+  lines.push("import { routes } from '@/app/_routes';");
+  lines.push("routes.getAllStaticRoutes().filter(r => !r.metadata.noindex)");
+  lines.push("```");
+  lines.push("");
   lines.push("## Link Guidelines");
   lines.push("");
   lines.push("### Nextra/MDX Projects (content/)");
@@ -2414,28 +2489,29 @@ function createRouteInfo(path6, filePath, type) {
 function compareWithSitemap(scanResult, sitemapUrls, baseUrl) {
   const appRoutes = scanResult.routes.filter((r) => r.type === "page");
   const sitemapPaths = new Set(
-    sitemapUrls.map((url) => {
+    sitemapUrls.filter((url) => url).map((url) => {
       try {
         return new URL(url).pathname;
       } catch {
         return url;
       }
-    })
+    }).filter(Boolean)
   );
   const missingFromSitemap = [];
   const matching = [];
   for (const route of scanResult.staticRoutes) {
-    const path6 = route.path === "/" ? "/" : route.path;
+    const path6 = route.path || "/";
     if (sitemapPaths.has(path6) || sitemapPaths.has(path6 + "/") || sitemapPaths.has(path6.replace(/\/$/, ""))) {
       matching.push(route);
     } else {
       missingFromSitemap.push(route);
     }
   }
-  const staticPaths = new Set(scanResult.staticRoutes.map((r) => r.path));
-  const dynamicPatterns = scanResult.dynamicRoutes.map((r) => routeToRegex(r.path));
+  const staticPaths = new Set(scanResult.staticRoutes.map((r) => r.path).filter(Boolean));
+  const dynamicPatterns = scanResult.dynamicRoutes.filter((r) => r.path).map((r) => routeToRegex(r.path));
   const extraInSitemap = [];
   for (const path6 of sitemapPaths) {
+    if (!path6) continue;
     if (staticPaths.has(path6) || staticPaths.has(path6 + "/") || staticPaths.has(path6.replace(/\/$/, ""))) {
       continue;
     }
@@ -2452,6 +2528,7 @@ function compareWithSitemap(scanResult, sitemapUrls, baseUrl) {
   };
 }
 function routeToRegex(routePath) {
+  if (!routePath) return /^$/;
   let pattern = routePath.replace(/[.+?^${}()|\\]/g, "\\$&").replace(/\[\[\.\.\.([^\]]+)\]\]/g, "(?:/.*)?").replace(/\[\.\.\.([^\]]+)\]/g, "/.+").replace(/\[([^\]]+)\]/g, "/[^/]+");
   pattern = `^${pattern}/?$`;
   return new RegExp(pattern);