@djangocfg/seo 2.1.50

package/src/routes/analyzer.ts

@@ -0,0 +1,324 @@
+/**
+ * @djangocfg/seo - Routes Analyzer
+ * Compare routes with sitemap and verify accessibility
+ */
+
+import pLimit from 'p-limit';
+import type { ScanResult, RouteInfo } from './scanner.js';
+import type { SeoIssue } from '../types/index.js';
+
+export interface SitemapUrl {
+  loc: string;
+  lastmod?: string;
+  changefreq?: string;
+  priority?: string;
+}
+
+export interface RouteComparisonResult {
+  /** Routes found in app/ */
+  appRoutes: RouteInfo[];
+  /** URLs found in sitemap */
+  sitemapUrls: string[];
+  /** Static routes missing from sitemap */
+  missingFromSitemap: RouteInfo[];
+  /** Sitemap URLs that don't match any route */
+  extraInSitemap: string[];
+  /** Matching routes */
+  matching: RouteInfo[];
+}
+
+export interface RouteVerificationResult {
+  /** Route being verified */
+  route: RouteInfo;
+  /** Full URL */
+  url: string;
+  /** HTTP status code */
+  statusCode: number;
+  /** Is accessible (2xx) */
+  isAccessible: boolean;
+  /** Error message if failed */
+  error?: string;
+  /** Response time in ms */
+  responseTime: number;
+}
+
+export interface VerifyOptions {
+  /** Base URL (e.g., http://localhost:3000) */
+  baseUrl: string;
+  /** Request timeout in ms */
+  timeout?: number;
+  /** Max concurrent requests */
+  concurrency?: number;
+  /** Only verify static routes */
+  staticOnly?: boolean;
+}
+
+/**
+ * Compare routes with sitemap URLs
+ */
+export function compareWithSitemap(
+  scanResult: ScanResult,
+  sitemapUrls: string[],
+  baseUrl: string
+): RouteComparisonResult {
+  const appRoutes = scanResult.routes.filter(r => r.type === 'page');
+
+  // Normalize sitemap URLs to paths
+  const sitemapPaths = new Set(
+    sitemapUrls.map(url => {
+      try {
+        return new URL(url).pathname;
+      } catch {
+        return url;
+      }
+    })
+  );
+
+  // Find static routes missing from sitemap
+  const missingFromSitemap: RouteInfo[] = [];
+  const matching: RouteInfo[] = [];
+
+  for (const route of scanResult.staticRoutes) {
+    const path = route.path === '/' ? '/' : route.path;
+    // Check with and without trailing slash
+    if (sitemapPaths.has(path) || sitemapPaths.has(path + '/') || sitemapPaths.has(path.replace(/\/$/, ''))) {
+      matching.push(route);
+    } else {
+      missingFromSitemap.push(route);
+    }
+  }
+
+  // Find sitemap URLs that don't match any static route
+  const staticPaths = new Set(scanResult.staticRoutes.map(r => r.path));
+  const dynamicPatterns = scanResult.dynamicRoutes.map(r => routeToRegex(r.path));
+
+  const extraInSitemap: string[] = [];
+  for (const path of sitemapPaths) {
+    // Skip if matches static route
+    if (staticPaths.has(path) || staticPaths.has(path + '/') || staticPaths.has(path.replace(/\/$/, ''))) {
+      continue;
+    }
+
+    // Skip if matches dynamic route pattern
+    const matchesDynamic = dynamicPatterns.some(regex => regex.test(path));
+    if (matchesDynamic) continue;
+
+    extraInSitemap.push(path);
+  }
+
+  return {
+    appRoutes,
+    sitemapUrls,
+    missingFromSitemap,
+    extraInSitemap,
+    matching,
+  };
+}
+
+/**
+ * Convert route path to regex for matching
+ */
+function routeToRegex(routePath: string): RegExp {
+  let pattern = routePath
+    // Escape special regex chars except [ ]
+    .replace(/[.+?^${}()|\\]/g, '\\$&')
+    // Optional catch-all: [[...slug]] matches zero or more segments
+    .replace(/\[\[\.\.\.([^\]]+)\]\]/g, '(?:/.*)?')
+    // Catch-all: [...slug] matches one or more segments
+    .replace(/\[\.\.\.([^\]]+)\]/g, '/.+')
+    // Dynamic: [slug] matches one segment
+    .replace(/\[([^\]]+)\]/g, '/[^/]+');
+
+  // Handle trailing slash optionally
+  pattern = `^${pattern}/?$`;
+
+  return new RegExp(pattern);
+}
+
+/**
+ * Verify routes are accessible
+ */
+export async function verifyRoutes(
+  scanResult: ScanResult,
+  options: VerifyOptions
+): Promise<RouteVerificationResult[]> {
+  const {
+    baseUrl,
+    timeout = 10000,
+    concurrency = 5,
+    staticOnly = true,
+  } = options;
+
+  const routes = staticOnly ? scanResult.staticRoutes : scanResult.routes.filter(r => r.type === 'page');
+  const limit = pLimit(concurrency);
+
+  const results = await Promise.all(
+    routes.map(route =>
+      limit(async () => {
+        const url = new URL(route.path, baseUrl).href;
+        const startTime = Date.now();
+
+        try {
+          const controller = new AbortController();
+          const timeoutId = setTimeout(() => controller.abort(), timeout);
+
+          const response = await fetch(url, {
+            method: 'HEAD',
+            signal: controller.signal,
+            redirect: 'follow',
+          });
+
+          clearTimeout(timeoutId);
+
+          return {
+            route,
+            url,
+            statusCode: response.status,
+            isAccessible: response.status >= 200 && response.status < 400,
+            responseTime: Date.now() - startTime,
+          };
+        } catch (error) {
+          return {
+            route,
+            url,
+            statusCode: 0,
+            isAccessible: false,
+            error: (error as Error).message,
+            responseTime: Date.now() - startTime,
+          };
+        }
+      })
+    )
+  );
+
+  return results;
+}
+
+/**
+ * Analyze routes and generate SEO issues
+ */
+export function analyzeRoutes(
+  scanResult: ScanResult,
+  comparison?: RouteComparisonResult,
+  verification?: RouteVerificationResult[]
+): SeoIssue[] {
+  const issues: SeoIssue[] = [];
+  const now = new Date().toISOString();
+
+  // Issue: Static routes missing from sitemap
+  if (comparison && comparison.missingFromSitemap.length > 0) {
+    for (const route of comparison.missingFromSitemap) {
+      issues.push({
+        id: `route-missing-sitemap-${route.path}`,
+        category: 'indexing',
+        severity: 'warning',
+        title: 'Route missing from sitemap',
+        description: `Static route ${route.path} is not in sitemap.xml`,
+        url: route.path,
+        recommendation: 'Add this route to your sitemap for better indexing',
+        detectedAt: now,
+      });
+    }
+  }
+
+  // Issue: Sitemap URLs that don't match routes (potential dead links)
+  if (comparison && comparison.extraInSitemap.length > 0) {
+    for (const path of comparison.extraInSitemap.slice(0, 20)) {
+      issues.push({
+        id: `sitemap-orphan-${path}`,
+        category: 'indexing',
+        severity: 'info',
+        title: 'Sitemap URL without matching route',
+        description: `URL ${path} in sitemap doesn't match any app/ route`,
+        url: path,
+        recommendation: 'Verify this URL is still valid or remove from sitemap',
+        detectedAt: now,
+      });
+    }
+  }
+
+  // Issue: Inaccessible routes
+  if (verification) {
+    for (const result of verification) {
+      if (!result.isAccessible) {
+        issues.push({
+          id: `route-inaccessible-${result.route.path}`,
+          category: 'technical',
+          severity: result.statusCode === 404 ? 'error' : 'warning',
+          title: `Route returns ${result.statusCode || 'error'}`,
+          description: result.error || `Route ${result.route.path} returned status ${result.statusCode}`,
+          url: result.url,
+          recommendation: 'Fix the route or remove it from app/',
+          detectedAt: now,
+        });
+      }
+    }
+  }
+
+  // Issue: Too many dynamic routes (SEO concern)
+  if (scanResult.dynamicRoutes.length > scanResult.staticRoutes.length * 2) {
+    issues.push({
+      id: 'too-many-dynamic-routes',
+      category: 'content',
+      severity: 'info',
+      title: 'High ratio of dynamic routes',
+      description: `${scanResult.dynamicRoutes.length} dynamic vs ${scanResult.staticRoutes.length} static routes`,
+      url: '/',
+      recommendation: 'Ensure dynamic routes have proper generateStaticParams for SSG',
+      detectedAt: now,
+    });
+  }
+
+  return issues;
+}
+
+/**
+ * Generate routes summary
+ */
+export function generateRoutesSummary(
+  scanResult: ScanResult,
+  comparison?: RouteComparisonResult,
+  verification?: RouteVerificationResult[]
+): string {
+  const lines: string[] = [];
+
+  lines.push('## Routes Summary');
+  lines.push('');
+  lines.push(`Total routes: ${scanResult.routes.length}`);
+  lines.push(`├── Static: ${scanResult.staticRoutes.length}`);
+  lines.push(`├── Dynamic: ${scanResult.dynamicRoutes.length}`);
+  lines.push(`└── API: ${scanResult.apiRoutes.length}`);
+  lines.push('');
+
+  if (comparison) {
+    lines.push('### Sitemap Comparison');
+    lines.push('');
+    lines.push(`Sitemap URLs: ${comparison.sitemapUrls.length}`);
+    lines.push(`├── Matching: ${comparison.matching.length}`);
+    lines.push(`├── Missing from sitemap: ${comparison.missingFromSitemap.length}`);
+    lines.push(`└── Extra in sitemap: ${comparison.extraInSitemap.length}`);
+    lines.push('');
+  }
+
+  if (verification) {
+    const accessible = verification.filter(r => r.isAccessible).length;
+    const broken = verification.filter(r => !r.isAccessible);
+
+    lines.push('### Route Verification');
+    lines.push('');
+    lines.push(`Accessible: ${accessible}/${verification.length}`);
+
+    if (broken.length > 0) {
+      lines.push('');
+      lines.push('Broken routes:');
+      for (const r of broken.slice(0, 10)) {
+        lines.push(`  - ${r.route.path} → ${r.statusCode || r.error}`);
+      }
+      if (broken.length > 10) {
+        lines.push(`  - ... +${broken.length - 10} more`);
+      }
+    }
+  }
+
+  return lines.join('\n');
+}

package/src/routes/index.ts

@@ -0,0 +1,25 @@
+/**
+ * @djangocfg/seo - Routes Module
+ * Next.js App Router route scanning and analysis
+ */
+
+export {
+  scanRoutes,
+  findAppDir,
+  routeToUrl,
+  getStaticUrls,
+  type RouteInfo,
+  type ScanResult,
+  type ScanOptions,
+} from './scanner.js';
+
+export {
+  compareWithSitemap,
+  verifyRoutes,
+  analyzeRoutes,
+  generateRoutesSummary,
+  type RouteComparisonResult,
+  type RouteVerificationResult,
+  type VerifyOptions,
+  type SitemapUrl,
+} from './analyzer.js';

package/src/routes/scanner.ts

@@ -0,0 +1,298 @@
+/**
+ * @djangocfg/seo - Next.js App Router Scanner
+ * Scans app/ directory to extract all routes
+ */
+
+import { readdirSync, statSync, existsSync } from 'node:fs';
+import { join, relative } from 'node:path';
+
+export interface RouteInfo {
+  /** Route path (e.g., /blog/[slug]) */
+  path: string;
+  /** File path relative to app/ */
+  filePath: string;
+  /** Route type */
+  type: 'page' | 'api' | 'layout' | 'loading' | 'error';
+  /** Is dynamic route */
+  isDynamic: boolean;
+  /** Dynamic segments (e.g., ['slug', '...path']) */
+  dynamicSegments: string[];
+  /** Is catch-all route */
+  isCatchAll: boolean;
+  /** Is optional catch-all route */
+  isOptionalCatchAll: boolean;
+  /** Route group (if any) */
+  routeGroup?: string;
+}
+
+export interface ScanResult {
+  /** All discovered routes */
+  routes: RouteInfo[];
+  /** Static routes (no dynamic segments) */
+  staticRoutes: RouteInfo[];
+  /** Dynamic routes (with [param]) */
+  dynamicRoutes: RouteInfo[];
+  /** API routes */
+  apiRoutes: RouteInfo[];
+  /** App directory path */
+  appDir: string;
+}
+
+export interface ScanOptions {
+  /** Path to app directory */
+  appDir?: string;
+  /** Include API routes */
+  includeApi?: boolean;
+  /** Include layouts, loading, error pages */
+  includeSpecial?: boolean;
+}
+
+const PAGE_FILES = ['page.tsx', 'page.ts', 'page.jsx', 'page.js'];
+const ROUTE_FILES = ['route.tsx', 'route.ts', 'route.jsx', 'route.js'];
+const SPECIAL_FILES = ['layout', 'loading', 'error', 'not-found', 'template'];
+
+/**
+ * Find app directory by searching common locations
+ */
+export function findAppDir(startDir: string = process.cwd()): string | null {
+  const candidates = [
+    join(startDir, 'app'),
+    join(startDir, 'src', 'app'),
+  ];
+
+  for (const dir of candidates) {
+    if (existsSync(dir) && statSync(dir).isDirectory()) {
+      return dir;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Scan Next.js app directory for routes
+ */
+export function scanRoutes(options: ScanOptions = {}): ScanResult {
+  const {
+    appDir = findAppDir() || './app',
+    includeApi = true,
+    includeSpecial = false,
+  } = options;
+
+  if (!existsSync(appDir)) {
+    throw new Error(`App directory not found: ${appDir}`);
+  }
+
+  const routes: RouteInfo[] = [];
+
+  scanDirectory(appDir, '', routes, { includeApi, includeSpecial });
+
+  return {
+    routes,
+    staticRoutes: routes.filter(r => !r.isDynamic && r.type === 'page'),
+    dynamicRoutes: routes.filter(r => r.isDynamic && r.type === 'page'),
+    apiRoutes: routes.filter(r => r.type === 'api'),
+    appDir,
+  };
+}
+
+function scanDirectory(
+  dir: string,
+  routePath: string,
+  routes: RouteInfo[],
+  options: { includeApi: boolean; includeSpecial: boolean }
+): void {
+  let entries: string[];
+
+  try {
+    entries = readdirSync(dir);
+  } catch {
+    return;
+  }
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry);
+
+    let stat;
+    try {
+      stat = statSync(fullPath);
+    } catch {
+      continue;
+    }
+
+    if (stat.isDirectory()) {
+      // Skip private folders
+      if (entry.startsWith('_') || entry.startsWith('.')) continue;
+
+      // Handle route groups (folder)
+      if (entry.startsWith('(') && entry.endsWith(')')) {
+        // Route groups don't add to URL path
+        scanDirectory(fullPath, routePath, routes, options);
+        continue;
+      }
+
+      // Handle parallel routes @folder
+      if (entry.startsWith('@')) {
+        // Parallel routes don't add to URL path
+        scanDirectory(fullPath, routePath, routes, options);
+        continue;
+      }
+
+      // Handle intercepting routes (.)folder, (..)folder, (...)folder
+      if (entry.startsWith('(') && !entry.endsWith(')')) {
+        continue; // Skip intercepting routes for now
+      }
+
+      // Build new route path
+      const segment = processSegment(entry);
+      const newRoutePath = routePath + '/' + segment.urlSegment;
+
+      scanDirectory(fullPath, newRoutePath, routes, options);
+    } else if (stat.isFile()) {
+      // Check for page files
+      if (PAGE_FILES.includes(entry)) {
+        const route = createRouteInfo(routePath || '/', dir, 'page');
+        routes.push(route);
+      }
+
+      // Check for API route files
+      if (options.includeApi && ROUTE_FILES.includes(entry)) {
+        const route = createRouteInfo(routePath || '/', dir, 'api');
+        routes.push(route);
+      }
+
+      // Check for special files
+      if (options.includeSpecial) {
+        const baseName = entry.replace(/\.(tsx?|jsx?|js)$/, '');
+        if (SPECIAL_FILES.includes(baseName)) {
+          const route = createRouteInfo(routePath || '/', dir, baseName as RouteInfo['type']);
+          routes.push(route);
+        }
+      }
+    }
+  }
+}
+
+interface SegmentInfo {
+  urlSegment: string;
+  isDynamic: boolean;
+  paramName?: string;
+  isCatchAll: boolean;
+  isOptionalCatchAll: boolean;
+}
+
+function processSegment(segment: string): SegmentInfo {
+  // Optional catch-all: [[...slug]]
+  if (segment.startsWith('[[...') && segment.endsWith(']]')) {
+    const paramName = segment.slice(5, -2);
+    return {
+      urlSegment: segment,
+      isDynamic: true,
+      paramName,
+      isCatchAll: true,
+      isOptionalCatchAll: true,
+    };
+  }
+
+  // Catch-all: [...slug]
+  if (segment.startsWith('[...') && segment.endsWith(']')) {
+    const paramName = segment.slice(4, -1);
+    return {
+      urlSegment: segment,
+      isDynamic: true,
+      paramName,
+      isCatchAll: true,
+      isOptionalCatchAll: false,
+    };
+  }
+
+  // Dynamic: [slug]
+  if (segment.startsWith('[') && segment.endsWith(']')) {
+    const paramName = segment.slice(1, -1);
+    return {
+      urlSegment: segment,
+      isDynamic: true,
+      paramName,
+      isCatchAll: false,
+      isOptionalCatchAll: false,
+    };
+  }
+
+  // Static segment
+  return {
+    urlSegment: segment,
+    isDynamic: false,
+    isCatchAll: false,
+    isOptionalCatchAll: false,
+  };
+}
+
+function createRouteInfo(path: string, filePath: string, type: RouteInfo['type']): RouteInfo {
+  const segments = path.split('/').filter(Boolean);
+  const dynamicSegments: string[] = [];
+  let isDynamic = false;
+  let isCatchAll = false;
+  let isOptionalCatchAll = false;
+
+  for (const segment of segments) {
+    const info = processSegment(segment);
+    if (info.isDynamic) {
+      isDynamic = true;
+      if (info.paramName) {
+        dynamicSegments.push(info.paramName);
+      }
+      if (info.isCatchAll) isCatchAll = true;
+      if (info.isOptionalCatchAll) isOptionalCatchAll = true;
+    }
+  }
+
+  // Extract route group from path
+  let routeGroup: string | undefined;
+  const groupMatch = filePath.match(/\(([^)]+)\)/);
+  if (groupMatch) {
+    routeGroup = groupMatch[1];
+  }
+
+  return {
+    path: path || '/',
+    filePath,
+    type,
+    isDynamic,
+    dynamicSegments,
+    isCatchAll,
+    isOptionalCatchAll,
+    routeGroup,
+  };
+}
+
+/**
+ * Convert route path to URL (replace dynamic segments with example values)
+ */
+export function routeToUrl(route: RouteInfo, params?: Record<string, string>): string {
+  let url = route.path;
+
+  for (const segment of route.dynamicSegments) {
+    const value = params?.[segment] || `{${segment}}`;
+
+    // Handle different dynamic segment types
+    if (route.isOptionalCatchAll) {
+      url = url.replace(`[[...${segment}]]`, value);
+    } else if (route.isCatchAll) {
+      url = url.replace(`[...${segment}]`, value);
+    } else {
+      url = url.replace(`[${segment}]`, value);
+    }
+  }
+
+  return url;
+}
+
+/**
+ * Get static routes as URLs (ready to check)
+ */
+export function getStaticUrls(scanResult: ScanResult, baseUrl: string): string[] {
+  return scanResult.staticRoutes.map(route => {
+    const url = new URL(route.path, baseUrl);
+    return url.href;
+  });
+}