@djangocfg/seo 2.1.50

package/src/crawler/crawler.ts

@@ -0,0 +1,425 @@
+/**
+ * @djangocfg/seo - Site Crawler
+ * Internal site crawler for SEO analysis
+ */
+
+import { load } from 'cheerio';
+import pLimit from 'p-limit';
+import consola from 'consola';
+import type { CrawlResult, CrawlerConfig, SeoIssue } from '../types/index.js';
+
+const DEFAULT_CONFIG: Required<CrawlerConfig> = {
+  maxPages: 100,
+  maxDepth: 3,
+  concurrency: 5,
+  timeout: 30000,
+  userAgent: 'DjangoCFG-SEO-Crawler/1.0 (+https://djangocfg.com/bot)',
+  respectRobotsTxt: true,
+  includePatterns: [],
+  excludePatterns: [
+    '/api/',
+    '/admin/',
+    '/_next/',
+    '/static/',
+    '.pdf',
+    '.jpg',
+    '.png',
+    '.gif',
+    '.svg',
+    '.css',
+    '.js',
+  ],
+};
+
+export class SiteCrawler {
+  private config: Required<CrawlerConfig>;
+  private baseUrl: URL;
+  private visited = new Set<string>();
+  private queue: Array<{ url: string; depth: number }> = [];
+  private results: CrawlResult[] = [];
+  private limit: ReturnType<typeof pLimit>;
+
+  constructor(siteUrl: string, config?: CrawlerConfig) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+    this.baseUrl = new URL(siteUrl);
+    this.limit = pLimit(this.config.concurrency);
+  }
+
+  /**
+   * Start crawling the site
+   */
+  async crawl(): Promise<CrawlResult[]> {
+    consola.info(`Starting crawl of ${this.baseUrl.origin}`);
+    consola.info(`Config: maxPages=${this.config.maxPages}, maxDepth=${this.config.maxDepth}`);
+
+    this.queue.push({ url: this.baseUrl.href, depth: 0 });
+
+    while (this.queue.length > 0 && this.results.length < this.config.maxPages) {
+      const batch = this.queue.splice(0, this.config.concurrency);
+
+      const promises = batch.map(({ url, depth }) =>
+        this.limit(() => this.crawlPage(url, depth))
+      );
+
+      await Promise.all(promises);
+    }
+
+    consola.success(`Crawl complete. Crawled ${this.results.length} pages.`);
+    return this.results;
+  }
+
+  /**
+   * Crawl a single page
+   */
+  private async crawlPage(url: string, depth: number): Promise<void> {
+    const normalizedUrl = this.normalizeUrl(url);
+
+    if (this.visited.has(normalizedUrl)) return;
+    if (this.shouldExclude(normalizedUrl)) return;
+
+    this.visited.add(normalizedUrl);
+
+    const startTime = Date.now();
+    const result: CrawlResult = {
+      url: normalizedUrl,
+      statusCode: 0,
+      links: { internal: [], external: [] },
+      images: [],
+      loadTime: 0,
+      errors: [],
+      warnings: [],
+      crawledAt: new Date().toISOString(),
+    };
+
+    try {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), this.config.timeout);
+
+      const response = await fetch(normalizedUrl, {
+        headers: {
+          'User-Agent': this.config.userAgent,
+          Accept: 'text/html,application/xhtml+xml',
+        },
+        signal: controller.signal,
+        redirect: 'follow',
+      });
+
+      // TTFB = time from request start to first response headers
+      result.ttfb = Date.now() - startTime;
+
+      clearTimeout(timeoutId);
+
+      result.statusCode = response.status;
+      result.contentType = response.headers.get('content-type') || undefined;
+      result.contentLength = Number(response.headers.get('content-length')) || undefined;
+
+      if (response.ok && result.contentType?.includes('text/html')) {
+        const html = await response.text();
+        this.parseHtml(html, result, normalizedUrl, depth);
+      } else if (!response.ok) {
+        result.errors.push(`HTTP ${response.status}: ${response.statusText}`);
+      }
+    } catch (error) {
+      if (error instanceof Error) {
+        if (error.name === 'AbortError') {
+          result.errors.push('Request timeout');
+        } else {
+          result.errors.push(error.message);
+        }
+      }
+    }
+
+    result.loadTime = Date.now() - startTime;
+    this.results.push(result);
+
+    consola.debug(`Crawled: ${normalizedUrl} (${result.statusCode}) - ${result.loadTime}ms`);
+  }
+
+  /**
+   * Parse HTML and extract SEO-relevant data
+   */
+  private parseHtml(html: string, result: CrawlResult, pageUrl: string, depth: number): void {
+    const $ = load(html);
+
+    // Title
+    result.title = $('title').first().text().trim() || undefined;
+    if (!result.title) {
+      result.warnings.push('Missing title tag');
+    } else if (result.title.length > 60) {
+      result.warnings.push(`Title too long (${result.title.length} chars, recommended: <60)`);
+    }
+
+    // Meta description
+    result.metaDescription =
+      $('meta[name="description"]').attr('content')?.trim() || undefined;
+    if (!result.metaDescription) {
+      result.warnings.push('Missing meta description');
+    } else if (result.metaDescription.length > 160) {
+      result.warnings.push(
+        `Meta description too long (${result.metaDescription.length} chars, recommended: <160)`
+      );
+    }
+
+    // Meta robots
+    result.metaRobots = $('meta[name="robots"]').attr('content')?.trim() || undefined;
+    const xRobots = $('meta[http-equiv="X-Robots-Tag"]').attr('content')?.trim();
+    if (xRobots) {
+      result.metaRobots = result.metaRobots ? `${result.metaRobots}, ${xRobots}` : xRobots;
+    }
+
+    // Canonical
+    result.canonicalUrl = $('link[rel="canonical"]').attr('href')?.trim() || undefined;
+    if (!result.canonicalUrl) {
+      result.warnings.push('Missing canonical tag');
+    }
+
+    // Headings
+    result.h1 = $('h1')
+      .map((_, el) => $(el).text().trim())
+      .get();
+    result.h2 = $('h2')
+      .map((_, el) => $(el).text().trim())
+      .get();
+
+    if (result.h1.length === 0) {
+      result.warnings.push('Missing H1 tag');
+    } else if (result.h1.length > 1) {
+      result.warnings.push(`Multiple H1 tags (${result.h1.length})`);
+    }
+
+    // Links
+    $('a[href]').each((_, el) => {
+      const href = $(el).attr('href');
+      if (!href) return;
+
+      try {
+        const linkUrl = new URL(href, pageUrl);
+
+        if (linkUrl.hostname === this.baseUrl.hostname) {
+          const internalUrl = this.normalizeUrl(linkUrl.href);
+          result.links.internal.push(internalUrl);
+
+          // Add to crawl queue
+          if (depth < this.config.maxDepth && !this.visited.has(internalUrl)) {
+            this.queue.push({ url: internalUrl, depth: depth + 1 });
+          }
+        } else {
+          result.links.external.push(linkUrl.href);
+        }
+      } catch {
+        // Invalid URL, skip
+      }
+    });
+
+    // Images
+    $('img').each((_, el) => {
+      const src = $(el).attr('src');
+      const alt = $(el).attr('alt');
+
+      if (src) {
+        result.images.push({
+          src,
+          alt,
+          hasAlt: alt !== undefined && alt.trim().length > 0,
+        });
+      }
+    });
+
+    const imagesWithoutAlt = result.images.filter((img) => !img.hasAlt);
+    if (imagesWithoutAlt.length > 0) {
+      result.warnings.push(`${imagesWithoutAlt.length} images without alt text`);
+    }
+  }
+
+  /**
+   * Normalize URL for deduplication
+   */
+  private normalizeUrl(url: string): string {
+    try {
+      const parsed = new URL(url, this.baseUrl.href);
+      // Remove trailing slash, hash, and sort query params
+      parsed.hash = '';
+      let pathname = parsed.pathname;
+      if (pathname.endsWith('/') && pathname !== '/') {
+        pathname = pathname.slice(0, -1);
+      }
+      parsed.pathname = pathname;
+      return parsed.href;
+    } catch {
+      return url;
+    }
+  }
+
+  /**
+   * Check if URL should be excluded
+   */
+  private shouldExclude(url: string): boolean {
+    // Check include patterns first
+    if (this.config.includePatterns.length > 0) {
+      const included = this.config.includePatterns.some((pattern) =>
+        url.includes(pattern)
+      );
+      if (!included) return true;
+    }
+
+    // Check exclude patterns
+    return this.config.excludePatterns.some((pattern) => url.includes(pattern));
+  }
+}
+
+/**
+ * Analyze crawl results for SEO issues
+ */
+export function analyzeCrawlResults(results: CrawlResult[]): SeoIssue[] {
+  const issues: SeoIssue[] = [];
+
+  for (const result of results) {
+    // HTTP errors
+    if (result.statusCode >= 400) {
+      issues.push({
+        id: `http-error-${hash(result.url)}`,
+        url: result.url,
+        category: 'technical',
+        severity: result.statusCode >= 500 ? 'critical' : 'error',
+        title: `HTTP ${result.statusCode} error`,
+        description: `Page returns ${result.statusCode} status code.`,
+        recommendation:
+          result.statusCode === 404
+            ? 'Either restore the content or set up a redirect.'
+            : 'Fix the server error and ensure the page is accessible.',
+        detectedAt: result.crawledAt,
+        metadata: { statusCode: result.statusCode },
+      });
+    }
+
+    // Missing title
+    if (!result.title && result.statusCode === 200) {
+      issues.push({
+        id: `missing-title-${hash(result.url)}`,
+        url: result.url,
+        category: 'content',
+        severity: 'error',
+        title: 'Missing title tag',
+        description: 'This page does not have a title tag.',
+        recommendation: 'Add a unique, descriptive title tag (50-60 characters).',
+        detectedAt: result.crawledAt,
+      });
+    }
+
+    // Missing meta description
+    if (!result.metaDescription && result.statusCode === 200) {
+      issues.push({
+        id: `missing-meta-desc-${hash(result.url)}`,
+        url: result.url,
+        category: 'content',
+        severity: 'warning',
+        title: 'Missing meta description',
+        description: 'This page does not have a meta description.',
+        recommendation: 'Add a unique meta description (120-160 characters).',
+        detectedAt: result.crawledAt,
+      });
+    }
+
+    // Missing H1
+    if (result.h1 && result.h1.length === 0 && result.statusCode === 200) {
+      issues.push({
+        id: `missing-h1-${hash(result.url)}`,
+        url: result.url,
+        category: 'content',
+        severity: 'warning',
+        title: 'Missing H1 heading',
+        description: 'This page does not have an H1 heading.',
+        recommendation: 'Add a single H1 heading that describes the page content.',
+        detectedAt: result.crawledAt,
+      });
+    }
+
+    // Multiple H1s
+    if (result.h1 && result.h1.length > 1) {
+      issues.push({
+        id: `multiple-h1-${hash(result.url)}`,
+        url: result.url,
+        category: 'content',
+        severity: 'warning',
+        title: 'Multiple H1 headings',
+        description: `This page has ${result.h1.length} H1 headings.`,
+        recommendation: 'Use only one H1 heading per page.',
+        detectedAt: result.crawledAt,
+        metadata: { h1Count: result.h1.length },
+      });
+    }
+
+    // Images without alt
+    const imagesWithoutAlt = result.images.filter((img) => !img.hasAlt);
+    if (imagesWithoutAlt.length > 0) {
+      issues.push({
+        id: `images-no-alt-${hash(result.url)}`,
+        url: result.url,
+        category: 'content',
+        severity: 'info',
+        title: 'Images without alt text',
+        description: `${imagesWithoutAlt.length} images are missing alt text.`,
+        recommendation: 'Add descriptive alt text to all images for accessibility and SEO.',
+        detectedAt: result.crawledAt,
+        metadata: { count: imagesWithoutAlt.length },
+      });
+    }
+
+    // Slow load time (> 3s)
+    if (result.loadTime > 3000) {
+      issues.push({
+        id: `slow-page-${hash(result.url)}`,
+        url: result.url,
+        category: 'performance',
+        severity: result.loadTime > 5000 ? 'error' : 'warning',
+        title: 'Slow page load time',
+        description: `Page took ${result.loadTime}ms to load.`,
+        recommendation: 'Optimize page load time. Target under 3 seconds.',
+        detectedAt: result.crawledAt,
+        metadata: { loadTime: result.loadTime },
+      });
+    }
+
+    // Slow TTFB (> 800ms)
+    if (result.ttfb && result.ttfb > 800) {
+      issues.push({
+        id: `slow-ttfb-${hash(result.url)}`,
+        url: result.url,
+        category: 'performance',
+        severity: result.ttfb > 1500 ? 'error' : 'warning',
+        title: 'Slow Time to First Byte',
+        description: `TTFB is ${result.ttfb}ms. Server responded slowly.`,
+        recommendation: 'Optimize server response. Target TTFB under 800ms. Consider CDN, caching, or server upgrades.',
+        detectedAt: result.crawledAt,
+        metadata: { ttfb: result.ttfb },
+      });
+    }
+
+    // Noindex check
+    if (result.metaRobots?.includes('noindex')) {
+      issues.push({
+        id: `noindex-${hash(result.url)}`,
+        url: result.url,
+        category: 'indexing',
+        severity: 'info',
+        title: 'Page marked as noindex',
+        description: 'This page has a noindex directive.',
+        recommendation: 'Verify this is intentional. Remove noindex if the page should be indexed.',
+        detectedAt: result.crawledAt,
+        metadata: { metaRobots: result.metaRobots },
+      });
+    }
+  }
+
+  return issues;
+}
+
+function hash(str: string): string {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i);
+    hash = (hash << 5) - hash + char;
+    hash = hash & hash;
+  }
+  return Math.abs(hash).toString(36);
+}

package/src/crawler/index.ts

@@ -0,0 +1,10 @@
+/**
+ * @djangocfg/seo - Crawler Module
+ * Site crawler and analysis tools
+ */
+
+export { SiteCrawler, analyzeCrawlResults } from './crawler.js';
+export { analyzeRobotsTxt, isUrlAllowed } from './robots-parser.js';
+export { analyzeSitemap, analyzeAllSitemaps } from './sitemap-validator.js';
+export type { RobotsAnalysis } from './robots-parser.js';
+export type { SitemapAnalysis } from './sitemap-validator.js';

package/src/crawler/robots-parser.ts

@@ -0,0 +1,171 @@
+/**
+ * @djangocfg/seo - Robots.txt Parser
+ * Parse and analyze robots.txt files
+ */
+
+import robotsParser from 'robots-parser';
+import consola from 'consola';
+import type { SeoIssue } from '../types/index.js';
+
+export interface RobotsAnalysis {
+  exists: boolean;
+  content?: string;
+  sitemaps: string[];
+  allowedPaths: string[];
+  disallowedPaths: string[];
+  crawlDelay?: number;
+  issues: SeoIssue[];
+}
+
+/**
+ * Fetch and parse robots.txt for a site
+ */
+export async function analyzeRobotsTxt(siteUrl: string): Promise<RobotsAnalysis> {
+  const robotsUrl = new URL('/robots.txt', siteUrl).href;
+
+  const analysis: RobotsAnalysis = {
+    exists: false,
+    sitemaps: [],
+    allowedPaths: [],
+    disallowedPaths: [],
+    issues: [],
+  };
+
+  try {
+    const response = await fetch(robotsUrl);
+
+    if (!response.ok) {
+      analysis.issues.push({
+        id: 'missing-robots-txt',
+        url: robotsUrl,
+        category: 'technical',
+        severity: 'warning',
+        title: 'Missing robots.txt',
+        description: `No robots.txt file found (HTTP ${response.status}).`,
+        recommendation: 'Create a robots.txt file to control crawler access.',
+        detectedAt: new Date().toISOString(),
+      });
+      return analysis;
+    }
+
+    analysis.exists = true;
+    analysis.content = await response.text();
+
+    // Parse robots.txt
+    const robots = robotsParser(robotsUrl, analysis.content);
+
+    // Extract sitemaps
+    analysis.sitemaps = robots.getSitemaps();
+
+    if (analysis.sitemaps.length === 0) {
+      analysis.issues.push({
+        id: 'no-sitemap-in-robots',
+        url: robotsUrl,
+        category: 'technical',
+        severity: 'info',
+        title: 'No sitemap in robots.txt',
+        description: 'No sitemap URL is declared in robots.txt.',
+        recommendation: 'Add a Sitemap directive pointing to your XML sitemap.',
+        detectedAt: new Date().toISOString(),
+      });
+    }
+
+    // Parse rules (simplified extraction)
+    const lines = analysis.content.split('\n');
+    let currentUserAgent = '*';
+
+    for (const line of lines) {
+      const trimmed = line.trim().toLowerCase();
+
+      if (trimmed.startsWith('user-agent:')) {
+        currentUserAgent = trimmed.replace('user-agent:', '').trim();
+      } else if (trimmed.startsWith('disallow:')) {
+        const path = line.trim().replace(/disallow:/i, '').trim();
+        if (path) {
+          analysis.disallowedPaths.push(path);
+        }
+      } else if (trimmed.startsWith('allow:')) {
+        const path = line.trim().replace(/allow:/i, '').trim();
+        if (path) {
+          analysis.allowedPaths.push(path);
+        }
+      } else if (trimmed.startsWith('crawl-delay:')) {
+        const delay = parseInt(trimmed.replace('crawl-delay:', '').trim(), 10);
+        if (!isNaN(delay)) {
+          analysis.crawlDelay = delay;
+        }
+      }
+    }
+
+    // Check for blocking important paths
+    const importantPaths = ['/', '/sitemap.xml'];
+    for (const path of importantPaths) {
+      if (!robots.isAllowed(new URL(path, siteUrl).href, 'Googlebot')) {
+        analysis.issues.push({
+          id: `blocked-important-path-${path.replace(/\//g, '-')}`,
+          url: siteUrl,
+          category: 'crawling',
+          severity: 'error',
+          title: `Important path blocked: ${path}`,
+          description: `The path ${path} is blocked in robots.txt.`,
+          recommendation: `Ensure ${path} is accessible to search engines.`,
+          detectedAt: new Date().toISOString(),
+          metadata: { path },
+        });
+      }
+    }
+
+    // Check for excessively restrictive rules
+    if (analysis.disallowedPaths.includes('/')) {
+      analysis.issues.push({
+        id: 'all-blocked',
+        url: robotsUrl,
+        category: 'crawling',
+        severity: 'critical',
+        title: 'Entire site blocked',
+        description: 'robots.txt blocks access to the entire site (Disallow: /).',
+        recommendation: 'Remove or modify this rule if you want your site to be indexed.',
+        detectedAt: new Date().toISOString(),
+      });
+    }
+
+    consola.debug(`Analyzed robots.txt: ${analysis.disallowedPaths.length} disallow rules`);
+  } catch (error) {
+    consola.error('Failed to fetch robots.txt:', error);
+    analysis.issues.push({
+      id: 'robots-txt-error',
+      url: robotsUrl,
+      category: 'technical',
+      severity: 'warning',
+      title: 'Failed to fetch robots.txt',
+      description: `Error fetching robots.txt: ${error instanceof Error ? error.message : 'Unknown error'}`,
+      recommendation: 'Ensure robots.txt is accessible.',
+      detectedAt: new Date().toISOString(),
+    });
+  }
+
+  return analysis;
+}
+
+/**
+ * Check if a URL is allowed by robots.txt
+ */
+export async function isUrlAllowed(
+  siteUrl: string,
+  url: string,
+  userAgent = 'Googlebot'
+): Promise<boolean> {
+  const robotsUrl = new URL('/robots.txt', siteUrl).href;
+
+  try {
+    const response = await fetch(robotsUrl);
+    if (!response.ok) return true; // No robots.txt = allow all
+
+    const content = await response.text();
+    const robots = robotsParser(robotsUrl, content);
+
+    return robots.isAllowed(url, userAgent) ?? true;
+  } catch {
+    return true; // Error fetching = allow
+  }
+}