@djangocfg/seo 2.1.50

package/src/link-checker/index.ts

@@ -0,0 +1,461 @@
+#!/usr/bin/env node
+
+/**
+ * @djangocfg/seo - Link Checker
+ *
+ * Smart link checker using linkinator with proper error handling,
+ * timeout management, and filtering of problematic URLs.
+ *
+ * @example
+ * ```typescript
+ * import { checkLinks } from '@djangocfg/seo/link-checker';
+ *
+ * const result = await checkLinks({
+ *   url: 'https://example.com',
+ *   timeout: 60000,
+ * });
+ * ```
+ *
+ * @example CLI
+ * ```bash
+ * djangocfg-seo links --site https://example.com
+ * # or
+ * djangocfg-seo links # Interactive mode
+ * ```
+ */
+
+import { mkdir, writeFile } from 'node:fs/promises';
+import * as linkinator from 'linkinator';
+import { dirname } from 'node:path';
+import chalk from 'chalk';
+import type { CheckOptions } from 'linkinator';
+import type { SeoIssue } from '../types/index.js';
+
+export interface CheckLinksOptions {
+  /** Base URL to check (defaults to NEXT_PUBLIC_SITE_URL env variable) */
+  url?: string;
+  /** Timeout in milliseconds (default: 60000) */
+  timeout?: number;
+  /** URLs to skip (regex pattern) */
+  skipPattern?: string;
+  /** Show only broken links (default: true) */
+  showOnlyBroken?: boolean;
+  /** Maximum number of concurrent requests (default: 50) */
+  concurrency?: number;
+  /** Output file path for report (optional) */
+  outputFile?: string;
+  /** Report format: 'json', 'markdown', 'text' (default: 'text') */
+  reportFormat?: 'json' | 'markdown' | 'text';
+  /** Verbose logging (default: true) */
+  verbose?: boolean;
+}
+
+const DEFAULT_SKIP_PATTERN = [
+  'github.com',
+  'twitter.com',
+  'linkedin.com',
+  'x.com',
+  '127.0.0.1',
+  'localhost:[0-9]+',
+  'api\\.localhost',
+  'demo\\.localhost',
+  'cdn-cgi', // Cloudflare email protection
+  'mailto:', // Email links
+  'tel:', // Phone links
+  'javascript:', // JavaScript links
+].join('|');
+
+export interface BrokenLink {
+  url: string;
+  status: number | string;
+  reason?: string;
+  isExternal: boolean;
+  sourceUrl?: string;
+}
+
+export interface CheckLinksResult {
+  success: boolean;
+  broken: number;
+  total: number;
+  errors: BrokenLink[];
+  /** Internal broken links (same domain) */
+  internalErrors: BrokenLink[];
+  /** External broken links (different domain) */
+  externalErrors: BrokenLink[];
+  url: string;
+  timestamp: string;
+  duration?: number;
+}
+
+/**
+ * Get site URL from options or environment variable
+ */
+function getSiteUrl(options: CheckLinksOptions): string {
+  if (options.url) {
+    return options.url;
+  }
+
+  // Try environment variables
+  const envUrl =
+    process.env.NEXT_PUBLIC_SITE_URL ||
+    process.env.SITE_URL ||
+    process.env.BASE_URL;
+
+  if (envUrl) {
+    return envUrl;
+  }
+
+  throw new Error(
+    'URL is required. Provide it via options.url or set NEXT_PUBLIC_SITE_URL environment variable.'
+  );
+}
+
+/**
+ * Check if URL is external (different domain)
+ */
+function isExternalUrl(linkUrl: string, baseUrl: string): boolean {
+  try {
+    const link = new URL(linkUrl);
+    const base = new URL(baseUrl);
+    return link.hostname !== base.hostname;
+  } catch {
+    return true;
+  }
+}
+
+/**
+ * Check all links on a website
+ */
+export async function checkLinks(options: CheckLinksOptions): Promise<CheckLinksResult> {
+  const url = getSiteUrl(options);
+  const {
+    timeout = 60000,
+    skipPattern = DEFAULT_SKIP_PATTERN,
+    showOnlyBroken = true,
+    concurrency = 50,
+    outputFile,
+    reportFormat = 'text',
+    verbose = true,
+  } = options;
+
+  const startTime = Date.now();
+
+  if (verbose) {
+    console.log(chalk.cyan(`\n🔍 Starting link check for: ${chalk.bold(url)}`));
+    console.log(chalk.dim(`   Timeout: ${timeout}ms | Concurrency: ${concurrency}`));
+    console.log('');
+  }
+
+  const skipRegex = new RegExp(skipPattern);
+
+  const checkOptions: CheckOptions = {
+    path: url,
+    recurse: true,
+    timeout,
+    concurrency,
+    linksToSkip: (link: string) => {
+      return Promise.resolve(skipRegex.test(link));
+    },
+  };
+
+  const broken: BrokenLink[] = [];
+  const internalErrors: BrokenLink[] = [];
+  const externalErrors: BrokenLink[] = [];
+  let total = 0;
+
+  try {
+    const results = await linkinator.check(checkOptions);
+
+    for (const result of results.links) {
+      total++;
+      const status = result.status || 0;
+      const isExternal = isExternalUrl(result.url, url);
+
+      if (status < 200 || status >= 400 || result.state === 'BROKEN') {
+        const statusValue = status || 'TIMEOUT';
+
+        // Skip TIMEOUT errors on external URLs (rate limiting, slow servers)
+        if (statusValue === 'TIMEOUT' && isExternal) {
+          continue;
+        }
+
+        const brokenLink: BrokenLink = {
+          url: result.url,
+          status: statusValue,
+          reason: result.state === 'BROKEN' ? 'BROKEN' : undefined,
+          isExternal,
+          sourceUrl: result.parent,
+        };
+
+        broken.push(brokenLink);
+
+        if (isExternal) {
+          externalErrors.push(brokenLink);
+        } else {
+          internalErrors.push(brokenLink);
+        }
+      }
+    }
+
+    const success = internalErrors.length === 0;
+
+    if (!showOnlyBroken || broken.length > 0) {
+      if (success && externalErrors.length === 0) {
+        console.log(`✅ All links are valid!`);
+        console.log(`   Checked ${total} links.`);
+      } else {
+        // Show internal errors first (more important)
+        if (internalErrors.length > 0) {
+          console.log(chalk.red(`❌ Found ${internalErrors.length} broken internal links:`));
+          for (const { url: linkUrl, status, reason } of internalErrors.slice(0, 20)) {
+            console.log(`   [${status}] ${linkUrl}${reason ? ` (${reason})` : ''}`);
+          }
+          if (internalErrors.length > 20) {
+            console.log(chalk.dim(`   ... and ${internalErrors.length - 20} more`));
+          }
+        }
+
+        // Show external errors (less critical)
+        if (externalErrors.length > 0) {
+          console.log('');
+          console.log(chalk.yellow(`⚠️  Found ${externalErrors.length} broken external links:`));
+          for (const { url: linkUrl, status } of externalErrors.slice(0, 10)) {
+            console.log(`   [${status}] ${linkUrl}`);
+          }
+          if (externalErrors.length > 10) {
+            console.log(chalk.dim(`   ... and ${externalErrors.length - 10} more`));
+          }
+        }
+      }
+    }
+
+    const duration = Date.now() - startTime;
+    const result: CheckLinksResult = {
+      success,
+      broken: broken.length,
+      total,
+      errors: broken,
+      internalErrors,
+      externalErrors,
+      url,
+      timestamp: new Date().toISOString(),
+      duration,
+    };
+
+    if (outputFile) {
+      await saveReport(result, outputFile, reportFormat);
+      console.log(chalk.green(`\n📄 Report saved to: ${chalk.cyan(outputFile)}`));
+    }
+
+    return result;
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    const errorName = error instanceof Error ? error.name : 'UnknownError';
+
+    if (
+      errorMessage.includes('timeout') ||
+      errorMessage.includes('TimeoutError') ||
+      errorName === 'TimeoutError' ||
+      errorMessage.includes('aborted')
+    ) {
+      console.warn(chalk.yellow(`⚠️  Some links timed out after ${timeout}ms`));
+      console.warn(chalk.dim(`   This is normal for slow or protected URLs.`));
+      if (total > 0) {
+        console.warn(chalk.dim(`   Checked ${total} links before timeout.`));
+      }
+
+      if (broken.length > 0) {
+        console.log(chalk.red(`\n❌ Found ${broken.length} broken links:`));
+        for (const { url, status, reason } of broken) {
+          const statusColor =
+            typeof status === 'number' && status >= 500 ? chalk.red : chalk.yellow;
+          console.log(
+            `   ${statusColor(`[${status}]`)} ${chalk.cyan(url)}${reason ? chalk.dim(` (${reason})`) : ''}`
+          );
+        }
+      }
+    } else {
+      console.error(chalk.red(`❌ Error checking links: ${errorMessage}`));
+    }
+
+    const duration = Date.now() - startTime;
+    const result: CheckLinksResult = {
+      success: internalErrors.length === 0 && total > 0,
+      broken: broken.length,
+      total,
+      errors: broken,
+      internalErrors,
+      externalErrors,
+      url,
+      timestamp: new Date().toISOString(),
+      duration,
+    };
+
+    if (outputFile) {
+      try {
+        await saveReport(result, outputFile, reportFormat);
+        console.log(chalk.green(`\n📄 Report saved to: ${chalk.cyan(outputFile)}`));
+      } catch (saveError) {
+        console.warn(
+          chalk.yellow(
+            `\n⚠️  Failed to save report: ${saveError instanceof Error ? saveError.message : String(saveError)}`
+          )
+        );
+      }
+    }
+
+    return result;
+  }
+}
+
+/**
+ * Convert link check results to SEO issues
+ * Separates internal (critical) from external (warning) issues
+ */
+export function linkResultsToSeoIssues(result: CheckLinksResult): SeoIssue[] {
+  const issues: SeoIssue[] = [];
+
+  // Internal broken links are more critical
+  for (const error of result.internalErrors) {
+    issues.push({
+      id: `broken-internal-link-${hash(error.url)}`,
+      url: error.url,
+      category: 'technical' as const,
+      severity: typeof error.status === 'number' && error.status >= 500 ? 'critical' as const : 'error' as const,
+      title: `Broken internal link: ${error.status}`,
+      description: `Internal link returned ${error.status} status${error.reason ? ` (${error.reason})` : ''}.`,
+      recommendation: 'Fix the internal link. This affects user experience and SEO.',
+      detectedAt: result.timestamp,
+      metadata: {
+        status: error.status,
+        reason: error.reason,
+        sourceUrl: error.sourceUrl || result.url,
+        isExternal: false,
+      },
+    });
+  }
+
+  // External broken links are warnings
+  for (const error of result.externalErrors) {
+    issues.push({
+      id: `broken-external-link-${hash(error.url)}`,
+      url: error.url,
+      category: 'technical' as const,
+      severity: 'warning' as const,
+      title: `Broken external link: ${error.status}`,
+      description: `External link returned ${error.status} status.`,
+      recommendation: 'Consider removing or updating the external link.',
+      detectedAt: result.timestamp,
+      metadata: {
+        status: error.status,
+        reason: error.reason,
+        sourceUrl: error.sourceUrl || result.url,
+        isExternal: true,
+      },
+    });
+  }
+
+  return issues;
+}
+
+async function saveReport(
+  result: CheckLinksResult,
+  filePath: string,
+  format: 'json' | 'markdown' | 'text'
+): Promise<void> {
+  const dir = dirname(filePath);
+  if (dir !== '.') {
+    await mkdir(dir, { recursive: true });
+  }
+
+  let content: string;
+
+  switch (format) {
+    case 'json':
+      content = JSON.stringify(result, null, 2);
+      break;
+
+    case 'markdown':
+      content = generateMarkdownReport(result);
+      break;
+
+    case 'text':
+    default:
+      content = generateTextReport(result);
+      break;
+  }
+
+  await writeFile(filePath, content, 'utf-8');
+}
+
+function generateMarkdownReport(result: CheckLinksResult): string {
+  const lines: string[] = [];
+
+  lines.push('# Link Check Report');
+  lines.push('');
+  lines.push(`**URL:** ${result.url}`);
+  lines.push(`**Timestamp:** ${result.timestamp}`);
+  if (result.duration) {
+    lines.push(`**Duration:** ${(result.duration / 1000).toFixed(2)}s`);
+  }
+  lines.push('');
+  lines.push(
+    `**Status:** ${result.success ? '✅ All links valid' : '❌ Broken links found'}`
+  );
+  lines.push(`**Total links:** ${result.total}`);
+  lines.push(`**Broken links:** ${result.broken}`);
+  lines.push('');
+
+  if (result.errors.length > 0) {
+    lines.push('## Broken Links');
+    lines.push('');
+    lines.push('| Status | URL | Reason |');
+    lines.push('|--------|-----|--------|');
+    for (const { url, status, reason } of result.errors) {
+      lines.push(`| ${status} | ${url} | ${reason || '-'} |`);
+    }
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+function generateTextReport(result: CheckLinksResult): string {
+  const lines: string[] = [];
+
+  lines.push('Link Check Report');
+  lines.push('='.repeat(50));
+  lines.push(`URL: ${result.url}`);
+  lines.push(`Timestamp: ${result.timestamp}`);
+  if (result.duration) {
+    lines.push(`Duration: ${(result.duration / 1000).toFixed(2)}s`);
+  }
+  lines.push('');
+  lines.push(
+    `Status: ${result.success ? '✅ All links valid' : '❌ Broken links found'}`
+  );
+  lines.push(`Total links: ${result.total}`);
+  lines.push(`Broken links: ${result.broken}`);
+  lines.push('');
+
+  if (result.errors.length > 0) {
+    lines.push('Broken Links:');
+    lines.push('-'.repeat(50));
+    for (const { url, status, reason } of result.errors) {
+      lines.push(`[${status}] ${url}${reason ? ` (${reason})` : ''}`);
+    }
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+function hash(str: string): string {
+  let h = 0;
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i);
+    h = (h << 5) - h + char;
+    h = h & h;
+  }
+  return Math.abs(h).toString(36);
+}

package/src/reports/claude-context.ts

@@ -0,0 +1,149 @@
+/**
+ * @djangocfg/seo - CLAUDE.md Generator
+ * Generates AI context file with package docs and current audit state
+ */
+
+import type { SeoReport } from '../types/index.js';
+
+/**
+ * Generate CLAUDE.md content for AI context
+ */
+export function generateClaudeContext(report: SeoReport): string {
+  const lines: string[] = [];
+
+  // Package docs (static)
+  lines.push('# @djangocfg/seo');
+  lines.push('');
+  lines.push('SEO audit toolkit. Generates AI-optimized split reports (max 1000 lines each).');
+  lines.push('');
+  lines.push('## Commands');
+  lines.push('');
+  lines.push('```bash');
+  lines.push('# Audit (HTTP-based, crawls live site)');
+  lines.push('pnpm seo:audit                    # Full audit (split reports)');
+  lines.push('pnpm seo:audit --env dev          # Audit local dev');
+  lines.push('pnpm seo:audit --format all       # All formats');
+  lines.push('');
+  lines.push('# Content (file-based, scans MDX/content/)');
+  lines.push('pnpm exec djangocfg-seo content check    # Check MDX links');
+  lines.push('pnpm exec djangocfg-seo content fix      # Show fixable links');
+  lines.push('pnpm exec djangocfg-seo content fix --fix # Apply fixes');
+  lines.push('pnpm exec djangocfg-seo content sitemap  # Generate sitemap.ts');
+  lines.push('```');
+  lines.push('');
+  lines.push('## Options');
+  lines.push('');
+  lines.push('- `--env, -e` - prod (default) or dev');
+  lines.push('- `--site, -s` - Site URL (overrides env)');
+  lines.push('- `--output, -o` - Output directory');
+  lines.push('- `--format, -f` - split (default), json, markdown, ai-summary, all');
+  lines.push('- `--max-pages` - Max pages (default: 100)');
+  lines.push('- `--service-account` - Google service account JSON path');
+  lines.push('- `--content-dir` - Content directory (default: content/)');
+  lines.push('- `--base-path` - Base URL path for docs (default: /docs)');
+  lines.push('');
+  lines.push('## Reports');
+  lines.push('');
+  lines.push('- `seo-*-index.md` - Summary + links to categories');
+  lines.push('- `seo-*-technical.md` - Broken links, sitemap issues');
+  lines.push('- `seo-*-content.md` - H1, meta, title issues');
+  lines.push('- `seo-*-performance.md` - Load time, TTFB issues');
+  lines.push('- `seo-ai-summary-*.md` - Quick overview');
+  lines.push('');
+
+  // Issue types - important for AI to understand priorities
+  lines.push('## Issue Severity');
+  lines.push('');
+  lines.push('- **critical** - Blocks indexing (fix immediately)');
+  lines.push('- **error** - SEO problems (high priority)');
+  lines.push('- **warning** - Recommendations (medium priority)');
+  lines.push('- **info** - Best practices (low priority)');
+  lines.push('');
+  lines.push('## Issue Categories');
+  lines.push('');
+  lines.push('- **technical** - Broken links, sitemap, robots.txt');
+  lines.push('- **content** - Missing H1, meta description, title');
+  lines.push('- **indexing** - Not indexed, crawl errors from GSC');
+  lines.push('- **performance** - Slow load time (>3s), high TTFB (>800ms)');
+  lines.push('');
+
+  // Routes info
+  lines.push('## Routes Scanner');
+  lines.push('');
+  lines.push('Scans Next.js App Router `app/` directory. Handles:');
+  lines.push('- Route groups `(group)` - ignored in URL');
+  lines.push('- Dynamic `[slug]` - shown as `:slug`');
+  lines.push('- Catch-all `[...slug]` - shown as `:...slug`');
+  lines.push('- Parallel `@folder` - skipped');
+  lines.push('- Private `_folder` - skipped');
+  lines.push('');
+
+  // Link guidelines
+  lines.push('## Link Guidelines');
+  lines.push('');
+  lines.push('### Nextra/MDX Projects (content/)');
+  lines.push('');
+  lines.push('For non-index files (e.g., `overview.mdx`):');
+  lines.push('- **Sibling file**: `../sibling` (one level up)');
+  lines.push('- **Other section**: `/docs/full/path` (absolute)');
+  lines.push('- **AVOID**: `./sibling` (browser adds filename to path!)');
+  lines.push('- **AVOID**: `../../deep/path` (hard to maintain)');
+  lines.push('');
+  lines.push('For index files (e.g., `index.mdx`):');
+  lines.push('- **Child file**: `./child` works correctly');
+  lines.push('- **Sibling folder**: `../sibling/` or absolute');
+  lines.push('');
+  lines.push('### Next.js App Router Projects');
+  lines.push('');
+  lines.push('Use declarative routes from `_routes/`:');
+  lines.push('```typescript');
+  lines.push('import { routes } from "@/app/_routes";');
+  lines.push('<Link href={routes.dashboard.machines}>Machines</Link>');
+  lines.push('```');
+  lines.push('');
+  lines.push('Benefits: type-safe, refactor-friendly, centralized.');
+  lines.push('');
+
+  // Current audit state (dynamic)
+  lines.push('---');
+  lines.push('');
+  lines.push('## Current Audit');
+  lines.push('');
+  lines.push(`Site: ${report.siteUrl}`);
+  lines.push(`Score: ${report.summary.healthScore}/100`);
+  lines.push(`Date: ${report.generatedAt.slice(0, 10)}`);
+  lines.push('');
+
+  // Issue summary
+  lines.push('### Issues');
+  lines.push('');
+  const { critical = 0, error = 0, warning = 0, info = 0 } = report.summary.issuesBySeverity;
+  if (critical > 0) lines.push(`- Critical: ${critical}`);
+  if (error > 0) lines.push(`- Error: ${error}`);
+  if (warning > 0) lines.push(`- Warning: ${warning}`);
+  if (info > 0) lines.push(`- Info: ${info}`);
+  lines.push('');
+
+  // Top 5 actions
+  lines.push('### Top Actions');
+  lines.push('');
+  const topRecs = report.recommendations.slice(0, 5);
+  for (let i = 0; i < topRecs.length; i++) {
+    const rec = topRecs[i];
+    if (!rec) continue;
+    lines.push(`${i + 1}. **${rec.title}** (${rec.affectedUrls.length} URLs)`);
+  }
+  lines.push('');
+
+  // Report files in this directory
+  lines.push('### Report Files');
+  lines.push('');
+  lines.push('See split reports in this directory:');
+  lines.push('- `seo-*-index.md` - Start here');
+  lines.push('- `seo-*-technical.md` - Technical issues');
+  lines.push('- `seo-*-content.md` - Content issues');
+  lines.push('- `seo-*-performance.md` - Performance issues');
+  lines.push('');
+
+  return lines.join('\n');
+}