@djangocfg/seo 2.1.50

package/src/crawler/sitemap-validator.ts

@@ -0,0 +1,204 @@
+/**
+ * @djangocfg/seo - Sitemap Validator
+ * Validate XML sitemaps
+ */
+
+import { load } from 'cheerio';
+import consola from 'consola';
+import type { SeoIssue } from '../types/index.js';
+
+export interface SitemapAnalysis {
+  url: string;
+  exists: boolean;
+  type: 'sitemap' | 'sitemap-index' | 'unknown';
+  urls: string[];
+  childSitemaps: string[];
+  lastmod?: string;
+  issues: SeoIssue[];
+}
+
+/**
+ * Analyze a sitemap URL
+ */
+export async function analyzeSitemap(sitemapUrl: string): Promise<SitemapAnalysis> {
+  const analysis: SitemapAnalysis = {
+    url: sitemapUrl,
+    exists: false,
+    type: 'unknown',
+    urls: [],
+    childSitemaps: [],
+    issues: [],
+  };
+
+  try {
+    const response = await fetch(sitemapUrl, {
+      headers: {
+        Accept: 'application/xml, text/xml, */*',
+      },
+    });
+
+    if (!response.ok) {
+      analysis.issues.push({
+        id: `sitemap-not-found-${hash(sitemapUrl)}`,
+        url: sitemapUrl,
+        category: 'technical',
+        severity: 'error',
+        title: 'Sitemap not accessible',
+        description: `Sitemap returned HTTP ${response.status}.`,
+        recommendation: 'Ensure the sitemap URL is correct and accessible.',
+        detectedAt: new Date().toISOString(),
+        metadata: { statusCode: response.status },
+      });
+      return analysis;
+    }
+
+    analysis.exists = true;
+    const content = await response.text();
+
+    // Check content type
+    const contentType = response.headers.get('content-type') || '';
+    if (!contentType.includes('xml') && !content.trim().startsWith('<?xml')) {
+      analysis.issues.push({
+        id: `sitemap-not-xml-${hash(sitemapUrl)}`,
+        url: sitemapUrl,
+        category: 'technical',
+        severity: 'warning',
+        title: 'Sitemap is not XML',
+        description: 'The sitemap does not have an XML content type.',
+        recommendation: 'Ensure sitemap is served with Content-Type: application/xml.',
+        detectedAt: new Date().toISOString(),
+        metadata: { contentType },
+      });
+    }
+
+    // Parse XML
+    const $ = load(content, { xmlMode: true });
+
+    // Check if it's a sitemap index
+    const sitemapIndex = $('sitemapindex');
+    if (sitemapIndex.length > 0) {
+      analysis.type = 'sitemap-index';
+
+      $('sitemap').each((_, el) => {
+        const loc = $('loc', el).text().trim();
+        if (loc) {
+          analysis.childSitemaps.push(loc);
+        }
+      });
+
+      consola.debug(`Sitemap index contains ${analysis.childSitemaps.length} sitemaps`);
+    } else {
+      analysis.type = 'sitemap';
+
+      $('url').each((_, el) => {
+        const loc = $('loc', el).text().trim();
+        if (loc) {
+          analysis.urls.push(loc);
+        }
+      });
+
+      const lastmod = $('url lastmod').first().text().trim();
+      if (lastmod) {
+        analysis.lastmod = lastmod;
+      }
+
+      consola.debug(`Sitemap contains ${analysis.urls.length} URLs`);
+    }
+
+    // Validate sitemap content
+    if (analysis.type === 'sitemap' && analysis.urls.length === 0) {
+      analysis.issues.push({
+        id: `sitemap-empty-${hash(sitemapUrl)}`,
+        url: sitemapUrl,
+        category: 'technical',
+        severity: 'warning',
+        title: 'Sitemap is empty',
+        description: 'The sitemap contains no URLs.',
+        recommendation: 'Add URLs to your sitemap or remove it if not needed.',
+        detectedAt: new Date().toISOString(),
+      });
+    }
+
+    // Check for too many URLs (Google limit is 50,000)
+    if (analysis.urls.length > 50000) {
+      analysis.issues.push({
+        id: `sitemap-too-large-${hash(sitemapUrl)}`,
+        url: sitemapUrl,
+        category: 'technical',
+        severity: 'error',
+        title: 'Sitemap exceeds URL limit',
+        description: `Sitemap contains ${analysis.urls.length} URLs. Maximum is 50,000.`,
+        recommendation: 'Split the sitemap into multiple files using a sitemap index.',
+        detectedAt: new Date().toISOString(),
+        metadata: { urlCount: analysis.urls.length },
+      });
+    }
+
+    // Check file size (Google limit is 50MB uncompressed)
+    const sizeInMB = new Blob([content]).size / (1024 * 1024);
+    if (sizeInMB > 50) {
+      analysis.issues.push({
+        id: `sitemap-too-large-size-${hash(sitemapUrl)}`,
+        url: sitemapUrl,
+        category: 'technical',
+        severity: 'error',
+        title: 'Sitemap exceeds size limit',
+        description: `Sitemap is ${sizeInMB.toFixed(2)}MB. Maximum is 50MB.`,
+        recommendation: 'Split the sitemap or compress it.',
+        detectedAt: new Date().toISOString(),
+        metadata: { sizeMB: sizeInMB },
+      });
+    }
+  } catch (error) {
+    consola.error('Failed to analyze sitemap:', error);
+    analysis.issues.push({
+      id: `sitemap-error-${hash(sitemapUrl)}`,
+      url: sitemapUrl,
+      category: 'technical',
+      severity: 'error',
+      title: 'Failed to parse sitemap',
+      description: `Error: ${error instanceof Error ? error.message : 'Unknown error'}`,
+      recommendation: 'Check sitemap validity using Google Search Console.',
+      detectedAt: new Date().toISOString(),
+    });
+  }
+
+  return analysis;
+}
+
+/**
+ * Recursively analyze a sitemap and all its children
+ */
+export async function analyzeAllSitemaps(
+  sitemapUrl: string,
+  maxDepth = 3
+): Promise<SitemapAnalysis[]> {
+  const results: SitemapAnalysis[] = [];
+  const visited = new Set<string>();
+
+  async function analyze(url: string, depth: number): Promise<void> {
+    if (depth > maxDepth || visited.has(url)) return;
+    visited.add(url);
+
+    const analysis = await analyzeSitemap(url);
+    results.push(analysis);
+
+    // Recursively analyze child sitemaps
+    for (const childUrl of analysis.childSitemaps) {
+      await analyze(childUrl, depth + 1);
+    }
+  }
+
+  await analyze(sitemapUrl, 0);
+  return results;
+}
+
+function hash(str: string): string {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i);
+    hash = (hash << 5) - hash + char;
+    hash = hash & hash;
+  }
+  return Math.abs(hash).toString(36);
+}

package/src/google-console/analyzer.ts

@@ -0,0 +1,317 @@
+/**
+ * @djangocfg/seo - Google Console Analyzer
+ * Analyze URL inspection results and detect SEO issues
+ */
+
+import type {
+  UrlInspectionResult,
+  SeoIssue,
+  IssueSeverity,
+  IssueCategory,
+} from '../types/index.js';
+
+/**
+ * Analyze URL inspection results and extract SEO issues
+ */
+export function analyzeInspectionResults(results: UrlInspectionResult[]): SeoIssue[] {
+  const issues: SeoIssue[] = [];
+
+  for (const result of results) {
+    issues.push(...analyzeUrlInspection(result));
+  }
+
+  return issues.sort((a, b) => severityOrder(a.severity) - severityOrder(b.severity));
+}
+
+function analyzeUrlInspection(result: UrlInspectionResult): SeoIssue[] {
+  const issues: SeoIssue[] = [];
+  const { indexStatusResult, mobileUsabilityResult, richResultsResult } = result;
+
+  // Indexing Issues
+  switch (indexStatusResult.coverageState) {
+    case 'CRAWLED_CURRENTLY_NOT_INDEXED':
+      issues.push({
+        id: `crawled-not-indexed-${hash(result.url)}`,
+        url: result.url,
+        category: 'indexing',
+        severity: 'error',
+        title: 'Page crawled but not indexed',
+        description:
+          'Google crawled this page but decided not to index it. This often indicates low content quality or duplicate content.',
+        recommendation:
+          'Improve content quality, ensure uniqueness, add more valuable information, and check for duplicate content issues.',
+        detectedAt: new Date().toISOString(),
+        metadata: { coverageState: indexStatusResult.coverageState },
+      });
+      break;
+
+    case 'DISCOVERED_CURRENTLY_NOT_INDEXED':
+      issues.push({
+        id: `discovered-not-indexed-${hash(result.url)}`,
+        url: result.url,
+        category: 'indexing',
+        severity: 'warning',
+        title: 'Page discovered but not crawled',
+        description:
+          'Google discovered this URL but has not crawled it yet. This may indicate crawl budget issues or low priority.',
+        recommendation:
+          'Improve internal linking to this page, submit URL through Google Search Console, or add to sitemap.',
+        detectedAt: new Date().toISOString(),
+        metadata: { coverageState: indexStatusResult.coverageState },
+      });
+      break;
+
+    case 'DUPLICATE_WITHOUT_USER_SELECTED_CANONICAL':
+      issues.push({
+        id: `duplicate-no-canonical-${hash(result.url)}`,
+        url: result.url,
+        category: 'indexing',
+        severity: 'warning',
+        title: 'Duplicate page without canonical',
+        description:
+          'This page is considered a duplicate but no canonical URL has been specified. Google chose a canonical for you.',
+        recommendation:
+          'Add a canonical tag pointing to the preferred version of this page.',
+        detectedAt: new Date().toISOString(),
+        metadata: {
+          coverageState: indexStatusResult.coverageState,
+          googleCanonical: indexStatusResult.googleCanonical,
+        },
+      });
+      break;
+
+    case 'DUPLICATE_GOOGLE_CHOSE_DIFFERENT_CANONICAL':
+      issues.push({
+        id: `canonical-mismatch-${hash(result.url)}`,
+        url: result.url,
+        category: 'indexing',
+        severity: 'warning',
+        title: 'Google chose different canonical',
+        description:
+          'You specified a canonical URL, but Google chose a different one. This may cause indexing issues.',
+        recommendation:
+          'Review canonical tags and ensure they point to the correct URL. Check for duplicate content.',
+        detectedAt: new Date().toISOString(),
+        metadata: {
+          coverageState: indexStatusResult.coverageState,
+          userCanonical: indexStatusResult.userCanonical,
+          googleCanonical: indexStatusResult.googleCanonical,
+        },
+      });
+      break;
+  }
+
+  // Indexing State Issues
+  switch (indexStatusResult.indexingState) {
+    case 'BLOCKED_BY_META_TAG':
+      issues.push({
+        id: `blocked-meta-noindex-${hash(result.url)}`,
+        url: result.url,
+        category: 'indexing',
+        severity: 'error',
+        title: 'Blocked by noindex meta tag',
+        description: 'This page has a noindex meta tag preventing it from being indexed.',
+        recommendation:
+          'Remove the noindex meta tag if you want this page to be indexed. If intentional, no action needed.',
+        detectedAt: new Date().toISOString(),
+        metadata: { indexingState: indexStatusResult.indexingState },
+      });
+      break;
+
+    case 'BLOCKED_BY_HTTP_HEADER':
+      issues.push({
+        id: `blocked-http-header-${hash(result.url)}`,
+        url: result.url,
+        category: 'indexing',
+        severity: 'error',
+        title: 'Blocked by X-Robots-Tag header',
+        description: 'This page has a noindex directive in the X-Robots-Tag HTTP header.',
+        recommendation:
+          'Remove the X-Robots-Tag: noindex header if you want this page to be indexed.',
+        detectedAt: new Date().toISOString(),
+        metadata: { indexingState: indexStatusResult.indexingState },
+      });
+      break;
+
+    case 'BLOCKED_BY_ROBOTS_TXT':
+      issues.push({
+        id: `blocked-robots-txt-${hash(result.url)}`,
+        url: result.url,
+        category: 'crawling',
+        severity: 'error',
+        title: 'Blocked by robots.txt',
+        description: 'This page is blocked from crawling by robots.txt rules.',
+        recommendation:
+          'Update robots.txt to allow crawling if you want this page to be indexed.',
+        detectedAt: new Date().toISOString(),
+        metadata: { indexingState: indexStatusResult.indexingState },
+      });
+      break;
+  }
+
+  // Page Fetch Issues
+  switch (indexStatusResult.pageFetchState) {
+    case 'SOFT_404':
+      issues.push({
+        id: `soft-404-${hash(result.url)}`,
+        url: result.url,
+        category: 'technical',
+        severity: 'error',
+        title: 'Soft 404 error',
+        description:
+          'This page returns a 200 status but Google detected it as a 404 page (empty or low-value content).',
+        recommendation:
+          'Either return a proper 404 status code or add meaningful content to this page.',
+        detectedAt: new Date().toISOString(),
+        metadata: { pageFetchState: indexStatusResult.pageFetchState },
+      });
+      break;
+
+    case 'NOT_FOUND':
+      issues.push({
+        id: `404-error-${hash(result.url)}`,
+        url: result.url,
+        category: 'technical',
+        severity: 'error',
+        title: '404 Not Found',
+        description: 'This page returns a 404 error.',
+        recommendation:
+          'Either restore the page content or set up a redirect to a relevant page.',
+        detectedAt: new Date().toISOString(),
+        metadata: { pageFetchState: indexStatusResult.pageFetchState },
+      });
+      break;
+
+    case 'SERVER_ERROR':
+      issues.push({
+        id: `server-error-${hash(result.url)}`,
+        url: result.url,
+        category: 'technical',
+        severity: 'critical',
+        title: 'Server error (5xx)',
+        description: 'This page returns a server error when Google tries to crawl it.',
+        recommendation:
+          'Fix the server-side error. Check server logs for details.',
+        detectedAt: new Date().toISOString(),
+        metadata: { pageFetchState: indexStatusResult.pageFetchState },
+      });
+      break;
+
+    case 'REDIRECT_ERROR':
+      issues.push({
+        id: `redirect-error-${hash(result.url)}`,
+        url: result.url,
+        category: 'technical',
+        severity: 'error',
+        title: 'Redirect error',
+        description:
+          'There is a redirect issue with this page (redirect loop, too many redirects, or invalid redirect).',
+        recommendation:
+          'Fix the redirect chain. Ensure redirects point to valid, accessible pages.',
+        detectedAt: new Date().toISOString(),
+        metadata: { pageFetchState: indexStatusResult.pageFetchState },
+      });
+      break;
+
+    case 'ACCESS_DENIED':
+    case 'ACCESS_FORBIDDEN':
+      issues.push({
+        id: `access-denied-${hash(result.url)}`,
+        url: result.url,
+        category: 'technical',
+        severity: 'error',
+        title: 'Access denied (401/403)',
+        description: 'Google cannot access this page due to authentication requirements.',
+        recommendation:
+          'Ensure the page is publicly accessible without authentication for Googlebot.',
+        detectedAt: new Date().toISOString(),
+        metadata: { pageFetchState: indexStatusResult.pageFetchState },
+      });
+      break;
+  }
+
+  // Mobile Usability Issues
+  if (mobileUsabilityResult?.verdict === 'FAIL' && mobileUsabilityResult.issues) {
+    for (const issue of mobileUsabilityResult.issues) {
+      issues.push({
+        id: `mobile-${issue.issueType}-${hash(result.url)}`,
+        url: result.url,
+        category: 'mobile',
+        severity: 'warning',
+        title: `Mobile usability: ${formatIssueType(issue.issueType)}`,
+        description: issue.message || 'Mobile usability issue detected.',
+        recommendation: getMobileRecommendation(issue.issueType),
+        detectedAt: new Date().toISOString(),
+        metadata: { issueType: issue.issueType },
+      });
+    }
+  }
+
+  // Rich Results Issues
+  if (richResultsResult?.verdict === 'FAIL' && richResultsResult.detectedItems) {
+    for (const item of richResultsResult.detectedItems) {
+      for (const i of item.items || []) {
+        for (const issueDetail of i.issues || []) {
+          issues.push({
+            id: `rich-result-${item.richResultType}-${hash(result.url)}`,
+            url: result.url,
+            category: 'structured-data',
+            severity: issueDetail.severity === 'ERROR' ? 'error' : 'warning',
+            title: `${item.richResultType}: ${i.name}`,
+            description: issueDetail.issueMessage,
+            recommendation:
+              'Fix the structured data markup according to Google guidelines.',
+            detectedAt: new Date().toISOString(),
+            metadata: { richResultType: item.richResultType },
+          });
+        }
+      }
+    }
+  }
+
+  return issues;
+}
+
+function severityOrder(severity: IssueSeverity): number {
+  const order: Record<IssueSeverity, number> = {
+    critical: 0,
+    error: 1,
+    warning: 2,
+    info: 3,
+  };
+  return order[severity];
+}
+
+function hash(str: string): string {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    const char = str.charCodeAt(i);
+    hash = (hash << 5) - hash + char;
+    hash = hash & hash;
+  }
+  return Math.abs(hash).toString(36);
+}
+
+function formatIssueType(type: string): string {
+  return type
+    .replace(/_/g, ' ')
+    .toLowerCase()
+    .replace(/\b\w/g, (c) => c.toUpperCase());
+}
+
+function getMobileRecommendation(issueType: string): string {
+  const recommendations: Record<string, string> = {
+    MOBILE_FRIENDLY_RULE_USES_INCOMPATIBLE_PLUGINS:
+      'Remove Flash or other incompatible plugins. Use HTML5 alternatives.',
+    MOBILE_FRIENDLY_RULE_CONFIGURE_VIEWPORT:
+      'Add a viewport meta tag: <meta name="viewport" content="width=device-width, initial-scale=1">',
+    MOBILE_FRIENDLY_RULE_CONTENT_NOT_SIZED_TO_VIEWPORT:
+      'Ensure content width fits the viewport. Use responsive CSS.',
+    MOBILE_FRIENDLY_RULE_TAP_TARGETS_TOO_SMALL:
+      'Increase the size of touch targets (buttons, links) to at least 48x48 pixels.',
+    MOBILE_FRIENDLY_RULE_TEXT_TOO_SMALL:
+      'Use at least 16px font size for body text.',
+  };
+
+  return recommendations[issueType] || 'Fix the mobile usability issue according to Google guidelines.';
+}

package/src/google-console/auth.ts

@@ -0,0 +1,100 @@
+/**
+ * @djangocfg/seo - Google Console Authentication
+ * Service Account authentication for Google Search Console API
+ */
+
+import { JWT } from 'google-auth-library';
+import { readFileSync, existsSync } from 'node:fs';
+import consola from 'consola';
+import type { GoogleConsoleConfig } from '../types/index.js';
+
+const SCOPES = [
+  'https://www.googleapis.com/auth/webmasters.readonly',
+  'https://www.googleapis.com/auth/webmasters',
+];
+
+export interface ServiceAccountCredentials {
+  client_email: string;
+  private_key: string;
+  project_id?: string;
+}
+
+/**
+ * Load service account credentials from file or config
+ */
+export function loadCredentials(config: GoogleConsoleConfig): ServiceAccountCredentials {
+  if (config.serviceAccountJson) {
+    return config.serviceAccountJson;
+  }
+
+  if (config.serviceAccountPath) {
+    if (!existsSync(config.serviceAccountPath)) {
+      throw new Error(`Service account file not found: ${config.serviceAccountPath}`);
+    }
+
+    const content = readFileSync(config.serviceAccountPath, 'utf-8');
+    return JSON.parse(content) as ServiceAccountCredentials;
+  }
+
+  // Try to load from environment variable
+  const envJson = process.env.GOOGLE_SERVICE_ACCOUNT_JSON;
+  if (envJson) {
+    return JSON.parse(envJson) as ServiceAccountCredentials;
+  }
+
+  // Try default path
+  const defaultPath = './service_account.json';
+  if (existsSync(defaultPath)) {
+    const content = readFileSync(defaultPath, 'utf-8');
+    return JSON.parse(content) as ServiceAccountCredentials;
+  }
+
+  throw new Error(
+    'No service account credentials found. Provide serviceAccountPath, serviceAccountJson, or set GOOGLE_SERVICE_ACCOUNT_JSON env variable.'
+  );
+}
+
+/**
+ * Create authenticated JWT client
+ */
+export function createAuthClient(config: GoogleConsoleConfig): JWT {
+  const credentials = loadCredentials(config);
+
+  const auth = new JWT({
+    email: credentials.client_email,
+    key: credentials.private_key,
+    scopes: SCOPES,
+  });
+
+  // Store email for later display
+  (auth as any)._serviceAccountEmail = credentials.client_email;
+
+  return auth;
+}
+
+/**
+ * Verify authentication is working
+ */
+export async function verifyAuth(auth: JWT, siteUrl?: string): Promise<boolean> {
+  const email = (auth as any)._serviceAccountEmail || auth.email;
+
+  try {
+    await auth.authorize();
+    consola.success('Google Search Console authentication verified');
+    consola.info(`Service account: ${email}`);
+
+    // Build GSC users URL with domain
+    if (siteUrl) {
+      const domain = new URL(siteUrl).hostname;
+      const gscUrl = `https://search.google.com/search-console/users?resource_id=sc-domain%3A${domain}`;
+      consola.info(`Ensure this email has Full access in GSC: ${gscUrl}`);
+    }
+
+    return true;
+  } catch (error) {
+    consola.error('Authentication failed');
+    consola.info(`Service account email: ${email}`);
+    consola.info('Make sure this email is added to GSC with Full access');
+    return false;
+  }
+}