@djangocfg/seo 2.1.50

package/src/google-console/client.ts

@@ -0,0 +1,281 @@
+/**
+ * @djangocfg/seo - Google Search Console Client
+ * Main client for interacting with Google Search Console API
+ */
+
+import { searchconsole, type searchconsole_v1 } from '@googleapis/searchconsole';
+import type { JWT } from 'google-auth-library';
+import consola from 'consola';
+import pLimit from 'p-limit';
+import pRetry from 'p-retry';
+import { createAuthClient, verifyAuth } from './auth.js';
+import type {
+  GoogleConsoleConfig,
+  UrlInspectionResult,
+  CoverageState,
+  IndexingState,
+  IndexingVerdict,
+  RobotsTxtState,
+  PageFetchState,
+} from '../types/index.js';
+
+export class GoogleConsoleClient {
+  private auth: JWT;
+  private searchconsole: searchconsole_v1.Searchconsole;
+  private siteUrl: string;
+  private gscSiteUrl: string; // Format for GSC API (may be sc-domain:xxx)
+  private limit = pLimit(2); // Max 2 concurrent requests (Cloudflare-friendly)
+  private requestDelay = 500; // Delay between requests in ms
+
+  constructor(config: GoogleConsoleConfig) {
+    this.auth = createAuthClient(config);
+    this.searchconsole = searchconsole({ version: 'v1', auth: this.auth });
+    this.siteUrl = config.siteUrl;
+
+    // Support both URL prefix and domain property formats
+    // If gscSiteUrl provided, use it; otherwise try domain property format
+    if (config.gscSiteUrl) {
+      this.gscSiteUrl = config.gscSiteUrl;
+    } else {
+      // Default to domain property format (most common)
+      const domain = new URL(config.siteUrl).hostname;
+      this.gscSiteUrl = `sc-domain:${domain}`;
+    }
+
+    consola.debug(`GSC site URL: ${this.gscSiteUrl}`);
+  }
+
+  /**
+   * Delay helper for rate limiting
+   */
+  private delay(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Verify the client is authenticated
+   */
+  async verify(): Promise<boolean> {
+    return verifyAuth(this.auth, this.siteUrl);
+  }
+
+  /**
+   * List all sites in Search Console
+   */
+  async listSites(): Promise<string[]> {
+    try {
+      const response = await this.searchconsole.sites.list();
+      return response.data.siteEntry?.map((site) => site.siteUrl || '') || [];
+    } catch (error) {
+      consola.error('Failed to list sites:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Inspect a single URL
+   */
+  async inspectUrl(url: string): Promise<UrlInspectionResult> {
+    return this.limit(async () => {
+      return pRetry(
+        async () => {
+          const response = await this.searchconsole.urlInspection.index.inspect({
+            requestBody: {
+              inspectionUrl: url,
+              siteUrl: this.gscSiteUrl,
+              languageCode: 'en-US',
+            },
+          });
+
+          const result = response.data.inspectionResult;
+
+          if (!result?.indexStatusResult) {
+            throw new Error(`No inspection result for URL: ${url}`);
+          }
+
+          return this.mapInspectionResult(url, result);
+        },
+        {
+          retries: 2,
+          minTimeout: 2000,
+          maxTimeout: 10000,
+          factor: 2, // Exponential backoff
+          onFailedAttempt: (ctx) => {
+            // Only log on final failure to reduce noise
+            if (ctx.retriesLeft === 0) {
+              consola.warn(`Failed: ${url}`);
+            }
+          },
+        }
+      );
+    });
+  }
+
+  /**
+   * Inspect multiple URLs in batch
+   * Stops early if too many consecutive errors (likely rate limiting)
+   */
+  async inspectUrls(urls: string[]): Promise<UrlInspectionResult[]> {
+    consola.info(`Inspecting ${urls.length} URLs...`);
+
+    const results: UrlInspectionResult[] = [];
+    const errors: Array<{ url: string; error: Error }> = [];
+    let consecutiveErrors = 0;
+    const maxConsecutiveErrors = 3; // Stop after 3 consecutive failures
+
+    // Process URLs sequentially with delay to avoid rate limiting
+    for (const url of urls) {
+      try {
+        const result = await this.inspectUrl(url);
+        results.push(result);
+        consecutiveErrors = 0; // Reset on success
+        // Add delay between requests
+        await this.delay(this.requestDelay);
+      } catch (error) {
+        const err = error as Error;
+        errors.push({ url, error: err });
+        consecutiveErrors++;
+
+        // Early exit on consecutive errors (likely rate limiting or auth issue)
+        if (consecutiveErrors >= maxConsecutiveErrors) {
+          console.log('');
+          consola.error(`Stopping after ${maxConsecutiveErrors} consecutive failures`);
+          this.showRateLimitHelp();
+          break;
+        }
+      }
+    }
+
+    if (errors.length > 0 && consecutiveErrors < maxConsecutiveErrors) {
+      consola.warn(`Failed to inspect ${errors.length} URLs`);
+    }
+
+    if (results.length > 0) {
+      consola.success(`Successfully inspected ${results.length}/${urls.length} URLs`);
+    } else if (errors.length > 0) {
+      consola.warn('No URLs were successfully inspected');
+    }
+
+    return results;
+  }
+
+  /**
+   * Show help message for rate limiting issues
+   */
+  private showRateLimitHelp(): void {
+    consola.info('Possible causes:');
+    consola.info('  1. Google API quota exceeded (2000 requests/day)');
+    consola.info('  2. Cloudflare blocking Google\'s crawler');
+    consola.info('  3. Service account not added to GSC');
+    console.log('');
+    consola.info('Solutions:');
+    consola.info('  • Check GSC access: https://search.google.com/search-console/users');
+    console.log('');
+    consola.info('  • Cloudflare WAF rule to allow Googlebot:');
+    consola.info('    1. Dashboard → Security → WAF → Custom rules → Create rule');
+    consola.info('    2. Name: "Allow Googlebot"');
+    consola.info('    3. Field: "Known Bots" | Operator: "equals" | Value: "true"');
+    consola.info('    4. Or click "Edit expression" and paste: (cf.client.bot)');
+    consola.info('    5. Action: Skip → check all rules');
+    consola.info('    6. Deploy');
+    consola.info('    Docs: https://developers.cloudflare.com/waf/custom-rules/use-cases/allow-traffic-from-verified-bots/');
+    console.log('');
+  }
+
+  /**
+   * Get search analytics data
+   */
+  async getSearchAnalytics(
+    options: {
+      startDate: string;
+      endDate: string;
+      dimensions?: ('query' | 'page' | 'country' | 'device' | 'date')[];
+      rowLimit?: number;
+    }
+  ): Promise<searchconsole_v1.Schema$ApiDataRow[]> {
+    try {
+      const response = await this.searchconsole.searchanalytics.query({
+        siteUrl: this.gscSiteUrl,
+        requestBody: {
+          startDate: options.startDate,
+          endDate: options.endDate,
+          dimensions: options.dimensions || ['page'],
+          rowLimit: options.rowLimit || 1000,
+        },
+      });
+
+      return response.data.rows || [];
+    } catch (error) {
+      consola.error('Failed to get search analytics:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Get list of sitemaps
+   */
+  async getSitemaps(): Promise<searchconsole_v1.Schema$WmxSitemap[]> {
+    try {
+      const response = await this.searchconsole.sitemaps.list({
+        siteUrl: this.gscSiteUrl,
+      });
+
+      return response.data.sitemap || [];
+    } catch (error) {
+      consola.error('Failed to get sitemaps:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Map API response to our types
+   */
+  private mapInspectionResult(
+    url: string,
+    result: searchconsole_v1.Schema$UrlInspectionResult
+  ): UrlInspectionResult {
+    const indexStatus = result.indexStatusResult!;
+
+    return {
+      url,
+      inspectionResultLink: result.inspectionResultLink || undefined,
+      indexStatusResult: {
+        verdict: (indexStatus.verdict as IndexingVerdict) || 'VERDICT_UNSPECIFIED',
+        coverageState: (indexStatus.coverageState as CoverageState) || 'COVERAGE_STATE_UNSPECIFIED',
+        indexingState: (indexStatus.indexingState as IndexingState) || 'INDEXING_STATE_UNSPECIFIED',
+        robotsTxtState: (indexStatus.robotsTxtState as RobotsTxtState) || 'ROBOTS_TXT_STATE_UNSPECIFIED',
+        pageFetchState: (indexStatus.pageFetchState as PageFetchState) || 'PAGE_FETCH_STATE_UNSPECIFIED',
+        lastCrawlTime: indexStatus.lastCrawlTime || undefined,
+        crawledAs: indexStatus.crawledAs as 'DESKTOP' | 'MOBILE' | undefined,
+        googleCanonical: indexStatus.googleCanonical || undefined,
+        userCanonical: indexStatus.userCanonical || undefined,
+        sitemap: indexStatus.sitemap || undefined,
+        referringUrls: indexStatus.referringUrls || undefined,
+      },
+      mobileUsabilityResult: result.mobileUsabilityResult
+        ? {
+            verdict: (result.mobileUsabilityResult.verdict as IndexingVerdict) || 'VERDICT_UNSPECIFIED',
+            issues: result.mobileUsabilityResult.issues?.map((issue) => ({
+              issueType: issue.issueType || 'UNKNOWN',
+              message: issue.message || '',
+            })),
+          }
+        : undefined,
+      richResultsResult: result.richResultsResult
+        ? {
+            verdict: (result.richResultsResult.verdict as IndexingVerdict) || 'VERDICT_UNSPECIFIED',
+            detectedItems: result.richResultsResult.detectedItems?.map((item) => ({
+              richResultType: item.richResultType || 'UNKNOWN',
+              items: item.items?.map((i) => ({
+                name: i.name || '',
+                issues: i.issues?.map((issue) => ({
+                  issueMessage: issue.issueMessage || '',
+                  severity: (issue.severity as 'ERROR' | 'WARNING') || 'WARNING',
+                })),
+              })),
+            })),
+          }
+        : undefined,
+    };
+  }
+}

package/src/google-console/index.ts

@@ -0,0 +1,9 @@
+/**
+ * @djangocfg/seo - Google Console Module
+ * Integration with Google Search Console API
+ */
+
+export { GoogleConsoleClient } from './client.js';
+export { createAuthClient, verifyAuth, loadCredentials } from './auth.js';
+export { analyzeInspectionResults } from './analyzer.js';
+export type { ServiceAccountCredentials } from './auth.js';

package/src/index.ts

@@ -0,0 +1,144 @@
+/**
+ * @djangocfg/seo
+ * SEO analytics and indexing diagnostics module
+ *
+ * Features:
+ * - Google Search Console integration (URL Inspection API)
+ * - Site crawler with SEO analysis
+ * - AI-ready reports (JSON + Markdown)
+ * - robots.txt and sitemap validation
+ *
+ * @example
+ * ```typescript
+ * import { SeoAnalyzer } from '@djangocfg/seo';
+ *
+ * const analyzer = new SeoAnalyzer({
+ *   siteUrl: 'https://example.com',
+ *   googleConsole: {
+ *     serviceAccountPath: './service_account.json',
+ *   },
+ * });
+ *
+ * const report = await analyzer.analyze();
+ * await analyzer.saveReport('./reports');
+ * ```
+ */
+
+// Main Analyzer Class
+export { SeoAnalyzer } from './analyzer.js';
+
+// Google Console
+export {
+  GoogleConsoleClient,
+  createAuthClient,
+  verifyAuth,
+  analyzeInspectionResults,
+} from './google-console/index.js';
+
+// Crawler
+export {
+  SiteCrawler,
+  analyzeCrawlResults,
+  analyzeRobotsTxt,
+  isUrlAllowed,
+  analyzeSitemap,
+  analyzeAllSitemaps,
+} from './crawler/index.js';
+
+// Link Checker
+export {
+  checkLinks,
+  linkResultsToSeoIssues,
+} from './link-checker/index.js';
+export type {
+  CheckLinksOptions,
+  CheckLinksResult,
+} from './link-checker/index.js';
+
+// Reports
+export {
+  generateAndSaveReports,
+  generateJsonReport,
+  generateMarkdownReport,
+  generateAiSummary,
+  printReportSummary,
+  mergeReports,
+  AI_REPORT_SCHEMA,
+} from './reports/index.js';
+
+// Types
+export type {
+  // Core types
+  SeoIssue,
+  SeoReport,
+  ReportSummary,
+  Recommendation,
+  IssueSeverity,
+  IssueCategory,
+
+  // Google Console types
+  GoogleConsoleConfig,
+  UrlInspectionResult,
+  IndexingVerdict,
+  CoverageState,
+  IndexingState,
+  RobotsTxtState,
+  PageFetchState,
+
+  // Crawler types
+  CrawlResult,
+  CrawlerConfig,
+
+  // Config types
+  SeoModuleConfig,
+} from './types/index.js';
+
+// Utils
+export {
+  loadUrlsFromFile,
+  normalizeUrl,
+  isSameDomain,
+  formatBytes,
+  formatDuration,
+  chunk,
+  sleep,
+  retry,
+} from './utils/index.js';
+
+// Content (MDX/Nextra tools)
+export {
+  checkContentLinks,
+  fixContentLinks,
+  generateSitemap,
+  generateSitemapData,
+  flattenSitemap,
+  countSitemapItems,
+  detectProjectType,
+  scanProject,
+  groupBrokenLinksByFile,
+} from './content/index.js';
+
+export type {
+  ContentConfig,
+  SitemapConfig,
+  SitemapItem,
+  SitemapData,
+  BrokenLink,
+  LinkCheckResult,
+  FixLinksResult,
+  ContentScanResult,
+} from './content/index.js';
+
+// Routes (App Router scanner)
+export {
+  scanRoutes,
+  findAppDir,
+  routeToUrl,
+  getStaticUrls,
+} from './routes/scanner.js';
+
+export type {
+  RouteInfo,
+  ScanResult,
+  ScanOptions,
+} from './routes/scanner.js';