@houtini/seo-crawler-mcp 2.0.1

package/src/core/UrlManager.ts

@@ -0,0 +1,114 @@
+/**
+ * UrlManager - URL normalization, deduplication, and tracking
+ * 
+ * Responsibilities:
+ * - Normalize URLs (www-agnostic, no fragments, no trailing slashes)
+ * - Track discovered vs visited URLs
+ * - Determine if URL is internal
+ * - Track source pages ("linked from" feature)
+ * - Track depth of each URL
+ */
+
+export class UrlManager {
+  private baseDomain: string;
+  private discovered: Map<string, number> = new Map();
+  private visited: Set<string> = new Set();
+  private sourcePagesMap: Map<string, Set<string>> = new Map();
+
+  constructor(baseDomain: string) {
+    this.baseDomain = this.normalizeDomain(baseDomain);
+  }
+
+  normalizeUrl(url: string): string {
+    try {
+      const parsed = new URL(url);
+      
+      let clean = `${parsed.protocol}//${parsed.hostname}${parsed.pathname}`;
+      
+      if (parsed.search) {
+        clean += parsed.search;
+      }
+      
+      if (clean.endsWith('/') && clean.length > clean.indexOf('://') + 4) {
+        clean = clean.slice(0, -1);
+      }
+      
+      return clean;
+    } catch {
+      return url;
+    }
+  }
+
+  private normalizeDomain(domain: string): string {
+    try {
+      const url = domain.startsWith('http') ? domain : `https://${domain}`;
+      return new URL(url).hostname.replace(/^www\./, '');
+    } catch {
+      return domain.replace(/^www\./, '');
+    }
+  }
+
+  isInternal(url: string): boolean {
+    try {
+      const urlDomain = new URL(url).hostname.replace(/^www\./, '');
+      return urlDomain === this.baseDomain;
+    } catch {
+      return false;
+    }
+  }
+
+  addDiscovered(url: string, depth: number, sourceUrl?: string): void {
+    const normalized = this.normalizeUrl(url);
+    
+    if (!this.discovered.has(normalized)) {
+      this.discovered.set(normalized, depth);
+    }
+    
+    if (sourceUrl) {
+      const normalizedSource = this.normalizeUrl(sourceUrl);
+      if (!this.sourcePagesMap.has(normalized)) {
+        this.sourcePagesMap.set(normalized, new Set());
+      }
+      this.sourcePagesMap.get(normalized)!.add(normalizedSource);
+    }
+  }
+
+  markVisited(url: string): void {
+    const normalized = this.normalizeUrl(url);
+    this.visited.add(normalized);
+  }
+
+  isVisited(url: string): boolean {
+    return this.visited.has(this.normalizeUrl(url));
+  }
+
+  isDiscovered(url: string): boolean {
+    return this.discovered.has(this.normalizeUrl(url));
+  }
+
+  getSourcePages(url: string): string[] {
+    const normalized = this.normalizeUrl(url);
+    const sources = this.sourcePagesMap.get(normalized);
+    return sources ? Array.from(sources) : [];
+  }
+
+  getDepth(url: string): number {
+    return this.discovered.get(this.normalizeUrl(url)) ?? 0;
+  }
+
+  getTotalDiscovered(): number {
+    return this.discovered.size;
+  }
+
+  getTotalVisited(): number {
+    return this.visited.size;
+  }
+
+  getMaxDepth(): number {
+    return Math.max(0, ...Array.from(this.discovered.values()));
+  }
+
+  getUnvisitedUrls(): string[] {
+    return Array.from(this.discovered.keys()).filter(url => !this.visited.has(url));
+  }
+}

package/src/formatters/structured-report-format.ts

@@ -0,0 +1,254 @@
+/**
+ * Structured SEO Report Formatter
+ * 
+ * Transforms raw SEO analysis data into structured, actionable reports
+ * with clear issue categorization and remediation guidance.
+ * 
+ * Output Format:
+ * - Issue Name | Issue Type | Priority | URLs | % of Total | Description | How To Fix
+ */
+
+interface SEOIssue {
+  query: string;
+  category: string;
+  priority: string;
+  description: string;
+  impact: string;
+  fix: string;
+  affectedCount: number;
+  examples: Array<{
+    url: string;
+    detail?: string;
+  }>;
+}
+
+interface StructuredIssue {
+  issueName: string;
+  issueType: 'Issue' | 'Warning' | 'Opportunity';
+  issuePriority: 'Critical' | 'High' | 'Medium' | 'Low';
+  urls: number;
+  percentOfTotal: number;
+  description: string;
+  howToFix: string;
+  examples: Array<{
+    url: string;
+    detail?: string;
+  }>;
+}
+
+interface StructuredReport {
+  overview: {
+    totalPages: number;
+    totalIssues: number;
+    criticalIssues: number;
+    highPriorityIssues: number;
+    mediumPriorityIssues: number;
+    lowPriorityIssues: number;
+  };
+  issues: StructuredIssue[];
+  executionTime: number;
+}
+
+export class StructuredReportFormatter {
+  
+  /**
+   * Convert internal SEO issues to structured format
+   */
+  static formatReport(
+    issues: SEOIssue[],
+    totalPages: number,
+    executionTime: number
+  ): StructuredReport {
+    
+    const formattedIssues = issues.map(issue => 
+      this.formatIssue(issue, totalPages)
+    );
+    
+    // Sort by priority (Critical > High > Medium > Low)
+    const priorityOrder = { Critical: 0, High: 1, Medium: 2, Low: 3 };
+    formattedIssues.sort((a, b) => 
+      priorityOrder[a.issuePriority] - priorityOrder[b.issuePriority]
+    );
+    
+    const overview = {
+      totalPages,
+      totalIssues: formattedIssues.length,
+      criticalIssues: formattedIssues.filter(i => i.issuePriority === 'Critical').length,
+      highPriorityIssues: formattedIssues.filter(i => i.issuePriority === 'High').length,
+      mediumPriorityIssues: formattedIssues.filter(i => i.issuePriority === 'Medium').length,
+      lowPriorityIssues: formattedIssues.filter(i => i.issuePriority === 'Low').length
+    };
+    
+    return {
+      overview,
+      issues: formattedIssues,
+      executionTime
+    };
+  }
+  
+  /**
+   * Format individual issue to structured style
+   */
+  private static formatIssue(
+    issue: SEOIssue,
+    totalPages: number
+  ): StructuredIssue {
+    
+    return {
+      issueName: this.formatIssueName(issue),
+      issueType: this.getIssueType(issue.category, issue.priority),
+      issuePriority: this.normalizePriority(issue.priority),
+      urls: issue.affectedCount,
+      percentOfTotal: parseFloat(((issue.affectedCount / totalPages) * 100).toFixed(2)),
+      description: issue.description,
+      howToFix: issue.fix,
+      examples: issue.examples
+    };
+  }
+  
+  /**
+   * Convert query name to user-friendly issue name
+   */
+  private static formatIssueName(issue: SEOIssue): string {
+    const nameMap: Record<string, string> = {
+      'missing-titles': 'Page Titles: Missing',
+      'broken-internal-links': 'Links: Broken Internal Links',
+      'server-errors': 'Response Codes: Internal Server Error (5xx)',
+      '404-errors': 'Response Codes: Internal Client Error (4xx)',
+      'duplicate-titles': 'Page Titles: Duplicate',
+      'duplicate-meta-descriptions': 'Meta Description: Duplicate',
+      'missing-meta-descriptions': 'Meta Description: Missing',
+      'thin-content': 'Content: Low Content Pages',
+      'missing-h1': 'H1: Missing',
+      'multiple-h1': 'H1: Multiple',
+      'duplicate-h1': 'H1: Duplicate',
+      'redirects': 'Response Codes: Internal Redirection (3xx)',
+      'orphan-pages': 'Links: Orphan Pages',
+      'canonical-issues': 'Canonical: Issues',
+      'non-https': 'Security: Non-HTTPS URLs',
+      'heading-hierarchy-issues': 'Heading Hierarchy: Issues',
+      'missing-hsts': 'Security: Missing HSTS Header',
+      'missing-csp': 'Security: Missing Content-Security-Policy Header',
+      'missing-x-frame-options': 'Security: Missing X-Frame-Options Header',
+      'missing-referrer-policy': 'Security: Missing Secure Referrer-Policy Header',
+      'unsafe-external-links': 'Security: Unsafe Cross-Origin Links',
+      'protocol-relative-links': 'Security: Protocol-Relative Resource Links',
+      'title-length': 'Page Titles: Length Issues',
+      'meta-description-length': 'Meta Description: Length Issues',
+      'title-equals-h1': 'Page Titles: Same as H1',
+      'no-outbound-links': 'Links: Pages With No Outbound Links',
+      'high-external-links': 'Links: Pages With High External Outlinks',
+      'missing-images': 'Images: Missing On Content Pages'
+    };
+    
+    return nameMap[issue.query] || issue.query;
+  }
+  
+  /**
+   * Determine issue type based on category and priority
+   */
+  private static getIssueType(
+    category: string,
+    priority: string
+  ): 'Issue' | 'Warning' | 'Opportunity' {
+    
+    // Critical/High priority = Issue
+    if (priority === 'CRITICAL' || priority === 'HIGH') {
+      return 'Issue';
+    }
+    
+    // Opportunities category = Opportunity
+    if (category === 'opportunities') {
+      return 'Opportunity';
+    }
+    
+    // Everything else = Warning
+    return 'Warning';
+  }
+  
+  /**
+   * Normalize priority to title case
+   */
+  private static normalizePriority(priority: string): 'Critical' | 'High' | 'Medium' | 'Low' {
+    const normalized = priority.charAt(0).toUpperCase() + priority.slice(1).toLowerCase();
+    return normalized as 'Critical' | 'High' | 'Medium' | 'Low';
+  }
+  
+  /**
+   * Generate plain text summary for terminal output
+   */
+  static generateTextSummary(report: StructuredReport): string {
+    const lines: string[] = [];
+    
+    lines.push('═══════════════════════════════════════════════════════');
+    lines.push('              SEO AUDIT SUMMARY');
+    lines.push('═══════════════════════════════════════════════════════');
+    lines.push('');
+    lines.push(`Total Pages Crawled:     ${report.overview.totalPages.toLocaleString()}`);
+    lines.push(`Total Issues Found:      ${report.overview.totalIssues}`);
+    lines.push('');
+    lines.push('Issues by Priority:');
+    lines.push(`  Critical:              ${report.overview.criticalIssues}`);
+    lines.push(`  High:                  ${report.overview.highPriorityIssues}`);
+    lines.push(`  Medium:                ${report.overview.mediumPriorityIssues}`);
+    lines.push(`  Low:                   ${report.overview.lowPriorityIssues}`);
+    lines.push('');
+    lines.push(`Analysis Time:           ${report.executionTime}ms`);
+    lines.push('═══════════════════════════════════════════════════════');
+    lines.push('');
+    
+    // Group by priority
+    const critical = report.issues.filter(i => i.issuePriority === 'Critical');
+    const high = report.issues.filter(i => i.issuePriority === 'High');
+    const medium = report.issues.filter(i => i.issuePriority === 'Medium');
+    const low = report.issues.filter(i => i.issuePriority === 'Low');
+    
+    if (critical.length > 0) {
+      lines.push('CRITICAL ISSUES (Fix Immediately)');
+      lines.push('───────────────────────────────────────────────────────');
+      critical.forEach(issue => {
+        lines.push('');
+        lines.push(`${issue.issueName}`);
+        lines.push(`  URLs Affected:     ${issue.urls} (${issue.percentOfTotal}%)`);
+        lines.push(`  Type:              ${issue.issueType}`);
+        lines.push(`  Description:       ${issue.description}`);
+        lines.push(`  How To Fix:        ${issue.howToFix}`);
+      });
+      lines.push('');
+    }
+    
+    if (high.length > 0) {
+      lines.push('HIGH PRIORITY ISSUES');
+      lines.push('───────────────────────────────────────────────────────');
+      high.forEach(issue => {
+        lines.push('');
+        lines.push(`${issue.issueName}`);
+        lines.push(`  URLs Affected:     ${issue.urls} (${issue.percentOfTotal}%)`);
+        lines.push(`  Type:              ${issue.issueType}`);
+        lines.push(`  Description:       ${issue.description}`);
+        lines.push(`  How To Fix:        ${issue.howToFix}`);
+      });
+      lines.push('');
+    }
+    
+    if (medium.length > 0) {
+      lines.push('MEDIUM PRIORITY ISSUES');
+      lines.push('───────────────────────────────────────────────────────');
+      medium.forEach(issue => {
+        lines.push(`${issue.issueName} - ${issue.urls} URLs (${issue.percentOfTotal}%)`);
+      });
+      lines.push('');
+    }
+    
+    if (low.length > 0) {
+      lines.push('LOW PRIORITY / OPPORTUNITIES');
+      lines.push('───────────────────────────────────────────────────────');
+      low.forEach(issue => {
+        lines.push(`${issue.issueName} - ${issue.urls} URLs (${issue.percentOfTotal}%)`);
+      });
+      lines.push('');
+    }
+    
+    return lines.join('\n');
+  }
+}

package/src/index.ts

@@ -0,0 +1,259 @@
+#!/usr/bin/env node
+
+// Copyright 2026 Richard Baxter
+// Licensed under the Apache License, Version 2.0
+
+// CRITICAL: Disable ALL Crawlee logging for MCP stdio compatibility
+process.env.CRAWLEE_LOG_LEVEL = 'OFF';
+
+/**
+ * Crawlee MCP Server v2
+ * 
+ * Professional website crawler and SEO analyzer
+ * Built with @modelcontextprotocol/sdk and Crawlee
+ * 
+ * Phase 1: MCP server skeleton with tool registration ✅
+ * Phase 2: Full crawling engine implementation ✅
+ * Phase 3: SEO analysis layer ✅
+ * Phase 4: Fixed RequestQueue persistence bug ✅
+ */
+
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import {
+  CallToolRequestSchema,
+  ListToolsRequestSchema,
+  Tool
+} from '@modelcontextprotocol/sdk/types.js';
+
+import { runSeoAudit } from './tools/run-seo-audit.js';
+import { analyzeSeo } from './tools/analyze-seo.js';
+import { querySeoData } from './tools/query-seo-data.js';
+import { listQueries } from './tools/list-queries.js';
+
+const SERVER_NAME = 'seo-crawler-mcp';
+const SERVER_VERSION = '2.0.1'; // Version bump for storage cleanup fix
+
+const tools: Tool[] = [
+  {
+    name: 'run_seo_audit',
+    description: 'Crawl a website and extract comprehensive SEO data using Crawlee HttpCrawler. Returns crawl ID and output path.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        url: {
+          type: 'string',
+          description: 'Starting URL to crawl (must include http:// or https://)'
+        },
+        maxPages: {
+          type: 'number',
+          description: 'Maximum number of pages to crawl (1-10000). Default: 1000',
+          minimum: 1,
+          maximum: 10000
+        },
+        depth: {
+          type: 'number',
+          description: 'Maximum crawl depth (1-10). Default: 3',
+          minimum: 1,
+          maximum: 10
+        },
+        userAgent: {
+          type: 'string',
+          enum: ['chrome', 'googlebot'],
+          description: 'User agent to identify as: "chrome" (default, Chrome browser) or "googlebot" (Googlebot crawler). Default: chrome'
+        }
+      },
+      required: ['url']
+    }
+  },
+  {
+    name: 'analyze_seo',
+    description: 'Analyze SEO data from a completed crawl. Runs 25+ SQL queries to detect critical issues, content problems, technical SEO issues, security vulnerabilities, and optimization opportunities. Returns structured report with affected URLs and fix recommendations.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        crawlPath: {
+          type: 'string',
+          description: 'Path to crawl output directory (e.g., C:/seo-audits/example.com_2026-02-01_abc123)'
+        },
+        includeCategories: {
+          type: 'array',
+          items: {
+            type: 'string',
+            enum: ['critical', 'content', 'technical', 'security', 'opportunities']
+          },
+          description: 'Optional: Filter analysis by categories. Default: all categories'
+        },
+        maxExamplesPerIssue: {
+          type: 'number',
+          description: 'Maximum example URLs to return per issue. Default: 10',
+          minimum: 1,
+          maximum: 100
+        },
+        format: {
+          type: 'string',
+          enum: ['detailed', 'summary', 'structured'],
+          description: 'Output format: "structured" (organized format, default), "summary" (text overview), "detailed" (full JSON). Default: structured'
+        }
+      },
+      required: ['crawlPath']
+    }
+  },
+  {
+    name: 'query_seo_data',
+    description: 'Execute a specific SEO analysis query by name. Use list_seo_queries to see available queries. Returns detailed results with affected URLs and context.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        crawlPath: {
+          type: 'string',
+          description: 'Path to crawl output directory'
+        },
+        query: {
+          type: 'string',
+          description: 'Query name (e.g., "missing-titles", "duplicate-h1", "orphan-pages"). Use list_seo_queries to see all available queries.'
+        },
+        limit: {
+          type: 'number',
+          description: 'Optional: Maximum number of results to return. Default: 100',
+          minimum: 1,
+          maximum: 1000
+        }
+      },
+      required: ['crawlPath', 'query']
+    }
+  },
+  {
+    name: 'list_seo_queries',
+    description: 'List all available SEO analysis queries with descriptions, priorities, and fix recommendations. Optionally filter by category or priority level.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        category: {
+          type: 'string',
+          enum: ['critical', 'content', 'technical', 'security', 'opportunities'],
+          description: 'Optional: Filter by category'
+        },
+        priority: {
+          type: 'string',
+          enum: ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW'],
+          description: 'Optional: Filter by priority level'
+        }
+      }
+    }
+  }
+];
+
+class CrawleeMcpServer {
+  private server: Server;
+  
+  constructor() {
+    this.server = new Server(
+      {
+        name: SERVER_NAME,
+        version: SERVER_VERSION
+      },
+      {
+        capabilities: {
+          tools: {}
+        }
+      }
+    );
+    
+    this.setupHandlers();
+    this.setupErrorHandling();
+  }
+  
+  private setupHandlers(): void {
+    this.server.setRequestHandler(ListToolsRequestSchema, async () => ({
+      tools
+    }));
+    
+    this.server.setRequestHandler(CallToolRequestSchema, async (request) => {
+      const { name, arguments: args } = request.params;
+      
+      try {
+        let result: any;
+        
+        switch (name) {
+          case 'run_seo_audit':
+            result = await runSeoAudit(args as any);
+            break;
+            
+          case 'analyze_seo':
+            result = await analyzeSeo(args as any);
+            break;
+            
+          case 'query_seo_data':
+            result = await querySeoData(args as any);
+            break;
+            
+          case 'list_seo_queries':
+            result = await listQueries(args as any);
+            break;
+            
+          default:
+            return {
+              content: [
+                {
+                  type: 'text',
+                  text: `Unknown tool: ${name}`
+                }
+              ],
+              isError: true
+            };
+        }
+        
+        return {
+          content: [
+            {
+              type: 'text',
+              text: JSON.stringify(result, null, 2)
+            }
+          ]
+        };
+        
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+        
+        return {
+          content: [
+            {
+              type: 'text',
+              text: JSON.stringify({
+                error: errorMessage,
+                tool: name
+              }, null, 2)
+            }
+          ],
+          isError: true
+        };
+      }
+    });
+  }
+  
+  private setupErrorHandling(): void {
+    this.server.onerror = (error) => {
+      console.error('[MCP Error]', error);
+    };
+    
+    process.on('SIGINT', async () => {
+      await this.server.close();
+      process.exit(0);
+    });
+  }
+  
+  async run(): Promise<void> {
+    const transport = new StdioServerTransport();
+    await this.server.connect(transport);
+    
+    console.error(`${SERVER_NAME} v${SERVER_VERSION} running on stdio`);
+    console.error('✅ Phase 1: MCP server active');
+    console.error('✅ Phase 2: Crawling engine ready');
+    console.error('✅ Phase 3: SEO analysis layer active');
+    console.error('✅ Phase 4: RequestQueue bug fixed');
+  }
+}
+
+const server = new CrawleeMcpServer();
+server.run().catch(console.error);