@houtini/seo-crawler-mcp 2.1.2 → 2.1.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@houtini/seo-crawler-mcp",
-  "version": "2.1.2",
+  "version": "2.1.3",
   "mcpName": "io.github.houtini-ai/seo-crawler-mcp",
   "description": "Crawl and analyse websites for SEO errors and issues using Crawlee with SQLite storage",
   "type": "module",
@@ -23,6 +23,14 @@
     "audit",
     "model-context-protocol"
   ],
+  "files": [
+    "build",
+    "README.md",
+    "LICENSE",
+    "NOTICE",
+    "houtini-logo.jpg",
+    "server.json"
+  ],
   "engines": {
     "node": ">=18.0.0"
   },

package/server.json

@@ -6,12 +6,12 @@
     "url": "https://github.com/houtini-ai/seo-crawler-mcp",
     "source": "github"
   },
-  "version": "2.0.1",
+  "version": "2.1.3",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "@houtini/seo-crawler-mcp",
-      "version": "2.0.1",
+      "version": "2.1.3",
       "transport": {
         "type": "stdio"
       },

package/.github/workflows/ci.yml

@@ -1,59 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        node-version: [18.x, 20.x, 22.x]
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    - name: Setup Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v4
-      with:
-        node-version: ${{ matrix.node-version }}
-        cache: 'npm'
-
-    - name: Install dependencies
-      run: npm ci
-
-    - name: Run TypeScript build
-      run: npm run build
-
-    - name: Run tests
-      run: npm test
-      continue-on-error: true
-
-    - name: Check for TypeScript errors
-      run: npx tsc --noEmit
-
-  lint:
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-
-    - name: Setup Node.js
-      uses: actions/setup-node@v4
-      with:
-        node-version: '20.x'
-        cache: 'npm'
-
-    - name: Install dependencies
-      run: npm ci
-
-    - name: Check formatting
-      run: |
-        echo "Checking for common code issues..."
-        npx tsc --noEmit || echo "TypeScript check completed"

package/src/analyzers/QueryLoader.ts

@@ -1,175 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-export interface QueryMetadata {
-  name: string;
-  category: 'critical' | 'content' | 'technical' | 'security' | 'opportunities';
-  priority: 'CRITICAL' | 'HIGH' | 'MEDIUM' | 'LOW';
-  description: string;
-  impact: string;
-  fix: string;
-  sql: string;
-}
-
-function findQueriesPath(): string {
-  // Try multiple possible locations
-  const candidates = [
-    // When running from compiled build/analyzers/QueryLoader.js
-    path.join(__dirname, 'queries'),
-    // When running from src/analyzers/QueryLoader.ts
-    path.join(__dirname, '..', '..', 'src', 'analyzers', 'queries'),
-    // When running from project root
-    path.join(process.cwd(), 'src', 'analyzers', 'queries'),
-    // When running tests from project root
-    path.join(process.cwd(), 'build', 'analyzers', 'queries'),
-  ];
-
-  for (const candidate of candidates) {
-    if (fs.existsSync(candidate)) {
-      return candidate;
-    }
-  }
-
-  throw new Error(`Could not find queries directory. Tried: ${candidates.join(', ')}`);
-}
-
-export class QueryLoader {
-  private queriesPath: string;
-  private queries: Map<string, QueryMetadata>;
-
-  constructor() {
-    this.queriesPath = findQueriesPath();
-    this.queries = new Map();
-    this.loadAllQueries();
-  }
-
-  private loadAllQueries(): void {
-    const categories = ['critical', 'content', 'technical', 'security', 'opportunities'];
-
-    for (const category of categories) {
-      const categoryPath = path.join(this.queriesPath, category);
-      
-      if (!fs.existsSync(categoryPath)) {
-        continue;
-      }
-
-      const files = fs.readdirSync(categoryPath).filter(f => f.endsWith('.sql'));
-
-      for (const file of files) {
-        const filePath = path.join(categoryPath, file);
-        const queryName = file.replace('.sql', '');
-        const metadata = this.parseQueryFile(filePath, queryName, category as any);
-        
-        this.queries.set(queryName, metadata);
-      }
-    }
-  }
-
-  private parseQueryFile(
-    filePath: string, 
-    name: string, 
-    category: QueryMetadata['category']
-  ): QueryMetadata {
-    const content = fs.readFileSync(filePath, 'utf-8');
-    const lines = content.split('\n');
-
-    let description = '';
-    let priority: QueryMetadata['priority'] = 'MEDIUM';
-    let impact = '';
-    let fix = '';
-    const sqlLines: string[] = [];
-    let inSQL = false;
-
-    for (const line of lines) {
-      const trimmed = line.trim();
-
-      if (trimmed.startsWith('--')) {
-        const comment = trimmed.substring(2).trim();
-        
-        if (comment.startsWith('Priority:')) {
-          priority = comment.replace('Priority:', '').trim() as QueryMetadata['priority'];
-        } else if (comment.startsWith('Impact:')) {
-          impact = comment.replace('Impact:', '').trim();
-        } else if (comment.startsWith('Fix:')) {
-          fix = comment.replace('Fix:', '').trim();
-        } else if (comment && !comment.startsWith('Category:')) {
-          if (!description) {
-            description = comment;
-          }
-        }
-      } else if (trimmed.startsWith('SELECT')) {
-        inSQL = true;
-      }
-
-      if (inSQL) {
-        sqlLines.push(line);
-      }
-    }
-
-    return {
-      name,
-      category,
-      priority,
-      description,
-      impact,
-      fix,
-      sql: sqlLines.join('\n').trim()
-    };
-  }
-
-  getQuery(name: string): QueryMetadata | undefined {
-    return this.queries.get(name);
-  }
-
-  getAllQueries(): QueryMetadata[] {
-    return Array.from(this.queries.values());
-  }
-
-  getQueriesByCategory(category: QueryMetadata['category']): QueryMetadata[] {
-    return this.getAllQueries().filter(q => q.category === category);
-  }
-
-  getQueriesByPriority(priority: QueryMetadata['priority']): QueryMetadata[] {
-    return this.getAllQueries().filter(q => q.priority === priority);
-  }
-
-  getCriticalQueries(): QueryMetadata[] {
-    return this.getQueriesByPriority('CRITICAL');
-  }
-
-  getHighPriorityQueries(): QueryMetadata[] {
-    return this.getQueriesByPriority('HIGH');
-  }
-
-  listQueryNames(): string[] {
-    return Array.from(this.queries.keys()).sort();
-  }
-
-  getQueryStats(): {
-    total: number;
-    byCategory: Record<string, number>;
-    byPriority: Record<string, number>;
-  } {
-    const queries = this.getAllQueries();
-    
-    const byCategory: Record<string, number> = {};
-    const byPriority: Record<string, number> = {};
-
-    for (const query of queries) {
-      byCategory[query.category] = (byCategory[query.category] || 0) + 1;
-      byPriority[query.priority] = (byPriority[query.priority] || 0) + 1;
-    }
-
-    return {
-      total: queries.length,
-      byCategory,
-      byPriority
-    };
-  }
-}
-
-export const queryLoader = new QueryLoader();

package/src/analyzers/queries/README.md

@@ -1,228 +0,0 @@
-# SEO Analysis Query Library
-
-**Version:** 1.0.0  
-**Last Updated:** 2026-02-01  
-**Coverage:** 25 detectable SEO issues
-
-## Query Organization
-
-All queries follow a standard format:
-- SQL comments describing the issue
-- Priority level (CRITICAL, HIGH, MEDIUM, LOW)
-- Category classification
-- Optimized SELECT statements with ORDER BY and LIMIT
-- Results limited to 100 rows for performance
-
-## Critical Issues (4 queries)
-
-**Indexability issues that must be fixed immediately**
-
-1. **missing-titles.sql** - Pages without title tags
-   - Priority: CRITICAL
-   - Impact: Major indexability problem
-   - Fix: Add unique, descriptive title tags
-
-2. **broken-internal-links.sql** - Internal links to 404/5xx pages
-   - Priority: CRITICAL
-   - Impact: Poor user experience, crawl budget waste
-   - Fix: Update or remove broken links
-
-3. **server-errors.sql** - Pages returning 5xx errors
-   - Priority: CRITICAL
-   - Impact: Prevents indexing
-   - Fix: Debug server issues immediately
-
-4. **404-errors.sql** - Pages not found
-   - Priority: CRITICAL
-   - Impact: Dead ends for users and crawlers
-   - Fix: Redirect to relevant pages or restore content
-
-## Content Quality Issues (7 queries)
-
-**Problems with page content and metadata**
-
-5. **duplicate-titles.sql** - Multiple pages with same title
-   - Priority: HIGH
-   - Impact: Cannibalization, poor CTR
-   - Fix: Create unique titles for each page
-
-6. **duplicate-meta-descriptions.sql** - Duplicate meta descriptions
-   - Priority: MEDIUM
-   - Impact: Reduced CTR, missed opportunities
-   - Fix: Write unique descriptions
-
-7. **missing-meta-descriptions.sql** - Pages without descriptions
-   - Priority: MEDIUM
-   - Impact: Search engines auto-generate poor snippets
-   - Fix: Add compelling meta descriptions
-
-8. **thin-content.sql** - Pages with < 300 words
-   - Priority: MEDIUM
-   - Impact: Low quality signals
-   - Fix: Expand content or consolidate pages
-
-9. **missing-h1.sql** - Pages without H1 tags
-   - Priority: HIGH
-   - Impact: Unclear page topic
-   - Fix: Add descriptive H1 tags
-
-10. **multiple-h1.sql** - Pages with multiple H1 tags
-    - Priority: MEDIUM
-    - Impact: Diluted topical focus
-    - Fix: Use single H1 per page
-
-11. **duplicate-h1.sql** - Multiple pages with same H1
-    - Priority: MEDIUM
-    - Impact: Content cannibalization
-    - Fix: Differentiate H1 tags
-
-## Technical SEO Issues (5 queries)
-
-**Infrastructure and architecture problems**
-
-12. **redirects.sql** - Pages with 3xx redirect status
-    - Priority: MEDIUM
-    - Impact: Crawl budget waste, slow page speed
-    - Fix: Update links to final destinations
-
-13. **orphan-pages.sql** - Pages with no internal links
-    - Priority: MEDIUM
-    - Impact: Difficult to discover and crawl
-    - Fix: Add internal links from related pages
-
-14. **canonical-issues.sql** - Canonical URL differs from actual URL
-    - Priority: MEDIUM
-    - Impact: Duplicate content confusion
-    - Fix: Review canonical implementation
-
-15. **non-https.sql** - Pages not using HTTPS
-    - Priority: MEDIUM
-    - Impact: Security warnings, ranking penalty
-    - Fix: Migrate to HTTPS
-
-16. **heading-hierarchy-issues.sql** - Non-sequential headings
-    - Priority: MEDIUM
-    - Impact: Poor document structure
-    - Fix: Correct heading order (h1 → h2 → h3)
-
-## Security Issues (6 queries)
-
-**Security header and link security problems**
-
-17. **missing-hsts.sql** - No Strict-Transport-Security header
-    - Priority: HIGH
-    - Impact: HTTPS downgrade attacks possible
-    - Fix: Add HSTS header to server config
-
-18. **missing-csp.sql** - No Content-Security-Policy header
-    - Priority: MEDIUM
-    - Impact: XSS vulnerability
-    - Fix: Implement CSP header
-
-19. **missing-x-frame-options.sql** - No X-Frame-Options header
-    - Priority: MEDIUM
-    - Impact: Clickjacking vulnerability
-    - Fix: Add X-Frame-Options: DENY
-
-20. **missing-referrer-policy.sql** - No Referrer-Policy header
-    - Priority: LOW
-    - Impact: Privacy leaks
-    - Fix: Add Referrer-Policy header
-
-21. **unsafe-external-links.sql** - target="_blank" without rel="noopener"
-    - Priority: MEDIUM
-    - Impact: Tabnabbing vulnerability
-    - Fix: Add rel="noopener noreferrer"
-
-22. **protocol-relative-links.sql** - Links using //example.com format
-    - Priority: LOW
-    - Impact: Mixed content warnings
-    - Fix: Use absolute HTTPS URLs
-
-## Optimization Opportunities (6 queries)
-
-**Enhancement opportunities for better SEO**
-
-23. **title-length.sql** - Titles too short (< 30) or too long (> 60)
-    - Priority: MEDIUM
-    - Impact: Truncated or poor SERP display
-    - Fix: Optimize title length to 30-60 characters
-
-24. **meta-description-length.sql** - Descriptions too short/long
-    - Priority: LOW
-    - Impact: Suboptimal SERP snippets
-    - Fix: Optimize to 120-160 characters
-
-25. **title-equals-h1.sql** - Title and H1 are identical
-    - Priority: LOW
-    - Impact: Missed keyword opportunity
-    - Fix: Differentiate title and H1 slightly
-
-26. **no-outbound-links.sql** - Pages with no links
-    - Priority: LOW
-    - Impact: Poor user experience, low PageRank flow
-    - Fix: Add relevant internal/external links
-
-27. **high-external-links.sql** - Pages with > 20 external links
-    - Priority: LOW
-    - Impact: Excessive PageRank dilution
-    - Fix: Review and reduce external links
-
-28. **missing-images.sql** - Content pages without images
-    - Priority: LOW
-    - Impact: Poor engagement, no image search visibility
-    - Fix: Add relevant images with alt text
-
-## Query Performance
-
-All queries are optimized for SQLite with:
-- Indexed columns (url, status_code, depth)
-- LIMIT clauses to prevent excessive results
-- Strategic WHERE clauses to filter early
-- Simple JOINs where necessary
-
-**Expected Performance:**
-- Simple queries: < 10ms
-- Complex queries (duplicates, orphans): < 100ms
-- Join queries (broken links): < 200ms
-
-## Usage Patterns
-
-### Direct SQL Execution
-```typescript
-import Database from 'better-sqlite3';
-import fs from 'fs';
-
-const db = new Database('./crawl-data.db');
-const query = fs.readFileSync('./queries/critical/missing-titles.sql', 'utf-8');
-const results = db.prepare(query).all();
-```
-
-### Programmatic Analysis
-```typescript
-import { SQLAnalyzer } from './SQLAnalyzer.js';
-
-const analyzer = new SQLAnalyzer(crawlId);
-const report = await analyzer.generateReport();
-// Returns structured SEOAnalysisReport
-```
-
-### MCP Tool Integration
-```bash
-seo-crawler-mcp:analyze_seo crawlId="431841d4"
-# Returns JSON report with all 25 issues checked
-```
-
-## Future Enhancements
-
-**Not yet implemented (requires additional data capture):**
-- Core Web Vitals analysis (requires Playwright)
-- Robots.txt validation (requires separate parser)
-- Readability scoring (requires text analysis library)
-- Mobile rendering issues (requires device emulation)
-
----
-
-**Query Coverage: 25 Production-Ready SEO Checks**  
-**Status: Production Ready**  
-**Next Step: Build SQLAnalyzer.ts class**

package/src/analyzers/queries/content/duplicate-h1.sql

@@ -1,18 +0,0 @@
--- Duplicate H1 Tags (MEDIUM)
--- Multiple pages sharing the same H1 tag
--- Priority: MEDIUM
--- Category: content
-
-SELECT 
-  h1,
-  COUNT(*) as page_count,
-  GROUP_CONCAT(url, '|||') as urls,
-  MIN(title) as example_title
-FROM pages
-WHERE h1 IS NOT NULL 
-  AND TRIM(h1) != ''
-  AND status_code = 200
-GROUP BY h1
-HAVING COUNT(*) > 1
-ORDER BY page_count DESC
-LIMIT 50;

package/src/analyzers/queries/content/duplicate-meta-descriptions.sql

@@ -1,18 +0,0 @@
--- Duplicate Meta Descriptions (MEDIUM)
--- Multiple pages sharing the same meta description
--- Priority: MEDIUM
--- Category: content
-
-SELECT 
-  meta_description,
-  COUNT(*) as page_count,
-  GROUP_CONCAT(url, '|||') as urls,
-  MIN(title) as example_title
-FROM pages
-WHERE meta_description IS NOT NULL 
-  AND TRIM(meta_description) != ''
-  AND status_code = 200
-GROUP BY meta_description
-HAVING COUNT(*) > 1
-ORDER BY page_count DESC
-LIMIT 50;

package/src/analyzers/queries/content/duplicate-titles.sql

@@ -1,19 +0,0 @@
--- Duplicate Title Tags (HIGH)
--- Multiple pages sharing the same title tag
--- Priority: HIGH
--- Category: content
--- Impact: Title duplication causes keyword cannibalization where pages compete against each other. Reduces click-through rates as users can't distinguish between pages in search results.
--- Fix: Create unique, descriptive title tags for each page that accurately reflect the page's specific content and target different keyword variations.
-
-SELECT 
-  title,
-  GROUP_CONCAT(url, ', ') as duplicate_urls,
-  COUNT(*) as count
-FROM pages
-WHERE title IS NOT NULL 
-  AND title != ''
-  AND status_code = 200
-GROUP BY title
-HAVING count > 1
-ORDER BY count DESC
-LIMIT 100;

package/src/analyzers/queries/content/missing-h1.sql

@@ -1,18 +0,0 @@
--- Missing H1 Tags (HIGH)
--- Pages without H1 headings
--- Priority: HIGH
--- Category: content
--- Impact: H1 tags are critical for SEO and accessibility. Their absence makes it unclear to search engines and users what the page is about. Impacts topic clarity and ranking potential.
--- Fix: Ensure every important page has exactly one descriptive H1 tag that clearly indicates the main topic. Include primary keywords naturally without keyword stuffing.
-
-SELECT 
-  url,
-  title,
-  word_count,
-  depth
-FROM pages
-WHERE (h1 IS NULL OR h1 = '' OR TRIM(h1) = '')
-  AND status_code = 200
-  AND content_type LIKE '%text/html%'
-ORDER BY word_count DESC
-LIMIT 100;

package/src/analyzers/queries/content/missing-meta-descriptions.sql

@@ -1,19 +0,0 @@
--- Missing Meta Descriptions (MEDIUM)
--- Pages without meta descriptions
--- Priority: MEDIUM
--- Category: content
--- Impact: Search engines auto-generate snippets which are often poor quality and don't entice clicks. Missed opportunity to control your SERP messaging and improve click-through rates.
--- Fix: Write unique, compelling meta descriptions (120-160 characters) for key pages. Focus on benefits and include a call-to-action. Avoid duplicating title tag content.
-
-SELECT 
-  url,
-  title,
-  word_count,
-  depth
-FROM pages
-WHERE (meta_description IS NULL OR meta_description = '' OR TRIM(meta_description) = '')
-  AND status_code = 200
-  AND content_type LIKE '%text/html%'
-  AND depth <= 5
-ORDER BY word_count DESC
-LIMIT 100;

package/src/analyzers/queries/content/multiple-h1.sql

@@ -1,17 +0,0 @@
--- Multiple H1 Tags (MEDIUM)
--- Pages with more than one H1 tag (can confuse search engines)
--- Priority: MEDIUM
--- Category: content
-
-SELECT 
-  url,
-  title,
-  heading_count_h1,
-  h1,
-  status_code,
-  depth
-FROM pages
-WHERE heading_count_h1 > 1
-  AND status_code = 200
-ORDER BY heading_count_h1 DESC, depth ASC
-LIMIT 100;

package/src/analyzers/queries/content/thin-content.sql

@@ -1,18 +0,0 @@
--- Thin Content (MEDIUM)
--- Pages with insufficient content (< 300 words)
--- Priority: MEDIUM
--- Category: content
-
-SELECT 
-  url,
-  title,
-  word_count,
-  status_code,
-  depth,
-  internal_links
-FROM pages
-WHERE word_count < 300
-  AND status_code = 200
-  AND content_type LIKE '%text/html%'
-ORDER BY word_count ASC
-LIMIT 100;

package/src/analyzers/queries/critical/404-errors.sql

@@ -1,14 +0,0 @@
--- 404 Not Found Errors (CRITICAL)
--- Pages returning 404 errors
--- Priority: CRITICAL
--- Category: indexability
-
-SELECT 
-  url,
-  depth,
-  linked_from,
-  crawled_at
-FROM pages
-WHERE status_code = 404
-ORDER BY depth ASC
-LIMIT 100;

package/src/analyzers/queries/critical/broken-internal-links.sql

@@ -1,22 +0,0 @@
--- Broken Internal Links (CRITICAL)
--- Internal links pointing to pages that actually returned HTTP errors (404, 500, etc.)
--- Priority: CRITICAL
--- Category: critical
--- Impact: Poor user experience, wasted crawl budget, and loss of link equity. Users encounter dead ends and search engines waste resources crawling broken links.
--- Fix: View URLs that link to errors using the 'inlinks' tab and export them in bulk. Update broken links to point to correct URLs or remove them. Consider implementing 301 redirects for permanently moved content.
--- Note: This query only reports links to pages that were crawled AND returned errors. Un-crawled links are reported separately in the opportunities category.
-
-SELECT 
-  l.source_url,
-  l.target_url,
-  l.anchor_text,
-  l.placement,
-  p.status_code,
-  COUNT(*) as occurrences
-FROM links l
-INNER JOIN pages p ON l.target_url = p.url  -- Only check pages we actually crawled
-WHERE l.is_internal = 1 
-  AND p.status_code >= 400  -- Only actual HTTP errors (404, 500, etc.)
-GROUP BY l.target_url, l.source_url, p.status_code
-ORDER BY occurrences DESC
-LIMIT 100;

package/src/analyzers/queries/critical/missing-titles.sql

@@ -1,17 +0,0 @@
--- Missing Title Tags (CRITICAL)
--- Pages without title tags - catastrophic for SEO
--- Priority: CRITICAL
--- Category: critical
--- Impact: Pages are essentially invisible to search engines. Title tags are the most important on-page SEO element and their absence prevents proper indexing and ranking.
--- Fix: Add unique, descriptive title tags (50-60 characters) to all pages immediately. Include primary keywords and brand name where appropriate.
-
-SELECT 
-  url,
-  word_count,
-  heading_count_h1
-FROM pages
-WHERE (title IS NULL OR title = '' OR TRIM(title) = '')
-  AND status_code = 200
-  AND depth <= 5
-ORDER BY word_count DESC
-LIMIT 100;