bmad-plus 0.12.1 → 0.13.0

package/src/bmad-plus/packs/pack-seo/scripts/seo_crawl.py

@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+"""
+SEO Crawl — Recursive mini-crawler for site structure discovery.
+
+Features:
+- Sitemap.xml parsing for initial page list
+- Recursive link-following with configurable depth
+- Internal link graph construction
+- Orphan page detection
+- robots.txt respect
+
+Author: Laurent Rochetta
+License: MIT
+"""
+
+import argparse
+import json
+import os
+import re
+import sys
+
+# XXE protection: require defusedxml. We deliberately FAIL CLOSED rather
+# than fall back to the stdlib parser, which is vulnerable to entity-
+# expansion / external-entity (XXE) attacks on attacker-controlled XML.
+try:
+    from defusedxml import ElementTree as ET
+except ImportError:
+    print(
+        "Error: defusedxml library required for safe XML parsing. "
+        "Install: pip install defusedxml",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+# Reuse the hardened SSRF guard from seo_fetch (same package/directory).
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+try:
+    from seo_fetch import is_safe_url
+except ImportError:
+    print(
+        "Error: seo_fetch.is_safe_url is required (SSRF protection). "
+        "Ensure seo_fetch.py is present alongside seo_crawl.py.",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
+try:
+    from bs4 import BeautifulSoup
+    HAS_BS4 = True
+except ImportError:
+    HAS_BS4 = False
+
+from collections import defaultdict
+from typing import Optional, Set
+from urllib.parse import urljoin, urlparse
+
+try:
+    import requests
+except ImportError:
+    print("Error: requests library required. Install: pip install requests", file=sys.stderr)
+    sys.exit(1)
+
+USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 BMADSEOEngine/2.0"
+)
+
+
+class SEOCrawler:
+    """Recursive mini-crawler for SEO site structure analysis."""
+
+    def __init__(self, base_url: str, max_depth: int = 2, max_pages: int = 25, timeout: int = 15):
+        self.base_url = base_url.rstrip("/")
+        self.base_domain = urlparse(self.base_url).netloc
+        self.max_depth = max_depth
+        self.max_pages = max_pages
+        self.timeout = timeout
+
+        self.visited: Set[str] = set()
+        self.pages: list = []
+        self.link_graph: dict = defaultdict(set)  # page -> set of linked pages
+        self.sitemap_urls: list = []
+        self.robots_txt: Optional[str] = None
+        self.errors: list = []
+
+    def normalize_url(self, url: str) -> str:
+        """Normalize URL for deduplication."""
+        parsed = urlparse(url)
+        path = parsed.path.rstrip("/") or "/"
+        return f"{parsed.scheme}://{parsed.netloc}{path}"
+
+    def is_internal(self, url: str) -> bool:
+        """Check if URL belongs to the same domain."""
+        return urlparse(url).netloc == self.base_domain
+
+    def _safe_get(self, url: str, error_prefix: str = "Blocked"):
+        """GET a URL, following redirects manually with per-hop SSRF revalidation.
+
+        Redirects are followed with allow_redirects=False so is_safe_url() runs on
+        EVERY hop: a public URL that 302s to an internal/metadata endpoint
+        (redirect-based SSRF) is refused. Returns the final requests.Response, or
+        None (with an entry appended to self.errors) when a hop is unsafe or too
+        many redirects occur. Used by fetch(), fetch_robots_txt() and
+        parse_sitemap() so every network path shares the same guard.
+        """
+        if not is_safe_url(url):
+            self.errors.append(
+                {"url": url, "error": f"{error_prefix}: private/internal URL (SSRF protection)"}
+            )
+            return None
+        current_url = url
+        hops = 0
+        while True:
+            response = requests.get(
+                current_url,
+                headers={"User-Agent": USER_AGENT},
+                timeout=self.timeout,
+                allow_redirects=False,
+            )
+            if not response.is_redirect:
+                return response
+            location = response.headers.get("Location")
+            if not location:
+                return response
+            next_url = urljoin(current_url, location)
+            if urlparse(next_url).scheme not in ("http", "https") or not is_safe_url(next_url):
+                self.errors.append(
+                    {"url": next_url,
+                     "error": f"{error_prefix} redirect: non-HTTP(S) or private/internal URL (SSRF protection)"}
+                )
+                return None
+            hops += 1
+            if hops > 5:
+                self.errors.append({"url": current_url, "error": "Too many redirects (max 5)"})
+                return None
+            current_url = next_url
+
+    def fetch(self, url: str) -> Optional[str]:
+        """Fetch a page (HTML), following redirects with per-hop SSRF revalidation."""
+        try:
+            response = self._safe_get(url)
+            if response is None:
+                return None
+            if response.status_code == 200 and "text/html" in response.headers.get("content-type", ""):
+                return response.text
+            self.pages.append({
+                "url": url,
+                "status": response.status_code,
+                "content_type": response.headers.get("content-type", ""),
+                "title": None,
+                "word_count": 0,
+                "depth": -1,
+            })
+        except requests.RequestException as e:
+            self.errors.append({"url": url, "error": str(e)})
+        return None
+
+    def fetch_robots_txt(self):
+        """Fetch and store robots.txt (redirects revalidated per hop)."""
+        robots_url = f"{self.base_url}/robots.txt"
+        try:
+            response = self._safe_get(robots_url)
+            if response is not None and response.status_code == 200:
+                self.robots_txt = response.text
+        except requests.RequestException:
+            pass
+
+    def parse_sitemap(self):
+        """Discover pages from sitemap.xml (redirects revalidated per hop)."""
+        sitemap_url = f"{self.base_url}/sitemap.xml"
+
+        # Check robots.txt for sitemap reference
+        if self.robots_txt:
+            for line in self.robots_txt.splitlines():
+                if line.strip().lower().startswith("sitemap:"):
+                    sitemap_url = line.split(":", 1)[1].strip()
+                    break
+
+        # The sitemap URL can come from an attacker-controlled robots.txt
+        # ("Sitemap:" line) — _safe_get validates it AND every redirect hop.
+        try:
+            response = self._safe_get(sitemap_url, error_prefix="Blocked sitemap")
+            if response is not None and response.status_code == 200 and "xml" in response.headers.get("content-type", ""):
+                root = ET.fromstring(response.content)
+                ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}
+
+                for url_el in root.findall(".//sm:url/sm:loc", ns):
+                    if url_el.text and self.is_internal(url_el.text):
+                        self.sitemap_urls.append(url_el.text)
+
+                # Handle sitemap index
+                for sitemap_el in root.findall(".//sm:sitemap/sm:loc", ns):
+                    self.sitemap_urls.append(f"[sitemap-index]: {sitemap_el.text}")
+
+        except (requests.RequestException, ET.ParseError):
+            pass
+
+    def extract_links(self, html: str, page_url: str) -> list:
+        """Extract internal links from HTML using BeautifulSoup (fallback to regex)."""
+        links = []
+
+        if HAS_BS4:
+            soup = BeautifulSoup(html, "html.parser")
+            for a_tag in soup.find_all("a", href=True):
+                href = a_tag["href"]
+                if href.startswith("#") or href.startswith("javascript:") or href.startswith("mailto:"):
+                    continue
+                full_url = urljoin(page_url, href)
+                if self.is_internal(full_url):
+                    normalized = self.normalize_url(full_url)
+                    links.append(normalized)
+                    self.link_graph[page_url].add(normalized)
+        else:
+            # Fallback: regex (less robust, no BS4 dependency)
+            for match in re.finditer(r'href=["\']([^"\']+)["\']', html):
+                href = match.group(1)
+                if href.startswith("#") or href.startswith("javascript:") or href.startswith("mailto:"):
+                    continue
+                full_url = urljoin(page_url, href)
+                if self.is_internal(full_url):
+                    normalized = self.normalize_url(full_url)
+                    links.append(normalized)
+                    self.link_graph[page_url].add(normalized)
+
+        return links
+
+    def extract_title(self, html: str) -> Optional[str]:
+        """Extract title from HTML."""
+        match = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
+        return match.group(1).strip() if match else None
+
+    def count_words(self, html: str) -> int:
+        """Count visible words in HTML."""
+        text = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.IGNORECASE)
+        text = re.sub(r"<style[^>]*>.*?</style>", "", text, flags=re.DOTALL | re.IGNORECASE)
+        text = re.sub(r"<[^>]+>", " ", text)
+        words = re.findall(r"\b\w+\b", text)
+        return len(words)
+
+    def crawl(self):
+        """Execute the recursive crawl."""
+        self.fetch_robots_txt()
+        self.parse_sitemap()
+
+        # Start with base URL
+        queue = [(self.base_url, 0)]  # (url, depth)
+
+        while queue and len(self.visited) < self.max_pages:
+            url, depth = queue.pop(0)
+            normalized = self.normalize_url(url)
+
+            if normalized in self.visited:
+                continue
+            if depth > self.max_depth:
+                continue
+
+            self.visited.add(normalized)
+            html = self.fetch(normalized)
+
+            if html:
+                title = self.extract_title(html)
+                word_count = self.count_words(html)
+
+                self.pages.append({
+                    "url": normalized,
+                    "status": 200,
+                    "title": title,
+                    "word_count": word_count,
+                    "depth": depth,
+                })
+
+                # Discover links for next level
+                if depth < self.max_depth:
+                    links = self.extract_links(html, normalized)
+                    for link in links:
+                        if link not in self.visited:
+                            queue.append((link, depth + 1))
+
+    def get_results(self) -> dict:
+        """Return crawl results as dictionary."""
+        # Detect orphan pages (in sitemap but not linked from any crawled page)
+        all_linked = set()
+        for targets in self.link_graph.values():
+            all_linked.update(targets)
+
+        orphans = [url for url in self.sitemap_urls
+                    if isinstance(url, str) and not url.startswith("[") and
+                    self.normalize_url(url) not in all_linked]
+
+        return {
+            "base_url": self.base_url,
+            "pages_crawled": len(self.pages),
+            "max_depth": self.max_depth,
+            "sitemap_urls_found": len([u for u in self.sitemap_urls if not str(u).startswith("[")]),
+            "has_robots_txt": self.robots_txt is not None,
+            "has_sitemap": len(self.sitemap_urls) > 0,
+            "pages": self.pages,
+            "orphan_pages": orphans[:10],
+            "link_graph_summary": {
+                "total_internal_links": sum(len(v) for v in self.link_graph.values()),
+                "avg_links_per_page": round(
+                    sum(len(v) for v in self.link_graph.values()) / max(len(self.link_graph), 1), 1
+                ),
+            },
+            "errors": self.errors,
+        }
+
+
+# ── CLI ────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="SEO Crawl — Recursive mini-crawler (BMAD+ SEO Engine)"
+    )
+    parser.add_argument("url", help="Base URL to crawl")
+    parser.add_argument("--depth", "-d", type=int, default=2, help="Max crawl depth (default: 2)")
+    parser.add_argument("--max", "-m", type=int, default=25, help="Max pages (default: 25)")
+    parser.add_argument("--timeout", "-t", type=int, default=15, help="Per-page timeout (default: 15s)")
+    parser.add_argument("--json", "-j", action="store_true", help="Output as JSON")
+
+    args = parser.parse_args()
+
+    crawler = SEOCrawler(
+        base_url=args.url,
+        max_depth=args.depth,
+        max_pages=args.max,
+        timeout=args.timeout,
+    )
+
+    print(f"Crawling {args.url} (depth={args.depth}, max={args.max})...", file=sys.stderr)
+    crawler.crawl()
+    results = crawler.get_results()
+
+    if args.json:
+        # Convert sets to lists for JSON serialization
+        print(json.dumps(results, indent=2, ensure_ascii=False, default=list))
+    else:
+        print(f"\n{'='*60}")
+        print(f"Crawl Summary: {results['base_url']}")
+        print(f"{'='*60}")
+        print(f"Pages crawled: {results['pages_crawled']}")
+        print(f"Sitemap URLs: {results['sitemap_urls_found']}")
+        print(f"robots.txt: {'✅' if results['has_robots_txt'] else '❌'}")
+        print(f"Internal links: {results['link_graph_summary']['total_internal_links']}")
+        print(f"Avg links/page: {results['link_graph_summary']['avg_links_per_page']}")
+        print(f"Orphan pages: {len(results['orphan_pages'])}")
+        print(f"Errors: {len(results['errors'])}")
+
+        print(f"\n{'─'*60}")
+        print("Pages:")
+        for page in results["pages"]:
+            status = "✅" if page["status"] == 200 else f"⚠️ {page['status']}"
+            title = (page["title"] or "No title")[:50]
+            print(f"  {status} [{page['depth']}] {title} ({page['word_count']} words)")
+            print(f"       {page['url']}")
+
+
+if __name__ == "__main__":
+    main()

package/src/bmad-plus/packs/pack-seo/scripts/seo_fetch.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+"""
+SEO Fetch — Secure HTTP page fetcher for SEO analysis.
+
+Features:
+- SSRF protection (blocks private/loopback/reserved IPs)
+- Multi-UA support (standard, Googlebot, GPTBot, ClaudeBot)
+- Redirect chain tracking
+- Cookie handling
+- Configurable timeout
+
+Author: Laurent Rochetta
+License: MIT
+"""
+
+import argparse
+import ipaddress
+import json
+import socket
+import sys
+from urllib.parse import urljoin, urlparse
+
+try:
+    import requests
+except ImportError:
+    print("Error: requests library required. Install: pip install requests", file=sys.stderr)
+    sys.exit(1)
+
+
+# ── User-Agent Presets ──────────────────────────────────────────────
+
+USER_AGENTS = {
+    "default": (
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 BMADSEOEngine/2.0"
+    ),
+    "googlebot": (
+        "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+    ),
+    "gptbot": (
+        "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; "
+        "+https://openai.com/gptbot)"
+    ),
+    "claudebot": (
+        "Mozilla/5.0 (compatible; ClaudeBot/1.0; +https://www.anthropic.com/claudebot)"
+    ),
+    "mobile": (
+        "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) "
+        "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"
+    ),
+}
+
+DEFAULT_HEADERS = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9,fr;q=0.8",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Connection": "keep-alive",
+    "Cache-Control": "no-cache",
+}
+
+
+# ── Security: SSRF Prevention ──────────────────────────────────────
+
+def _ip_is_blocked(ip: "ipaddress._BaseAddress") -> bool:
+    """Return True if an IP falls in any range that must never be reached."""
+    return bool(
+        ip.is_private
+        or ip.is_loopback
+        or ip.is_reserved
+        or ip.is_link_local
+        or ip.is_multicast
+        or ip.is_unspecified
+    )
+
+
+def is_safe_url(url: str) -> bool:
+    """Block requests to private, loopback, and reserved IP addresses.
+
+    Fails CLOSED: a missing host, a non-HTTP(S) scheme, a DNS resolution
+    error, or an unparseable/blocked address all cause the URL to be
+    rejected. Every resolved address (IPv4 and IPv6) must be public.
+    """
+    parsed = urlparse(url)
+    hostname = parsed.hostname
+
+    if not hostname:
+        return False
+
+    if parsed.scheme not in ("http", "https"):
+        return False
+
+    try:
+        # Resolve ALL IP addresses (IPv4 and IPv6) via getaddrinfo
+        addrinfo = socket.getaddrinfo(hostname, None)
+    except socket.gaierror:
+        return False  # Fail closed: unresolvable host is treated as unsafe
+
+    if not addrinfo:
+        return False  # Fail closed: no addresses resolved
+
+    for entry in addrinfo:
+        ip_str = entry[4][0]  # sockaddr[0] contains the IP string
+        try:
+            ip = ipaddress.ip_address(ip_str)
+        except ValueError:
+            return False  # Fail closed: unparseable address
+        # IPv4-mapped IPv6 (::ffff:a.b.c.d) must be checked as its IPv4 form
+        mapped = getattr(ip, "ipv4_mapped", None)
+        if mapped is not None and _ip_is_blocked(mapped):
+            return False
+        if _ip_is_blocked(ip):
+            return False
+
+    return True
+
+
+# ── Core Fetcher ───────────────────────────────────────────────────
+
+def fetch_page(
+    url: str,
+    timeout: int = 30,
+    follow_redirects: bool = True,
+    max_redirects: int = 5,
+    user_agent: str = "default",
+) -> dict:
+    """
+    Fetch a web page with security checks and detailed response tracking.
+
+    Returns dict with: url, status_code, content, headers, redirect_chain,
+    content_length, response_time_ms, error
+    """
+    result = {
+        "url": url,
+        "final_url": None,
+        "status_code": None,
+        "content": None,
+        "headers": {},
+        "redirect_chain": [],
+        "content_length": 0,
+        "response_time_ms": 0,
+        "error": None,
+    }
+
+    # Normalize URL
+    parsed = urlparse(url)
+    if not parsed.scheme:
+        url = f"https://{url}"
+        parsed = urlparse(url)
+
+    if parsed.scheme not in ("http", "https"):
+        result["error"] = f"Invalid URL scheme: {parsed.scheme}"
+        return result
+
+    # SSRF check
+    if not is_safe_url(url):
+        resolved = "unknown"
+        try:
+            # Use getaddrinfo for consistent multi-address resolution
+            addrinfo = socket.getaddrinfo(parsed.hostname, None)
+            resolved = ", ".join(set(entry[4][0] for entry in addrinfo))
+        except Exception:
+            pass
+        result["error"] = f"Blocked: URL resolves to private/internal IP ({resolved})"
+        return result
+
+    try:
+        session = requests.Session()
+
+        headers = dict(DEFAULT_HEADERS)
+        ua_string = USER_AGENTS.get(user_agent, user_agent)
+        headers["User-Agent"] = ua_string
+
+        import time
+        start = time.monotonic()
+
+        # Follow redirects manually so is_safe_url() runs on EVERY hop.
+        # Letting requests follow redirects internally would allow a
+        # public URL to redirect (302) to an internal/metadata endpoint
+        # (redirect-based SSRF), bypassing the initial check.
+        current_url = url
+        redirect_chain = []
+        hops = 0
+        response = None
+
+        while True:
+            response = session.get(
+                current_url,
+                headers=headers,
+                timeout=timeout,
+                allow_redirects=False,
+            )
+
+            if not follow_redirects or not response.is_redirect:
+                break
+
+            location = response.headers.get("Location")
+            if not location:
+                break
+
+            next_url = urljoin(current_url, location)
+            next_parsed = urlparse(next_url)
+
+            if next_parsed.scheme not in ("http", "https"):
+                result["error"] = (
+                    f"Blocked redirect to non-HTTP(S) scheme: {next_parsed.scheme}"
+                )
+                return result
+
+            # Re-validate the redirect target (blocks redirect-based SSRF)
+            if not is_safe_url(next_url):
+                result["error"] = (
+                    f"Blocked: redirect to private/internal URL ({next_url})"
+                )
+                return result
+
+            hops += 1
+            if hops > max_redirects:
+                result["error"] = f"Too many redirects (max {max_redirects})"
+                return result
+
+            redirect_chain.append(
+                {"url": current_url, "status": response.status_code}
+            )
+            current_url = next_url
+
+        elapsed_ms = round((time.monotonic() - start) * 1000)
+
+        result["final_url"] = current_url
+        result["status_code"] = response.status_code
+        result["content"] = response.text
+        result["headers"] = dict(response.headers)
+        result["content_length"] = len(response.content)
+        result["response_time_ms"] = elapsed_ms
+        result["redirect_chain"] = redirect_chain
+
+    except requests.exceptions.Timeout:
+        result["error"] = f"Request timed out after {timeout}s"
+    except requests.exceptions.TooManyRedirects:
+        result["error"] = f"Too many redirects (max {max_redirects})"
+    except requests.exceptions.SSLError as e:
+        result["error"] = f"SSL error: {e}"
+    except requests.exceptions.ConnectionError as e:
+        result["error"] = f"Connection error: {e}"
+    except requests.exceptions.RequestException as e:
+        result["error"] = f"Request failed: {e}"
+
+    return result
+
+
+# ── CLI ────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="SEO Fetch — Secure HTTP fetcher for SEO analysis (BMAD+ SEO Engine)"
+    )
+    parser.add_argument("url", help="URL to fetch")
+    parser.add_argument("--output", "-o", help="Save HTML to file")
+    parser.add_argument("--timeout", "-t", type=int, default=30, help="Timeout in seconds")
+    parser.add_argument("--no-redirects", action="store_true", help="Don't follow redirects")
+    parser.add_argument(
+        "--ua", choices=list(USER_AGENTS.keys()), default="default",
+        help="User-Agent preset (default, googlebot, gptbot, claudebot, mobile)"
+    )
+    parser.add_argument("--json", "-j", action="store_true", help="Output full result as JSON")
+
+    args = parser.parse_args()
+
+    result = fetch_page(
+        args.url,
+        timeout=args.timeout,
+        follow_redirects=not args.no_redirects,
+        user_agent=args.ua,
+    )
+
+    if result["error"]:
+        print(f"Error: {result['error']}", file=sys.stderr)
+        sys.exit(1)
+
+    if args.json:
+        # Output metadata as JSON (without full HTML content for readability)
+        output = {k: v for k, v in result.items() if k != "content"}
+        output["content_preview"] = result["content"][:500] if result["content"] else None
+        print(json.dumps(output, indent=2))
+    elif args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(result["content"])
+        print(f"Saved to {args.output}")
+    else:
+        print(result["content"])
+
+    # Metadata to stderr
+    print("\n--- Fetch Summary ---", file=sys.stderr)
+    print(f"Final URL: {result['final_url']}", file=sys.stderr)
+    print(f"Status: {result['status_code']}", file=sys.stderr)
+    print(f"Size: {result['content_length']:,} bytes", file=sys.stderr)
+    print(f"Time: {result['response_time_ms']}ms", file=sys.stderr)
+    if result["redirect_chain"]:
+        chain = " → ".join(r["url"] for r in result["redirect_chain"])
+        print(f"Redirects: {chain}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()