omnius 1.0.0

package/dist/scripts/crawlee-scraper.py

@@ -0,0 +1,358 @@
+#!/usr/bin/env python3
+"""
+crawlee-scraper.py — Self-contained web scraping script using Crawlee.
+
+Supports two crawling strategies:
+  - beautifulsoup (default): Fast HTTP-based scraping via httpx + BeautifulSoup.
+    Best for static content, documentation pages, and API docs.
+  - playwright: Headless browser scraping for JS-rendered pages, SPAs, and
+    sites that require JavaScript execution to display content.
+
+Usage:
+  python3 crawlee-scraper.py --url URL [--strategy beautifulsoup|playwright]
+    [--max-pages N] [--max-depth N] [--output json|text] [--extract links|text|all]
+
+Auto-installs crawlee and playwright if not present.
+"""
+
+import argparse
+import asyncio
+import json
+import subprocess
+import sys
+import importlib
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Auto-install crawlee if missing
+# ---------------------------------------------------------------------------
+
+def ensure_crawlee():
+    """Ensure crawlee is installed, installing it if needed."""
+    try:
+        importlib.import_module("crawlee")
+        return True
+    except ImportError:
+        pass
+
+    print("[crawlee-scraper] Installing crawlee...", file=sys.stderr)
+    try:
+        subprocess.check_call(
+            [sys.executable, "-m", "pip", "install", "crawlee[all]"],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+        )
+        return True
+    except subprocess.CalledProcessError as e:
+        # Try without [all] extras (minimal install — beautifulsoup only)
+        try:
+            subprocess.check_call(
+                [sys.executable, "-m", "pip", "install", "crawlee[beautifulsoup]"],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.PIPE,
+            )
+            return True
+        except subprocess.CalledProcessError:
+            print(f"[crawlee-scraper] Failed to install crawlee: {e}", file=sys.stderr)
+            return False
+
+
+def ensure_playwright():
+    """Ensure playwright browsers are installed."""
+    try:
+        importlib.import_module("playwright")
+        subprocess.check_call(
+            [sys.executable, "-m", "playwright", "install", "chromium"],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+        )
+        return True
+    except (ImportError, subprocess.CalledProcessError):
+        try:
+            subprocess.check_call(
+                [sys.executable, "-m", "pip", "install", "playwright"],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.PIPE,
+            )
+            subprocess.check_call(
+                [sys.executable, "-m", "playwright", "install", "chromium"],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.PIPE,
+            )
+            return True
+        except subprocess.CalledProcessError as e:
+            print(f"[crawlee-scraper] Failed to install playwright: {e}", file=sys.stderr)
+            return False
+
+
+# ---------------------------------------------------------------------------
+# BeautifulSoup crawler
+# ---------------------------------------------------------------------------
+
+async def crawl_beautifulsoup(
+    start_url: str,
+    max_pages: int = 5,
+    max_depth: int = 1,
+    extract: str = "all",
+) -> list[dict[str, Any]]:
+    """Crawl using BeautifulSoup (HTTP-based, fast, no JS rendering)."""
+    from crawlee.beautifulsoup_crawler import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
+    from crawlee import ConcurrencySettings
+
+    results: list[dict[str, Any]] = []
+    pages_crawled = 0
+
+    crawler = BeautifulSoupCrawler(
+        max_request_retries=2,
+        request_handler_timeout=asyncio.timedelta(seconds=30),
+        max_requests_per_crawl=max_pages,
+        concurrency_settings=ConcurrencySettings(
+            max_concurrency=3,
+        ),
+    )
+
+    @crawler.router.default_handler
+    async def handler(context: BeautifulSoupCrawlingContext) -> None:
+        nonlocal pages_crawled
+        if pages_crawled >= max_pages:
+            return
+        pages_crawled += 1
+
+        soup = context.soup
+        page_data: dict[str, Any] = {
+            "url": context.request.url,
+            "status": 200,
+        }
+
+        # Extract title
+        title_tag = soup.find("title")
+        page_data["title"] = title_tag.get_text(strip=True) if title_tag else ""
+
+        # Extract text content
+        if extract in ("text", "all"):
+            # Remove script and style elements
+            for tag in soup(["script", "style", "nav", "footer", "header"]):
+                tag.decompose()
+            text = soup.get_text(separator="\n", strip=True)
+            # Collapse whitespace
+            lines = [line.strip() for line in text.splitlines() if line.strip()]
+            page_data["text"] = "\n".join(lines)
+
+        # Extract links
+        if extract in ("links", "all"):
+            links = []
+            for a_tag in soup.find_all("a", href=True):
+                href = str(a_tag["href"])
+                link_text = a_tag.get_text(strip=True)
+                if href.startswith(("http://", "https://")):
+                    links.append({"url": href, "text": link_text})
+            page_data["links"] = links[:50]  # Cap at 50 links per page
+
+        # Extract metadata
+        meta_tags = {}
+        for meta in soup.find_all("meta"):
+            name = meta.get("name", meta.get("property", ""))
+            content = meta.get("content", "")
+            if name and content:
+                meta_tags[str(name)] = str(content)
+        if meta_tags:
+            page_data["meta"] = meta_tags
+
+        results.append(page_data)
+
+        # Enqueue links for deeper crawling if depth allows
+        if max_depth > 0:
+            await context.enqueue_links(strategy="same-domain")
+
+    await crawler.run([start_url])
+    return results
+
+
+# ---------------------------------------------------------------------------
+# Playwright crawler
+# ---------------------------------------------------------------------------
+
+async def crawl_playwright(
+    start_url: str,
+    max_pages: int = 5,
+    max_depth: int = 1,
+    extract: str = "all",
+) -> list[dict[str, Any]]:
+    """Crawl using Playwright (headless browser, JS rendering)."""
+    from crawlee.playwright_crawler import PlaywrightCrawler, PlaywrightCrawlingContext
+    from crawlee import ConcurrencySettings
+
+    results: list[dict[str, Any]] = []
+    pages_crawled = 0
+
+    crawler = PlaywrightCrawler(
+        max_request_retries=2,
+        request_handler_timeout=asyncio.timedelta(seconds=60),
+        max_requests_per_crawl=max_pages,
+        headless=True,
+        browser_type="chromium",
+        concurrency_settings=ConcurrencySettings(
+            max_concurrency=2,
+        ),
+    )
+
+    @crawler.router.default_handler
+    async def handler(context: PlaywrightCrawlingContext) -> None:
+        nonlocal pages_crawled
+        if pages_crawled >= max_pages:
+            return
+        pages_crawled += 1
+
+        page = context.page
+        page_data: dict[str, Any] = {
+            "url": context.request.url,
+            "status": 200,
+        }
+
+        # Wait for content to load
+        try:
+            await page.wait_for_load_state("networkidle", timeout=15000)
+        except Exception:
+            await page.wait_for_load_state("domcontentloaded", timeout=10000)
+
+        # Extract title
+        page_data["title"] = await page.title()
+
+        # Extract text content
+        if extract in ("text", "all"):
+            # Remove non-content elements via JS
+            text = await page.evaluate("""() => {
+                const remove = document.querySelectorAll('script, style, nav, footer, header, [role="navigation"]');
+                remove.forEach(el => el.remove());
+                return document.body ? document.body.innerText : '';
+            }""")
+            lines = [line.strip() for line in str(text).splitlines() if line.strip()]
+            page_data["text"] = "\n".join(lines)
+
+        # Extract links
+        if extract in ("links", "all"):
+            links = await page.evaluate("""() => {
+                const anchors = document.querySelectorAll('a[href]');
+                return Array.from(anchors).slice(0, 50).map(a => ({
+                    url: a.href,
+                    text: a.innerText.trim()
+                })).filter(l => l.url.startsWith('http'));
+            }""")
+            page_data["links"] = links
+
+        # Extract meta tags
+        meta_tags = await page.evaluate("""() => {
+            const metas = document.querySelectorAll('meta[name], meta[property]');
+            const result = {};
+            metas.forEach(m => {
+                const key = m.getAttribute('name') || m.getAttribute('property');
+                const val = m.getAttribute('content');
+                if (key && val) result[key] = val;
+            });
+            return result;
+        }""")
+        if meta_tags:
+            page_data["meta"] = meta_tags
+
+        results.append(page_data)
+
+        # Enqueue links for deeper crawling
+        if max_depth > 0:
+            await context.enqueue_links(strategy="same-domain")
+
+    await crawler.run([start_url])
+    return results
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+async def main():
+    parser = argparse.ArgumentParser(description="Crawlee-based web scraper")
+    parser.add_argument("--url", required=True, help="Starting URL to crawl")
+    parser.add_argument(
+        "--strategy",
+        choices=["beautifulsoup", "playwright"],
+        default="beautifulsoup",
+        help="Crawling strategy (default: beautifulsoup)",
+    )
+    parser.add_argument(
+        "--max-pages", type=int, default=5, help="Max pages to crawl (default: 5)"
+    )
+    parser.add_argument(
+        "--max-depth", type=int, default=1, help="Max crawl depth (default: 1)"
+    )
+    parser.add_argument(
+        "--output",
+        choices=["json", "text"],
+        default="json",
+        help="Output format (default: json)",
+    )
+    parser.add_argument(
+        "--extract",
+        choices=["links", "text", "all"],
+        default="all",
+        help="What to extract (default: all)",
+    )
+
+    args = parser.parse_args()
+
+    # Ensure crawlee is installed
+    if not ensure_crawlee():
+        result = {"error": "Failed to install crawlee. Install manually: pip install 'crawlee[all]'"}
+        print(json.dumps(result))
+        sys.exit(1)
+
+    # For playwright strategy, ensure browsers are installed
+    if args.strategy == "playwright":
+        if not ensure_playwright():
+            result = {"error": "Failed to install playwright browsers. Install manually: playwright install chromium"}
+            print(json.dumps(result))
+            sys.exit(1)
+
+    # Run the crawler
+    try:
+        if args.strategy == "playwright":
+            results = await crawl_playwright(
+                args.url,
+                max_pages=args.max_pages,
+                max_depth=args.max_depth,
+                extract=args.extract,
+            )
+        else:
+            results = await crawl_beautifulsoup(
+                args.url,
+                max_pages=args.max_pages,
+                max_depth=args.max_depth,
+                extract=args.extract,
+            )
+    except Exception as e:
+        result = {"error": f"Crawl failed: {str(e)}"}
+        print(json.dumps(result))
+        sys.exit(1)
+
+    # Output results
+    if args.output == "text":
+        for page in results:
+            print(f"=== {page.get('title', 'Untitled')} ===")
+            print(f"URL: {page.get('url', '')}")
+            if "text" in page:
+                print(page["text"][:5000])
+            if "links" in page:
+                print(f"\nLinks ({len(page['links'])}):")
+                for link in page["links"][:20]:
+                    print(f"  - {link.get('text', '')}: {link.get('url', '')}")
+            print()
+    else:
+        output = {
+            "success": True,
+            "strategy": args.strategy,
+            "pages_crawled": len(results),
+            "results": results,
+        }
+        print(json.dumps(output, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())