@booklib/skills 1.2.0 → 1.3.0

package/web-scraping-python/scripts/new_scraper.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""
+new_scraper.py — Scaffold a best-practice web scraper.
+Usage: python new_scraper.py <scraper-name> <target-url>
+
+Generates <scraper-name>.py — a real, runnable scraper with retry, rate limiting,
+robots.txt checking, BeautifulSoup parsing, and CSV output.
+"""
+
+import sys
+from pathlib import Path
+from string import Template
+
+SCRAPER_TEMPLATE = '''\
+#!/usr/bin/env python3
+"""
+$scraper_name — scraper for $target_url
+Generated by new_scraper.py. Edit the parse() function for your target site.
+"""
+
+import csv
+import logging
+import time
+import urllib.parse
+import urllib.robotparser
+from datetime import datetime
+from pathlib import Path
+
+try:
+    import requests
+    from requests.adapters import HTTPAdapter
+    from urllib3.util.retry import Retry
+    from bs4 import BeautifulSoup
+except ImportError as exc:
+    raise SystemExit(
+        f"Missing dependency: {exc}\\n"
+        "Install with: pip install requests beautifulsoup4"
+    ) from exc
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s  %(levelname)-8s  %(message)s",
+    datefmt="%Y-%m-%dT%H:%M:%S",
+)
+logger = logging.getLogger(__name__)
+
+TARGET_URL = "$target_url"
+OUTPUT_CSV = "$scraper_name_output.csv"
+REQUEST_DELAY = 1.5   # seconds between requests — be polite
+USER_AGENT = "research-bot/1.0 (+https://example.com/bot)"
+
+
+# ---------------------------------------------------------------------------
+# Session with retry
+# ---------------------------------------------------------------------------
+
+def make_session() -> requests.Session:
+    """Build a requests Session with automatic retries on transient errors."""
+    session = requests.Session()
+    retry_strategy = Retry(
+        total=3,
+        backoff_factor=1.5,
+        status_forcelist=[429, 500, 502, 503, 504],
+        allowed_methods=["GET", "HEAD"],
+    )
+    adapter = HTTPAdapter(max_retries=retry_strategy)
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    session.headers.update({"User-Agent": USER_AGENT})
+    return session
+
+
+# ---------------------------------------------------------------------------
+# Robots.txt
+# ---------------------------------------------------------------------------
+
+def check_robots(url: str, user_agent: str = USER_AGENT) -> bool:
+    """Return True if scraping the URL is permitted by robots.txt."""
+    parsed = urllib.parse.urlparse(url)
+    robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
+    rp = urllib.robotparser.RobotFileParser()
+    rp.set_url(robots_url)
+    try:
+        rp.read()
+        allowed = rp.can_fetch(user_agent, url)
+        if not allowed:
+            logger.warning("robots.txt disallows scraping: %s", url)
+        return allowed
+    except Exception as exc:
+        logger.warning("Could not read robots.txt (%s) — proceeding cautiously.", exc)
+        return True   # assume allowed if robots.txt is unreachable
+
+
+# ---------------------------------------------------------------------------
+# Parse — EDIT THIS FUNCTION for your target site
+# ---------------------------------------------------------------------------
+
+def parse(html: str, source_url: str) -> list[dict]:
+    """
+    Extract structured data from a page. Returns a list of dicts.
+    Edit the selectors below for your actual target.
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    records = []
+
+    # Example: scrape all hyperlinks with their text
+    # Replace this block with selectors for your target site.
+    for link in soup.find_all("a", href=True):
+        href = link["href"]
+        text = link.get_text(strip=True)
+        if not text:
+            continue
+        # Resolve relative URLs
+        full_url = urllib.parse.urljoin(source_url, href)
+        records.append({
+            "text": text,
+            "url": full_url,
+            "source_page": source_url,
+            "scraped_at": datetime.utcnow().isoformat(),
+        })
+
+    return records
+
+
+# ---------------------------------------------------------------------------
+# Core fetch + crawl logic
+# ---------------------------------------------------------------------------
+
+def fetch_page(session: requests.Session, url: str) -> str | None:
+    """Fetch a single page and return HTML. Returns None on failure."""
+    try:
+        response = session.get(url, timeout=20)
+        response.raise_for_status()
+        return response.text
+    except requests.exceptions.RequestException as exc:
+        logger.error("Failed to fetch %s: %s", url, exc)
+        return None
+
+
+def scrape(urls: list[str] | None = None) -> list[dict]:
+    """
+    Main scrape loop. Pass a list of URLs or leave None to scrape TARGET_URL.
+    Respects robots.txt and rate-limits requests.
+    """
+    urls = urls or [TARGET_URL]
+    session = make_session()
+    all_records: list[dict] = []
+
+    for i, url in enumerate(urls):
+        if not check_robots(url):
+            logger.info("Skipping disallowed URL: %s", url)
+            continue
+
+        logger.info("Fetching (%d/%d): %s", i + 1, len(urls), url)
+        html = fetch_page(session, url)
+        if html is None:
+            continue
+
+        records = parse(html, url)
+        logger.info("  -> %d records found", len(records))
+        all_records.extend(records)
+
+        if i < len(urls) - 1:
+            time.sleep(REQUEST_DELAY)   # rate limit between pages
+
+    return all_records
+
+
+# ---------------------------------------------------------------------------
+# CSV output
+# ---------------------------------------------------------------------------
+
+def save_csv(records: list[dict], path: str = OUTPUT_CSV) -> None:
+    """Write records to a CSV file."""
+    if not records:
+        logger.warning("No records to save.")
+        return
+    out = Path(path)
+    with out.open("w", newline="", encoding="utf-8") as fh:
+        writer = csv.DictWriter(fh, fieldnames=records[0].keys())
+        writer.writeheader()
+        writer.writerows(records)
+    logger.info("Saved %d records to %s", len(records), out)
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    records = scrape()
+    save_csv(records)
+'''
+
+
+def main():
+    if len(sys.argv) < 3:
+        print("Usage: python new_scraper.py <scraper-name> <target-url>")
+        sys.exit(1)
+
+    scraper_name = sys.argv[1]
+    target_url = sys.argv[2]
+
+    # Basic URL sanity check
+    if not target_url.startswith(("http://", "https://")):
+        print(f"Warning: target URL '{target_url}' doesn't look like a full URL.")
+
+    output_path = Path(f"{scraper_name}.py")
+    if output_path.exists():
+        print(f"Error: '{output_path}' already exists. Choose a different name.")
+        sys.exit(1)
+
+    safe_name = scraper_name.replace("-", "_")
+    content = Template(SCRAPER_TEMPLATE).safe_substitute(
+        scraper_name=safe_name,
+        target_url=target_url,
+    )
+    output_path.write_text(content, encoding="utf-8")
+    output_path.chmod(0o755)
+
+    print(f"\nScraper '{scraper_name}' created: {output_path}\n")
+    print(f"  Target URL : {target_url}")
+    print(f"  Output CSV : {safe_name}_output.csv")
+    print(f"\nNext steps:")
+    print(f"  1. pip install requests beautifulsoup4")
+    print(f"  2. Edit the parse() function in {output_path} for your target site")
+    print(f"  3. python {output_path}")
+
+
+if __name__ == "__main__":
+    main()