@booklib/skills 1.2.0 → 1.3.1

package/using-asyncio-python/scripts/check_blocking.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+"""
+check_blocking.py — Static analyser for blocking calls inside async functions.
+
+Usage: python check_blocking.py <file_or_directory> [<file_or_directory> ...]
+
+Flags:
+  --exit-zero   Exit 0 even when issues are found (useful in CI to report only)
+  --summary     Print a summary table at the end
+"""
+
+import ast
+import argparse
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Iterator
+
+# ---------------------------------------------------------------------------
+# Rules
+# ---------------------------------------------------------------------------
+# Each rule is (description, fix_hint, matcher_function)
+# matcher_function(node) -> bool
+
+
+def _call_matches(node: ast.expr, *name_parts: str) -> bool:
+    """True if node is a Call whose function matches the dotted name."""
+    if not isinstance(node, ast.Call):
+        return False
+    func = node.func
+    # Simple name: open, sleep, etc.
+    if len(name_parts) == 1 and isinstance(func, ast.Name):
+        return func.id == name_parts[0]
+    # Attribute: requests.get, time.sleep, etc.
+    if len(name_parts) == 2 and isinstance(func, ast.Attribute):
+        obj = func.value
+        return isinstance(obj, ast.Name) and obj.id == name_parts[0] and func.attr == name_parts[1]
+    return False
+
+
+def _is_sync_open(node: ast.expr) -> bool:
+    """Flags open() calls that are not preceded by 'async with'."""
+    return _call_matches(node, "open")
+
+
+def _is_file_rw(node: ast.expr) -> bool:
+    """Flags .read() / .write() attribute calls (heuristic)."""
+    if not isinstance(node, ast.Call):
+        return False
+    func = node.func
+    return isinstance(func, ast.Attribute) and func.attr in {"read", "write", "readlines"}
+
+
+@dataclass
+class Rule:
+    id: str
+    description: str
+    fix: str
+    matcher: object  # callable(node) -> bool
+
+
+RULES: list[Rule] = [
+    Rule(
+        id="ASYNC001",
+        description="requests.get() blocks the event loop",
+        fix="Use aiohttp.ClientSession().get() or httpx.AsyncClient().get()",
+        matcher=lambda n: _call_matches(n, "requests", "get"),
+    ),
+    Rule(
+        id="ASYNC002",
+        description="requests.post() blocks the event loop",
+        fix="Use aiohttp.ClientSession().post() or httpx.AsyncClient().post()",
+        matcher=lambda n: _call_matches(n, "requests", "post"),
+    ),
+    Rule(
+        id="ASYNC003",
+        description="requests.put() blocks the event loop",
+        fix="Use aiohttp.ClientSession().put() or httpx.AsyncClient().put()",
+        matcher=lambda n: _call_matches(n, "requests", "put"),
+    ),
+    Rule(
+        id="ASYNC004",
+        description="requests.delete() blocks the event loop",
+        fix="Use aiohttp.ClientSession().delete() or httpx.AsyncClient().delete()",
+        matcher=lambda n: _call_matches(n, "requests", "delete"),
+    ),
+    Rule(
+        id="ASYNC005",
+        description="time.sleep() blocks the event loop",
+        fix="Use 'await asyncio.sleep(seconds)' instead",
+        matcher=lambda n: _call_matches(n, "time", "sleep"),
+    ),
+    Rule(
+        id="ASYNC006",
+        description="open() is a synchronous file operation",
+        fix="Use 'async with aiofiles.open(...)' from the aiofiles package",
+        matcher=_is_sync_open,
+    ),
+    Rule(
+        id="ASYNC007",
+        description="subprocess.run() blocks the event loop",
+        fix="Use 'await asyncio.create_subprocess_exec()' or asyncio.create_subprocess_shell()",
+        matcher=lambda n: _call_matches(n, "subprocess", "run"),
+    ),
+    Rule(
+        id="ASYNC008",
+        description="subprocess.call() blocks the event loop",
+        fix="Use 'await asyncio.create_subprocess_exec()' instead",
+        matcher=lambda n: _call_matches(n, "subprocess", "call"),
+    ),
+    Rule(
+        id="ASYNC009",
+        description=".read()/.write()/.readlines() on a synchronous file handle",
+        fix="Open the file with aiofiles and use 'await file.read()' / 'await file.write()'",
+        matcher=_is_file_rw,
+    ),
+]
+
+
+# ---------------------------------------------------------------------------
+# Finding
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Finding:
+    file: Path
+    line: int
+    col: int
+    async_func: str
+    rule: Rule
+
+
+def _collect_async_funcs(tree: ast.AST) -> Iterator[ast.AsyncFunctionDef]:
+    """Yield all async def nodes in the tree, including nested ones."""
+    for node in ast.walk(tree):
+        if isinstance(node, ast.AsyncFunctionDef):
+            yield node
+
+
+def _nodes_inside_sync_context(func_node: ast.AsyncFunctionDef) -> set[int]:
+    """
+    Return the set of node ids that are inside a nested sync def or class,
+    so we don't flag blocking calls that are legitimately in sync helpers.
+    """
+    excluded: set[int] = set()
+    for node in ast.walk(func_node):
+        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
+            for child in ast.walk(node):
+                excluded.add(id(child))
+    return excluded
+
+
+def check_file(path: Path) -> list[Finding]:
+    try:
+        source = path.read_text(encoding="utf-8", errors="replace")
+    except OSError as exc:
+        print(f"ERROR: Cannot read {path}: {exc}", file=sys.stderr)
+        return []
+
+    try:
+        tree = ast.parse(source, filename=str(path))
+    except SyntaxError as exc:
+        print(f"ERROR: Syntax error in {path}: {exc}", file=sys.stderr)
+        return []
+
+    findings: list[Finding] = []
+
+    for async_func in _collect_async_funcs(tree):
+        excluded = _nodes_inside_sync_context(async_func)
+        for node in ast.walk(async_func):
+            if id(node) in excluded:
+                continue
+            for rule in RULES:
+                if rule.matcher(node):
+                    findings.append(
+                        Finding(
+                            file=path,
+                            line=node.lineno,
+                            col=node.col_offset,
+                            async_func=async_func.name,
+                            rule=rule,
+                        )
+                    )
+    return findings
+
+
+def iter_python_files(path: Path) -> Iterator[Path]:
+    if path.is_file():
+        if path.suffix == ".py":
+            yield path
+    elif path.is_dir():
+        yield from sorted(path.rglob("*.py"))
+    else:
+        print(f"WARNING: {path} is not a file or directory — skipping.", file=sys.stderr)
+
+
+# ---------------------------------------------------------------------------
+# Reporting
+# ---------------------------------------------------------------------------
+
+def print_findings(findings: list[Finding]) -> None:
+    for f in findings:
+        print(
+            f"{f.file}:{f.line}:{f.col}: [{f.rule.id}] "
+            f"In 'async def {f.async_func}': {f.rule.description}"
+        )
+        print(f"  Fix: {f.rule.fix}")
+
+
+def print_summary(all_findings: list[Finding]) -> None:
+    if not all_findings:
+        print("\nSummary: No blocking call issues found.")
+        return
+
+    from collections import Counter
+    by_rule: Counter = Counter(f.rule.id for f in all_findings)
+    by_file: Counter = Counter(str(f.file) for f in all_findings)
+
+    print("\n--- Summary ---")
+    print(f"Total issues: {len(all_findings)}")
+    print("\nBy rule:")
+    for rule_id, count in sorted(by_rule.items()):
+        rule = next(r for r in RULES if r.id == rule_id)
+        print(f"  {rule_id}: {count}x  ({rule.description})")
+    print("\nBy file:")
+    for filepath, count in sorted(by_file.items()):
+        print(f"  {count:3d}  {filepath}")
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Find blocking calls inside async functions."
+    )
+    parser.add_argument(
+        "paths", nargs="+", type=Path, metavar="file_or_dir",
+        help="Python file(s) or director(ies) to analyse"
+    )
+    parser.add_argument(
+        "--exit-zero", action="store_true",
+        help="Always exit 0 (useful for non-blocking CI report)"
+    )
+    parser.add_argument(
+        "--summary", action="store_true",
+        help="Print a summary table after the findings"
+    )
+    args = parser.parse_args()
+
+    all_findings: list[Finding] = []
+    for raw_path in args.paths:
+        for py_file in iter_python_files(raw_path):
+            findings = check_file(py_file)
+            all_findings.extend(findings)
+            print_findings(findings)
+
+    if args.summary:
+        print_summary(all_findings)
+
+    if not all_findings:
+        print("No blocking call issues detected.")
+
+    if all_findings and not args.exit_zero:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/web-scraping-python/evals/evals.json

@@ -0,0 +1,46 @@
+{
+  "evals": [
+    {
+      "id": "eval-01-no-rate-limiting-no-error-handling-no-robots",
+      "prompt": "Review this web scraper:\n\n```python\nimport requests\nfrom bs4 import BeautifulSoup\nimport json\n\nBASE_URL = 'https://books.example.com'\n\ndef scrape_all_books():\n    all_books = []\n    page = 1\n\n    while True:\n        url = f'{BASE_URL}/catalogue/page-{page}.html'\n        response = requests.get(url)\n        soup = BeautifulSoup(response.text, 'html.parser')\n\n        books = soup.find_all('article', class_='product_pod')\n        if not books:\n            break\n\n        for book in books:\n            title = book.find('h3').find('a')['title']\n            price = book.find('p', class_='price_color').text\n            rating = book.find('p', class_='star-rating')['class'][1]\n            all_books.append({'title': title, 'price': price, 'rating': rating})\n\n        page += 1\n\n    return all_books\n\nresult = scrape_all_books()\nwith open('books.json', 'w') as f:\n    json.dump(result, f)\n```",
+      "expectations": [
+        "Flags no robots.txt check: the scraper does not check or respect the site's robots.txt before crawling (Ch 18: always check and honor robots.txt)",
+        "Flags no rate limiting: requests are issued as fast as possible with no delay between pages; recommends adding `time.sleep()` of at least 1-3 seconds between requests (Ch 14: rate limit requests)",
+        "Flags no error handling on `requests.get()`: a network error, timeout, or non-200 response will raise an exception or silently produce garbage HTML (Ch 1, 14: wrap requests in try/except, check response status)",
+        "Flags no User-Agent header: the scraper uses the default requests User-Agent which may be blocked and does not identify the bot (Ch 14: set a descriptive User-Agent header)",
+        "Flags no session reuse: `requests.get()` called in a loop creates a new connection for each page; recommends `requests.Session()` for connection pooling (Ch 10: use sessions for connection pooling)",
+        "Flags defensive parsing issues: `book.find('h3').find('a')['title']` will raise AttributeError if any element is missing; recommends checking for None before accessing attributes (Ch 2: parse defensively)",
+        "Flags no logging of progress or errors (Ch 5: log page fetches, errors, items extracted)"
+      ]
+    },
+    {
+      "id": "eval-02-regex-for-html-parsing",
+      "prompt": "Review this data extraction code:\n\n```python\nimport requests\nimport re\n\ndef extract_product_data(url: str) -> dict:\n    response = requests.get(url)\n    html = response.text\n\n    # Extract product name\n    name_match = re.search(r'<h1[^>]*>([^<]+)</h1>', html)\n    name = name_match.group(1) if name_match else None\n\n    # Extract price\n    price_match = re.search(r'<span class=\"price\">\\$([\\d\\.]+)</span>', html)\n    price = float(price_match.group(1)) if price_match else None\n\n    # Extract description paragraphs\n    desc_matches = re.findall(r'<p class=\"desc\">(.+?)</p>', html, re.DOTALL)\n    description = ' '.join(desc_matches)\n\n    # Extract all href links on the page\n    links = re.findall(r'href=[\"\\']([^\"\\']+)[\"\\']', html)\n\n    # Check if in stock\n    in_stock = bool(re.search(r'<span class=\"stock\">In Stock</span>', html))\n\n    return {\n        'name': name,\n        'price': price,\n        'description': description,\n        'links': links,\n        'in_stock': in_stock\n    }\n```",
+      "expectations": [
+        "Flags parsing HTML with regex as the primary anti-pattern: regex cannot reliably parse HTML because HTML is not a regular language; attribute order can vary, whitespace can differ, and nested tags break simple patterns (Ch 2: use BeautifulSoup or lxml, not regex, for HTML parsing)",
+        "Flags that the price regex `\\$([\\d\\.]+)` will fail silently on prices with commas (e.g., $1,299.99) or different currency formats without any warning (Ch 2: parse defensively)",
+        "Flags the description regex with `re.DOTALL` will incorrectly merge content from separate `<p>` tags that contain nested HTML tags like `<strong>` or `<a>` (Ch 2: regex cannot handle nested HTML)",
+        "Flags the link extraction regex `href=[\"\\']([^\"\\']+)[\"\\']` will match hrefs in script tags, style tags, and HTML comments, returning many false positives (Ch 2: use a parser with proper DOM traversal)",
+        "Flags no error handling on `requests.get()` and no status code check (Ch 1, 14: check response.raise_for_status())",
+        "Flags no session usage for connection pooling (Ch 10: use requests.Session())",
+        "Recommends replacing all regex parsing with BeautifulSoup CSS selectors or XPath, providing a corrected example using soup.select_one() and soup.select()"
+      ]
+    },
+    {
+      "id": "eval-03-clean-scraper-session-retry-css-selectors",
+      "prompt": "Review this web scraper:\n\n```python\nimport logging\nimport time\nfrom urllib.robotparser import RobotFileParser\nimport requests\nfrom requests.adapters import HTTPAdapter\nfrom urllib3.util.retry import Retry\nfrom bs4 import BeautifulSoup\n\nlogger = logging.getLogger(__name__)\n\nUSER_AGENT = 'ResearchBot/1.0 (contact: bot@example.com)'\nREQUEST_DELAY = 1.5  # seconds between requests\n\n\ndef build_session() -> requests.Session:\n    session = requests.Session()\n    session.headers['User-Agent'] = USER_AGENT\n    retry = Retry(\n        total=3,\n        backoff_factor=1,\n        status_forcelist=[429, 500, 502, 503, 504]\n    )\n    session.mount('https://', HTTPAdapter(max_retries=retry))\n    return session\n\n\ndef can_fetch(base_url: str, path: str) -> bool:\n    rp = RobotFileParser()\n    rp.set_url(f'{base_url}/robots.txt')\n    rp.read()\n    return rp.can_fetch(USER_AGENT, f'{base_url}{path}')\n\n\ndef parse_listing(html: str) -> list[dict]:\n    soup = BeautifulSoup(html, 'html.parser')\n    items = []\n    for card in soup.select('article.product-card'):\n        title_el = card.select_one('h2.product-title')\n        price_el = card.select_one('span.price')\n        if title_el is None or price_el is None:\n            logger.warning('Skipping card with missing elements')\n            continue\n        items.append({\n            'title': title_el.get_text(strip=True),\n            'price': price_el.get_text(strip=True),\n        })\n    return items\n\n\ndef scrape_category(base_url: str, category_path: str) -> list[dict]:\n    if not can_fetch(base_url, category_path):\n        logger.error('robots.txt disallows scraping %s', category_path)\n        return []\n\n    session = build_session()\n    all_items: list[dict] = []\n    page = 1\n\n    while True:\n        url = f'{base_url}{category_path}?page={page}'\n        try:\n            resp = session.get(url, timeout=10)\n            resp.raise_for_status()\n        except requests.RequestException as exc:\n            logger.error('Request failed for %s: %s', url, exc)\n            break\n\n        items = parse_listing(resp.text)\n        if not items:\n            break\n\n        logger.info('Page %d: extracted %d items', page, len(items))\n        all_items.extend(items)\n        page += 1\n        time.sleep(REQUEST_DELAY)\n\n    return all_items\n```",
+      "expectations": [
+        "Recognizes this is a well-structured, responsible scraper and says so explicitly",
+        "Praises robots.txt check via `RobotFileParser` before any requests are made (Ch 18: always check and honor robots.txt)",
+        "Praises the descriptive User-Agent with contact information making the bot identifiable (Ch 14: identify yourself with a descriptive User-Agent)",
+        "Praises `requests.Session()` with a `Retry` adapter providing automatic retry on transient server errors and rate-limit responses (Ch 14, 10: sessions with retry logic)",
+        "Praises CSS selectors via `soup.select()` and `soup.select_one()` instead of regex for HTML parsing (Ch 2: use BeautifulSoup CSS selectors)",
+        "Praises defensive None checks on extracted elements before accessing text, with a warning log for skipped cards (Ch 2: parse defensively)",
+        "Praises `resp.raise_for_status()` and catching `requests.RequestException` for all HTTP/network errors (Ch 1, 14: handle connection errors, timeouts, and HTTP errors)",
+        "Praises `time.sleep(REQUEST_DELAY)` between pages to be polite to the server (Ch 14: rate limit requests)",
+        "Praises structured logging of page number and item counts at each step (Ch 5: log progress)",
+        "Does NOT manufacture issues to appear thorough; any suggestions are explicitly framed as minor optional improvements"
+      ]
+    }
+  ]
+}

package/web-scraping-python/examples/after.md

@@ -0,0 +1,109 @@
+# After
+
+A scraper using `requests.Session` for connection reuse, `BeautifulSoup` for HTML parsing, per-request retry logic, and polite rate limiting between pages.
+
+```python
+import logging
+import time
+from dataclasses import dataclass
+
+import requests
+from bs4 import BeautifulSoup
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+logger = logging.getLogger(__name__)
+
+USER_AGENT = "JobResearchBot/1.0 (contact: scraping@mycompany.com)"
+REQUEST_DELAY_SECONDS = 2.0
+
+
+@dataclass
+class JobListing:
+    title: str
+    company: str
+    salary: str
+
+
+def make_session() -> requests.Session:
+    """Create a session with retry logic and a descriptive User-Agent."""
+    session = requests.Session()
+    session.headers.update({"User-Agent": USER_AGENT})
+
+    retry_policy = Retry(
+        total=3,
+        backoff_factor=1.5,
+        status_forcelist=[429, 500, 502, 503, 504],
+        allowed_methods=["GET"],
+    )
+    adapter = HTTPAdapter(max_retries=retry_policy)
+    session.mount("https://", adapter)
+    session.mount("http://", adapter)
+    return session
+
+
+def parse_job_listings(html: str) -> list[JobListing]:
+    """Extract job listings from a page of HTML using BeautifulSoup."""
+    soup = BeautifulSoup(html, "html.parser")
+    jobs = []
+
+    for card in soup.select("article.job-card"):
+        title_el   = card.select_one("h2.job-title")
+        company_el = card.select_one("span.company")
+        salary_el  = card.select_one("div.salary")
+
+        if title_el is None:
+            logger.debug("Skipping card with no title element")
+            continue
+
+        jobs.append(JobListing(
+            title=title_el.get_text(strip=True),
+            company=company_el.get_text(strip=True) if company_el else "",
+            salary=salary_el.get_text(strip=True) if salary_el else "Not specified",
+        ))
+
+    return jobs
+
+
+def scrape_jobs(base_url: str, num_pages: int) -> list[JobListing]:
+    """Scrape job listings across multiple pages with rate limiting."""
+    session = make_session()
+    all_jobs: list[JobListing] = []
+
+    for page in range(1, num_pages + 1):
+        url = f"{base_url}?page={page}"
+        logger.info("Fetching page %d: %s", page, url)
+
+        try:
+            response = session.get(url, timeout=15)
+            response.raise_for_status()
+        except requests.HTTPError as exc:
+            logger.error("HTTP error on page %d: %s", page, exc)
+            break
+        except requests.RequestException as exc:
+            logger.error("Request failed on page %d: %s — stopping", page, exc)
+            break
+
+        page_jobs = parse_job_listings(response.text)
+        logger.info("Extracted %d listings from page %d", len(page_jobs), page)
+        all_jobs.extend(page_jobs)
+
+        if page < num_pages:
+            time.sleep(REQUEST_DELAY_SECONDS)  # be polite
+
+    return all_jobs
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    jobs = scrape_jobs("https://jobs.example.com/listings", num_pages=20)
+    print(f"Total jobs scraped: {len(jobs)}")
+```
+
+Key improvements:
+- `requests.Session` with `HTTPAdapter` reuses TCP connections and retries on transient server errors — one session for all pages instead of a new connection per request (Ch 1, 14: Session reuse and retry)
+- `BeautifulSoup` with CSS selectors replaces regex HTML parsing — correct, readable, and resilient to attribute ordering changes (Ch 2: Use BeautifulSoup, not regex, for HTML)
+- `parse_job_listings` is a pure function that takes an HTML string and returns typed `JobListing` dataclasses — easily unit-tested with saved HTML fixtures (Ch 15: Testing scrapers)
+- `None` checks on each element before `.get_text()` prevent `AttributeError` when elements are missing (Ch 2: Defensive parsing)
+- `time.sleep(REQUEST_DELAY_SECONDS)` between pages respects the server; `USER_AGENT` identifies the bot with a contact address (Ch 14, 18: Rate limiting and identification)
+- Specific `requests.HTTPError` and `requests.RequestException` replace the bare `except` — errors are logged with page context and the crawl stops gracefully (Ch 1, 14: Error handling)

package/web-scraping-python/examples/before.md

@@ -0,0 +1,40 @@
+# Before
+
+A scraper that hammers a job listings site with no delays, parses HTML with regex, swallows all errors, and creates a new TCP connection for every page.
+
+```python
+import urllib.request
+import re
+
+def scrape_jobs(base_url, num_pages):
+    all_jobs = []
+
+    for page in range(1, num_pages + 1):
+        url = base_url + "?page=" + str(page)
+        try:
+            # New connection every request, no headers, no rate limiting
+            response = urllib.request.urlopen(url)
+            html = response.read().decode("utf-8")
+        except:
+            # Swallows every error — silent failures
+            continue
+
+        # Parsing HTML with regex — fragile and incorrect
+        titles = re.findall(r'<h2 class="job-title">(.*?)</h2>', html)
+        companies = re.findall(r'<span class="company">(.*?)</span>', html)
+        salaries = re.findall(r'<div class="salary">(.*?)</div>', html)
+
+        for i in range(len(titles)):
+            job = {
+                "title": titles[i],
+                "company": companies[i] if i < len(companies) else "",
+                "salary": salaries[i] if i < len(salaries) else "",
+            }
+            all_jobs.append(job)
+
+    return all_jobs
+
+
+jobs = scrape_jobs("https://jobs.example.com/listings", 20)
+print(f"Scraped {len(jobs)} jobs")
+```