@booklib/core 2.0.0

package/skills/using-asyncio-python/scripts/check_blocking.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+"""
+check_blocking.py — Static analyser for blocking calls inside async functions.
+
+Usage: python check_blocking.py <file_or_directory> [<file_or_directory> ...]
+
+Flags:
+  --exit-zero   Exit 0 even when issues are found (useful in CI to report only)
+  --summary     Print a summary table at the end
+"""
+
+import ast
+import argparse
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Iterator
+
+# ---------------------------------------------------------------------------
+# Rules
+# ---------------------------------------------------------------------------
+# Each rule is (description, fix_hint, matcher_function)
+# matcher_function(node) -> bool
+
+
+def _call_matches(node: ast.expr, *name_parts: str) -> bool:
+    """True if node is a Call whose function matches the dotted name."""
+    if not isinstance(node, ast.Call):
+        return False
+    func = node.func
+    # Simple name: open, sleep, etc.
+    if len(name_parts) == 1 and isinstance(func, ast.Name):
+        return func.id == name_parts[0]
+    # Attribute: requests.get, time.sleep, etc.
+    if len(name_parts) == 2 and isinstance(func, ast.Attribute):
+        obj = func.value
+        return isinstance(obj, ast.Name) and obj.id == name_parts[0] and func.attr == name_parts[1]
+    return False
+
+
+def _is_sync_open(node: ast.expr) -> bool:
+    """Flags open() calls that are not preceded by 'async with'."""
+    return _call_matches(node, "open")
+
+
+def _is_file_rw(node: ast.expr) -> bool:
+    """Flags .read() / .write() attribute calls (heuristic)."""
+    if not isinstance(node, ast.Call):
+        return False
+    func = node.func
+    return isinstance(func, ast.Attribute) and func.attr in {"read", "write", "readlines"}
+
+
+@dataclass
+class Rule:
+    id: str
+    description: str
+    fix: str
+    matcher: object  # callable(node) -> bool
+
+
+RULES: list[Rule] = [
+    Rule(
+        id="ASYNC001",
+        description="requests.get() blocks the event loop",
+        fix="Use aiohttp.ClientSession().get() or httpx.AsyncClient().get()",
+        matcher=lambda n: _call_matches(n, "requests", "get"),
+    ),
+    Rule(
+        id="ASYNC002",
+        description="requests.post() blocks the event loop",
+        fix="Use aiohttp.ClientSession().post() or httpx.AsyncClient().post()",
+        matcher=lambda n: _call_matches(n, "requests", "post"),
+    ),
+    Rule(
+        id="ASYNC003",
+        description="requests.put() blocks the event loop",
+        fix="Use aiohttp.ClientSession().put() or httpx.AsyncClient().put()",
+        matcher=lambda n: _call_matches(n, "requests", "put"),
+    ),
+    Rule(
+        id="ASYNC004",
+        description="requests.delete() blocks the event loop",
+        fix="Use aiohttp.ClientSession().delete() or httpx.AsyncClient().delete()",
+        matcher=lambda n: _call_matches(n, "requests", "delete"),
+    ),
+    Rule(
+        id="ASYNC005",
+        description="time.sleep() blocks the event loop",
+        fix="Use 'await asyncio.sleep(seconds)' instead",
+        matcher=lambda n: _call_matches(n, "time", "sleep"),
+    ),
+    Rule(
+        id="ASYNC006",
+        description="open() is a synchronous file operation",
+        fix="Use 'async with aiofiles.open(...)' from the aiofiles package",
+        matcher=_is_sync_open,
+    ),
+    Rule(
+        id="ASYNC007",
+        description="subprocess.run() blocks the event loop",
+        fix="Use 'await asyncio.create_subprocess_exec()' or asyncio.create_subprocess_shell()",
+        matcher=lambda n: _call_matches(n, "subprocess", "run"),
+    ),
+    Rule(
+        id="ASYNC008",
+        description="subprocess.call() blocks the event loop",
+        fix="Use 'await asyncio.create_subprocess_exec()' instead",
+        matcher=lambda n: _call_matches(n, "subprocess", "call"),
+    ),
+    Rule(
+        id="ASYNC009",
+        description=".read()/.write()/.readlines() on a synchronous file handle",
+        fix="Open the file with aiofiles and use 'await file.read()' / 'await file.write()'",
+        matcher=_is_file_rw,
+    ),
+]
+
+
+# ---------------------------------------------------------------------------
+# Finding
+# ---------------------------------------------------------------------------
+
+@dataclass
+class Finding:
+    file: Path
+    line: int
+    col: int
+    async_func: str
+    rule: Rule
+
+
+def _collect_async_funcs(tree: ast.AST) -> Iterator[ast.AsyncFunctionDef]:
+    """Yield all async def nodes in the tree, including nested ones."""
+    for node in ast.walk(tree):
+        if isinstance(node, ast.AsyncFunctionDef):
+            yield node
+
+
+def _nodes_inside_sync_context(func_node: ast.AsyncFunctionDef) -> set[int]:
+    """
+    Return the set of node ids that are inside a nested sync def or class,
+    so we don't flag blocking calls that are legitimately in sync helpers.
+    """
+    excluded: set[int] = set()
+    for node in ast.walk(func_node):
+        if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
+            for child in ast.walk(node):
+                excluded.add(id(child))
+    return excluded
+
+
+def check_file(path: Path) -> list[Finding]:
+    try:
+        source = path.read_text(encoding="utf-8", errors="replace")
+    except OSError as exc:
+        print(f"ERROR: Cannot read {path}: {exc}", file=sys.stderr)
+        return []
+
+    try:
+        tree = ast.parse(source, filename=str(path))
+    except SyntaxError as exc:
+        print(f"ERROR: Syntax error in {path}: {exc}", file=sys.stderr)
+        return []
+
+    findings: list[Finding] = []
+
+    for async_func in _collect_async_funcs(tree):
+        excluded = _nodes_inside_sync_context(async_func)
+        for node in ast.walk(async_func):
+            if id(node) in excluded:
+                continue
+            for rule in RULES:
+                if rule.matcher(node):
+                    findings.append(
+                        Finding(
+                            file=path,
+                            line=node.lineno,
+                            col=node.col_offset,
+                            async_func=async_func.name,
+                            rule=rule,
+                        )
+                    )
+    return findings
+
+
+def iter_python_files(path: Path) -> Iterator[Path]:
+    if path.is_file():
+        if path.suffix == ".py":
+            yield path
+    elif path.is_dir():
+        yield from sorted(path.rglob("*.py"))
+    else:
+        print(f"WARNING: {path} is not a file or directory — skipping.", file=sys.stderr)
+
+
+# ---------------------------------------------------------------------------
+# Reporting
+# ---------------------------------------------------------------------------
+
+def print_findings(findings: list[Finding]) -> None:
+    for f in findings:
+        print(
+            f"{f.file}:{f.line}:{f.col}: [{f.rule.id}] "
+            f"In 'async def {f.async_func}': {f.rule.description}"
+        )
+        print(f"  Fix: {f.rule.fix}")
+
+
+def print_summary(all_findings: list[Finding]) -> None:
+    if not all_findings:
+        print("\nSummary: No blocking call issues found.")
+        return
+
+    from collections import Counter
+    by_rule: Counter = Counter(f.rule.id for f in all_findings)
+    by_file: Counter = Counter(str(f.file) for f in all_findings)
+
+    print("\n--- Summary ---")
+    print(f"Total issues: {len(all_findings)}")
+    print("\nBy rule:")
+    for rule_id, count in sorted(by_rule.items()):
+        rule = next(r for r in RULES if r.id == rule_id)
+        print(f"  {rule_id}: {count}x  ({rule.description})")
+    print("\nBy file:")
+    for filepath, count in sorted(by_file.items()):
+        print(f"  {count:3d}  {filepath}")
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Find blocking calls inside async functions."
+    )
+    parser.add_argument(
+        "paths", nargs="+", type=Path, metavar="file_or_dir",
+        help="Python file(s) or director(ies) to analyse"
+    )
+    parser.add_argument(
+        "--exit-zero", action="store_true",
+        help="Always exit 0 (useful for non-blocking CI report)"
+    )
+    parser.add_argument(
+        "--summary", action="store_true",
+        help="Print a summary table after the findings"
+    )
+    args = parser.parse_args()
+
+    all_findings: list[Finding] = []
+    for raw_path in args.paths:
+        for py_file in iter_python_files(raw_path):
+            findings = check_file(py_file)
+            all_findings.extend(findings)
+            print_findings(findings)
+
+    if args.summary:
+        print_summary(all_findings)
+
+    if not all_findings:
+        print("No blocking call issues detected.")
+
+    if all_findings and not args.exit_zero:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/skills/using-asyncio-python/scripts/example.py

@@ -0,0 +1 @@
+

package/skills/web-scraping-python/SKILL.md

@@ -0,0 +1,280 @@
+---
+name: web-scraping-python
+version: "1.0"
+license: MIT
+tags: [python, web-scraping, data]
+description: >
+  Apply Web Scraping with Python practices (Ryan Mitchell). Covers First
+  Scrapers (Ch 1: urllib, BeautifulSoup), HTML Parsing (Ch 2: find, findAll,
+  CSS selectors, regex, lambda), Crawling (Ch 3-4: single-domain, cross-site,
+  crawl models), Scrapy (Ch 5: spiders, items, pipelines, rules), Storing Data
+  (Ch 6: CSV, MySQL, files, email), Reading Documents (Ch 7: PDF, Word,
+  encoding), Cleaning Data (Ch 8: normalization, OpenRefine), NLP (Ch 9: n-grams,
+  Markov, NLTK), Forms & Logins (Ch 10: POST, sessions, cookies), JavaScript
+  (Ch 11: Selenium, headless, Ajax), APIs (Ch 12: REST, undocumented), Image/OCR
+  (Ch 13: Pillow, Tesseract), Avoiding Traps (Ch 14: headers, honeypots),
+  Testing (Ch 15: unittest, Selenium), Parallel (Ch 16: threads, processes),
+  Remote (Ch 17: Tor, proxies), Legalities (Ch 18: robots.txt, CFAA, ethics).
+  Trigger on "web scraping", "BeautifulSoup", "Scrapy", "crawler", "spider",
+  "scraper", "parse HTML", "Selenium scraping", "data extraction".
+---
+
+# Web Scraping with Python Skill
+
+You are an expert web scraping engineer grounded in the 18 chapters from
+*Web Scraping with Python* (Collecting More Data from the Modern Web)
+by Ryan Mitchell. You help developers in two modes:
+
+1. **Scraper Building** — Design and implement web scrapers with idiomatic, production-ready patterns
+2. **Scraper Review** — Analyze existing scrapers against the book's practices and recommend improvements
+
+## How to Decide Which Mode
+
+- If the user asks to *build*, *create*, *scrape*, *extract*, *crawl*, or *collect* data → **Scraper Building**
+- If the user asks to *review*, *audit*, *improve*, *debug*, *optimize*, or *fix* a scraper → **Scraper Review**
+- If ambiguous, ask briefly which mode they'd prefer
+
+---
+
+## Mode 1: Scraper Building
+
+When designing or building web scrapers, follow this decision flow:
+
+### Step 1 — Understand the Requirements
+
+Ask (or infer from context):
+
+- **What target?** — Single page, single domain, multiple domains, API endpoints?
+- **What data?** — Text, tables, images, documents, forms, dynamic JavaScript content?
+- **What scale?** — One-off extraction, recurring crawl, large-scale parallel scraping?
+- **What challenges?** — Login required, JavaScript rendering, rate limiting, anti-bot measures?
+
+### Step 2 — Apply the Right Practices
+
+Read `references/practices-catalog.md` for the full chapter-by-chapter catalog. Quick decision guide:
+
+| Concern | Chapters to Apply |
+|---------|-------------------|
+| Basic page fetching and parsing | Ch 1: urllib/requests, BeautifulSoup setup, first scraper |
+| Finding elements in HTML | Ch 2: find/findAll, CSS selectors, navigating DOM trees, regex, lambda filters |
+| Crawling within a site | Ch 3: Following links, building crawlers, breadth-first vs depth-first |
+| Crawling across sites | Ch 4: Planning crawl models, handling different site layouts, normalizing data |
+| Framework-based scraping | Ch 5: Scrapy spiders, items, pipelines, rules, CrawlSpider, logging |
+| Saving scraped data | Ch 6: CSV, MySQL/database storage, downloading files, sending email |
+| Non-HTML documents | Ch 7: PDF text extraction, Word docs, encoding handling |
+| Data cleaning | Ch 8: String normalization, regex cleaning, OpenRefine, UTF-8 handling |
+| Text analysis on scraped data | Ch 9: N-grams, Markov models, NLTK, summarization |
+| Login-protected pages | Ch 10: POST requests, sessions, cookies, HTTP basic auth, handling tokens |
+| JavaScript-rendered pages | Ch 11: Selenium WebDriver, headless browsers, waiting for Ajax, executing JS |
+| Working with APIs | Ch 12: REST methods, JSON parsing, authentication, undocumented APIs |
+| Images and OCR | Ch 13: Pillow image processing, Tesseract OCR, CAPTCHA handling |
+| Avoiding detection | Ch 14: User-Agent headers, cookie handling, timing/delays, honeypot avoidance |
+| Testing scrapers | Ch 15: unittest for scrapers, Selenium-based testing, handling site changes |
+| Parallel scraping | Ch 16: Multithreading, multiprocessing, thread-safe queues |
+| Remote/anonymous scraping | Ch 17: Tor, proxies, rotating IPs, cloud-based scraping |
+| Legal and ethical concerns | Ch 18: robots.txt, Terms of Service, CFAA, copyright, ethical scraping |
+
+### Step 3 — Follow Web Scraping Principles
+
+Every scraper implementation should honor these principles:
+
+1. **Respect robots.txt** — Always check and honor robots.txt directives; be a good citizen of the web
+2. **Identify yourself** — Set a descriptive User-Agent string; consider providing contact info
+3. **Rate limit requests** — Add delays between requests (1-3 seconds minimum); never hammer servers
+4. **Handle errors gracefully** — Catch connection errors, timeouts, HTTP errors, and missing elements
+5. **Use sessions wisely** — Reuse HTTP sessions for connection pooling and cookie persistence
+6. **Parse defensively** — Never assume HTML structure is stable; use multiple selectors as fallbacks
+7. **Store raw data first** — Save raw HTML/responses before parsing; enables re-parsing without re-scraping
+8. **Validate extracted data** — Check for None/empty values; verify data types and formats
+9. **Design for re-runs** — Make scrapers idempotent; track what's already been scraped
+10. **Stay legal and ethical** — Understand applicable laws (CFAA, GDPR); respect Terms of Service
+
+### Step 4 — Build the Scraper
+
+Follow these guidelines:
+
+- **Production-ready** — Include error handling, retries, logging, rate limiting from the start
+- **Configurable** — Externalize URLs, selectors, delays, credentials; use config files or arguments
+- **Testable** — Write unit tests for parsing functions; integration tests for full scrape flows
+- **Observable** — Log page fetches, items extracted, errors encountered, timing stats
+- **Documented** — README with setup, usage, target site info, legal notes
+
+When building scrapers, produce:
+
+1. **Approach identification** — Which chapters/concepts apply and why
+2. **Target analysis** — Site structure, pagination, authentication needs, JS rendering
+3. **Implementation** — Production-ready code with error handling and rate limiting
+4. **Storage setup** — How and where data is stored (CSV, database, files)
+5. **Monitoring notes** — What to watch for (site changes, blocks, data quality)
+
+### Scraper Building Examples
+
+**Example 1 — Static Site Data Extraction:**
+```
+User: "Scrape product listings from an e-commerce category page"
+
+Apply: Ch 1 (fetching pages), Ch 2 (parsing product elements),
+       Ch 3 (pagination/crawling), Ch 6 (storing to CSV/DB)
+
+Generate:
+- requests + BeautifulSoup scraper
+- CSS selector-based product extraction
+- Pagination handler following next-page links
+- CSV or database storage with schema
+- Rate limiting and error handling
+```
+
+**Example 2 — JavaScript-Heavy Site:**
+```
+User: "Extract data from a React single-page application"
+
+Apply: Ch 11 (Selenium, headless browser), Ch 2 (parsing rendered HTML),
+       Ch 14 (avoiding detection), Ch 15 (testing)
+
+Generate:
+- Selenium WebDriver with headless Chrome
+- Explicit waits for dynamic content loading
+- JavaScript execution for scrolling/interaction
+- Data extraction from rendered DOM
+- Headless browser configuration
+```
+
+**Example 3 — Authenticated Scraping:**
+```
+User: "Scrape data from a site that requires login"
+
+Apply: Ch 10 (forms, sessions, cookies), Ch 14 (headers, tokens),
+       Ch 6 (data storage)
+
+Generate:
+- Session-based login with CSRF token handling
+- Cookie persistence across requests
+- POST request for form submission
+- Authenticated page navigation
+- Session expiry detection and re-login
+```
+
+**Example 4 — Large-Scale Crawl with Scrapy:**
+```
+User: "Build a crawler to scrape thousands of pages from multiple domains"
+
+Apply: Ch 5 (Scrapy framework), Ch 4 (crawl models),
+       Ch 16 (parallel scraping), Ch 14 (avoiding blocks)
+
+Generate:
+- Scrapy spider with item definitions and pipelines
+- CrawlSpider with Rule and LinkExtractor
+- Pipeline for database storage
+- Settings for concurrent requests, delays, user agents
+- Middleware for proxy rotation
+```
+
+---
+
+## Mode 2: Scraper Review
+
+When reviewing web scrapers, read `references/review-checklist.md` for the full checklist.
+
+### Review Process
+
+1. **Fetching scan** — Check Ch 1, 10, 11: HTTP method, session usage, JS rendering needs, authentication
+2. **Parsing scan** — Check Ch 2, 7: selector quality, defensive parsing, edge case handling
+3. **Crawling scan** — Check Ch 3-5: URL management, deduplication, pagination, depth control
+4. **Storage scan** — Check Ch 6: data format, schema, duplicates, file management
+5. **Resilience scan** — Check Ch 14-16: error handling, retries, rate limiting, parallel safety
+6. **Ethics scan** — Check Ch 17-18: robots.txt, legal compliance, identification, respectful crawling
+7. **Quality scan** — Check Ch 8, 15: data cleaning, testing, validation
+
+### Calibrating Review Tone
+
+**CRITICAL: Match your tone to what you actually find.**
+
+- If the scraper is well-structured and follows best practices, say so explicitly in the summary and spend the majority of the review praising what it does right. Specifically praise:
+  - `RobotFileParser` / robots.txt check before fetching (Ch 18)
+  - Descriptive User-Agent with contact info (Ch 14)
+  - `requests.Session()` with `Retry` adapter (Ch 10, 14)
+  - CSS selectors via `soup.select()` / `soup.select_one()` (Ch 2)
+  - Defensive None checks on extracted elements before accessing text (Ch 2)
+  - `resp.raise_for_status()` and catching `requests.RequestException` (Ch 1, 14)
+  - `time.sleep()` between requests (Ch 14)
+  - Structured logging of page number and item counts at each step (Ch 5)
+- Any suggestions on an already-good scraper MUST be framed as **minor optional improvements**, never as critical or high-priority issues. Do not manufacture severity.
+
+### Review Output Format
+
+Structure your review as:
+
+```
+## Summary
+One paragraph: overall scraper quality, pattern adherence, main concerns.
+
+## Fetching & Connection Issues
+For each issue (Ch 1, 10-11):
+- **Topic**: chapter and concept
+- **Location**: where in the code
+- **Problem**: what's wrong
+- **Fix**: recommended change with code snippet
+
+## Parsing & Extraction Issues
+For each issue (Ch 2, 7):
+- Same structure
+
+## Crawling & Navigation Issues
+For each issue (Ch 3-5):
+- Same structure
+
+## Storage & Data Issues
+For each issue (Ch 6, 8):
+- Same structure
+
+## Resilience & Performance Issues
+For each issue (Ch 14-16):
+- Same structure
+
+## Ethics & Legal Issues
+For each issue (Ch 17-18):
+- Same structure
+
+## Testing & Quality Issues
+For each issue (Ch 9, 15):
+- Same structure
+
+## Recommendations
+Priority-ordered from most critical to nice-to-have.
+Each recommendation references the specific chapter/concept.
+```
+
+### Common Web Scraping Anti-Patterns to Flag
+
+- **No error handling on requests** → Ch 1, 14: Wrap requests in try/except; handle `requests.RequestException` (covers ConnectionError, Timeout, HTTPError); always call `resp.raise_for_status()` to surface non-200 responses
+- **Hardcoded selectors without fallbacks** → Ch 2: Use multiple selector strategies; check for None before accessing attributes
+- **No rate limiting** → Ch 14: Add `time.sleep()` between requests; respect server resources
+- **Missing User-Agent header** → Ch 14: Set a descriptive User-Agent with contact info; rotate if needed for scale
+- **Not using sessions** → Ch 10: Use `requests.Session()` for cookie persistence and connection pooling
+- **Ignoring robots.txt** → Ch 18: Parse and respect robots.txt via `RobotFileParser` before crawling
+- **No URL deduplication** → Ch 3: Track visited URLs in a set; normalize URLs before comparing
+- **Using regex to parse HTML** → Ch 2: Use BeautifulSoup or lxml, not regex, for HTML parsing. In particular:
+  - `re.DOTALL` patterns on `<p>` or block elements will incorrectly merge content from nested inline tags (`<strong>`, `<a>`, etc.) producing wrong output
+  - Regex patterns like `href=["\'](.*?)["\']` will match `href` attributes inside `<script>` blocks, `<style>` blocks, and HTML comments, producing many false positives
+  - Recommend `soup.select_one()` and `soup.select()` CSS-selector API as the idiomatic BeautifulSoup replacement (preferred over `find()`/`find_all()` for clarity)
+- **Not handling JavaScript content** → Ch 11: If data loads via Ajax, use Selenium or find the underlying API
+- **Storing data without validation** → Ch 6, 8: Validate and clean data before storage; handle encoding
+- **No logging** → Ch 5: Log page fetches, item counts, and errors at each step; use structured logging with page number and item count per page
+- **Sequential when parallel is needed** → Ch 16: Use threading/multiprocessing for large-scale scraping
+- **Ignoring encoding issues** → Ch 7, 8: Handle UTF-8, detect encoding, normalize Unicode
+- **No tests for parsers** → Ch 15: Write unit tests with saved HTML fixtures; test selector robustness
+- **Credentials in code** → Ch 10: Use environment variables or config files for login credentials
+- **Not storing raw responses** → Ch 6: Save raw HTML for re-parsing; don't rely only on extracted data
+
+---
+
+## General Guidelines
+
+- **BeautifulSoup for simple scraping, Scrapy for scale** — Match the tool to the complexity
+- **Check for APIs first** — Many sites have APIs (documented or undocumented) that are easier than scraping
+- **Respect the site** — Rate limit, identify yourself, follow robots.txt, check ToS
+- **Parse defensively** — HTML structure changes; always handle missing elements gracefully
+- **Test with saved pages** — Save HTML fixtures and test parsers offline; reduces requests and enables CI
+- **Clean data early** — Normalize strings, handle encoding, strip whitespace at extraction time
+- For deeper practice details, read `references/practices-catalog.md` before building scrapers.
+- For review checklists, read `references/review-checklist.md` before reviewing scrapers.

package/skills/web-scraping-python/assets/example_asset.txt

@@ -0,0 +1 @@
+

package/skills/web-scraping-python/evals/evals.json

@@ -0,0 +1,46 @@
+{
+  "evals": [
+    {
+      "id": "eval-01-no-rate-limiting-no-error-handling-no-robots",
+      "prompt": "Review this web scraper:\n\n```python\nimport requests\nfrom bs4 import BeautifulSoup\nimport json\n\nBASE_URL = 'https://books.example.com'\n\ndef scrape_all_books():\n    all_books = []\n    page = 1\n\n    while True:\n        url = f'{BASE_URL}/catalogue/page-{page}.html'\n        response = requests.get(url)\n        soup = BeautifulSoup(response.text, 'html.parser')\n\n        books = soup.find_all('article', class_='product_pod')\n        if not books:\n            break\n\n        for book in books:\n            title = book.find('h3').find('a')['title']\n            price = book.find('p', class_='price_color').text\n            rating = book.find('p', class_='star-rating')['class'][1]\n            all_books.append({'title': title, 'price': price, 'rating': rating})\n\n        page += 1\n\n    return all_books\n\nresult = scrape_all_books()\nwith open('books.json', 'w') as f:\n    json.dump(result, f)\n```",
+      "expectations": [
+        "Flags no robots.txt check: the scraper does not check or respect the site's robots.txt before crawling (Ch 18: always check and honor robots.txt)",
+        "Flags no rate limiting: requests are issued as fast as possible with no delay between pages; recommends adding `time.sleep()` of at least 1-3 seconds between requests (Ch 14: rate limit requests)",
+        "Flags no error handling on `requests.get()`: a network error, timeout, or non-200 response will raise an exception or silently produce garbage HTML (Ch 1, 14: wrap requests in try/except, check response status)",
+        "Flags no User-Agent header: the scraper uses the default requests User-Agent which may be blocked and does not identify the bot (Ch 14: set a descriptive User-Agent header)",
+        "Flags no session reuse: `requests.get()` called in a loop creates a new connection for each page; recommends `requests.Session()` for connection pooling (Ch 10: use sessions for connection pooling)",
+        "Flags defensive parsing issues: `book.find('h3').find('a')['title']` will raise AttributeError if any element is missing; recommends checking for None before accessing attributes (Ch 2: parse defensively)",
+        "Flags no logging of progress or errors (Ch 5: log page fetches, errors, items extracted)"
+      ]
+    },
+    {
+      "id": "eval-02-regex-for-html-parsing",
+      "prompt": "Review this data extraction code:\n\n```python\nimport requests\nimport re\n\ndef extract_product_data(url: str) -> dict:\n    response = requests.get(url)\n    html = response.text\n\n    # Extract product name\n    name_match = re.search(r'<h1[^>]*>([^<]+)</h1>', html)\n    name = name_match.group(1) if name_match else None\n\n    # Extract price\n    price_match = re.search(r'<span class=\"price\">\\$([\\d\\.]+)</span>', html)\n    price = float(price_match.group(1)) if price_match else None\n\n    # Extract description paragraphs\n    desc_matches = re.findall(r'<p class=\"desc\">(.+?)</p>', html, re.DOTALL)\n    description = ' '.join(desc_matches)\n\n    # Extract all href links on the page\n    links = re.findall(r'href=[\"\\']([^\"\\']+)[\"\\']', html)\n\n    # Check if in stock\n    in_stock = bool(re.search(r'<span class=\"stock\">In Stock</span>', html))\n\n    return {\n        'name': name,\n        'price': price,\n        'description': description,\n        'links': links,\n        'in_stock': in_stock\n    }\n```",
+      "expectations": [
+        "Flags parsing HTML with regex as the primary anti-pattern: regex cannot reliably parse HTML because HTML is not a regular language; attribute order can vary, whitespace can differ, and nested tags break simple patterns (Ch 2: use BeautifulSoup or lxml, not regex, for HTML parsing)",
+        "Flags that the price regex `\\$([\\d\\.]+)` will fail silently on prices with commas (e.g., $1,299.99) or different currency formats without any warning (Ch 2: parse defensively)",
+        "Flags the description regex with `re.DOTALL` will incorrectly merge content from separate `<p>` tags that contain nested HTML tags like `<strong>` or `<a>` (Ch 2: regex cannot handle nested HTML)",
+        "Flags the link extraction regex `href=[\"\\']([^\"\\']+)[\"\\']` will match hrefs in script tags, style tags, and HTML comments, returning many false positives (Ch 2: use a parser with proper DOM traversal)",
+        "Flags no error handling on `requests.get()` and no status code check (Ch 1, 14: check response.raise_for_status())",
+        "Flags no session usage for connection pooling (Ch 10: use requests.Session())",
+        "Recommends replacing all regex parsing with BeautifulSoup CSS selectors or XPath, providing a corrected example using soup.select_one() and soup.select()"
+      ]
+    },
+    {
+      "id": "eval-03-clean-scraper-session-retry-css-selectors",
+      "prompt": "Review this web scraper:\n\n```python\nimport logging\nimport time\nfrom urllib.robotparser import RobotFileParser\nimport requests\nfrom requests.adapters import HTTPAdapter\nfrom urllib3.util.retry import Retry\nfrom bs4 import BeautifulSoup\n\nlogger = logging.getLogger(__name__)\n\nUSER_AGENT = 'ResearchBot/1.0 (contact: bot@example.com)'\nREQUEST_DELAY = 1.5  # seconds between requests\n\n\ndef build_session() -> requests.Session:\n    session = requests.Session()\n    session.headers['User-Agent'] = USER_AGENT\n    retry = Retry(\n        total=3,\n        backoff_factor=1,\n        status_forcelist=[429, 500, 502, 503, 504]\n    )\n    session.mount('https://', HTTPAdapter(max_retries=retry))\n    return session\n\n\ndef can_fetch(base_url: str, path: str) -> bool:\n    rp = RobotFileParser()\n    rp.set_url(f'{base_url}/robots.txt')\n    rp.read()\n    return rp.can_fetch(USER_AGENT, f'{base_url}{path}')\n\n\ndef parse_listing(html: str) -> list[dict]:\n    soup = BeautifulSoup(html, 'html.parser')\n    items = []\n    for card in soup.select('article.product-card'):\n        title_el = card.select_one('h2.product-title')\n        price_el = card.select_one('span.price')\n        if title_el is None or price_el is None:\n            logger.warning('Skipping card with missing elements')\n            continue\n        items.append({\n            'title': title_el.get_text(strip=True),\n            'price': price_el.get_text(strip=True),\n        })\n    return items\n\n\ndef scrape_category(base_url: str, category_path: str) -> list[dict]:\n    if not can_fetch(base_url, category_path):\n        logger.error('robots.txt disallows scraping %s', category_path)\n        return []\n\n    session = build_session()\n    all_items: list[dict] = []\n    page = 1\n\n    while True:\n        url = f'{base_url}{category_path}?page={page}'\n        try:\n            resp = session.get(url, timeout=10)\n            resp.raise_for_status()\n        except requests.RequestException as exc:\n            logger.error('Request failed for %s: %s', url, exc)\n            break\n\n        items = parse_listing(resp.text)\n        if not items:\n            break\n\n        logger.info('Page %d: extracted %d items', page, len(items))\n        all_items.extend(items)\n        page += 1\n        time.sleep(REQUEST_DELAY)\n\n    return all_items\n```",
+      "expectations": [
+        "Recognizes this is a well-structured, responsible scraper and says so explicitly",
+        "Praises robots.txt check via `RobotFileParser` before any requests are made (Ch 18: always check and honor robots.txt)",
+        "Praises the descriptive User-Agent with contact information making the bot identifiable (Ch 14: identify yourself with a descriptive User-Agent)",
+        "Praises `requests.Session()` with a `Retry` adapter providing automatic retry on transient server errors and rate-limit responses (Ch 14, 10: sessions with retry logic)",
+        "Praises CSS selectors via `soup.select()` and `soup.select_one()` instead of regex for HTML parsing (Ch 2: use BeautifulSoup CSS selectors)",
+        "Praises defensive None checks on extracted elements before accessing text, with a warning log for skipped cards (Ch 2: parse defensively)",
+        "Praises `resp.raise_for_status()` and catching `requests.RequestException` for all HTTP/network errors (Ch 1, 14: handle connection errors, timeouts, and HTTP errors)",
+        "Praises `time.sleep(REQUEST_DELAY)` between pages to be polite to the server (Ch 14: rate limit requests)",
+        "Praises structured logging of page number and item counts at each step (Ch 5: log progress)",
+        "Does NOT manufacture issues to appear thorough; any suggestions are explicitly framed as minor optional improvements"
+      ]
+    }
+  ]
+}

package/skills/web-scraping-python/evals/results.json

@@ -0,0 +1,13 @@
+{
+  "pass_rate": 0.958,
+  "passed": 23,
+  "total": 24,
+  "baseline_pass_rate": 0.375,
+  "baseline_passed": 9,
+  "baseline_total": 24,
+  "delta": 0.583,
+  "model": "default",
+  "evals_run": 3,
+  "date": "2026-03-28",
+  "non_standard_provider": true
+}