@fbraza/pi-cite 0.1.0 → 0.2.0

package/skills/literature/scripts/scihub_pdf_resolver.py

@@ -0,0 +1,289 @@
+#!/usr/bin/env python3
+"""Resolve a DOI to a direct PDF URL through Sci-Hub.
+
+Zero dependencies. Python standard library only.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import http.client
+import http.cookiejar
+import json
+import os
+import re
+import sys
+from typing import Iterable
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote, urljoin, urlsplit, urlunsplit
+from urllib.request import (
+    HTTPCookieProcessor,
+    HTTPRedirectHandler,
+    Request,
+    build_opener,
+)
+
+TIMEOUT = 20
+STATUS_FOUND = "FOUND"
+STATUS_NOT_FOUND = "NOT_FOUND"
+STATUS_MIRROR_ERROR = "MIRROR_ERROR"
+STATUS_INVALID_INPUT = "INVALID_INPUT"
+DEFAULT_MIRRORS = (
+    "https://sci-hub.st",
+    "https://sci-hub.ru",
+    "https://sci-hub.se",
+)
+UA = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/133.0.0.0 Safari/537.36"
+)
+PDF_PATTERNS = (
+    re.compile(r'<(?:iframe|embed|object)[^>]+(?:src|data)=["\']([^"\']+)["\']', re.I),
+    re.compile(r'["\']((?:https?:)?//[^"\']+?(?:\.pdf|/pdf)[^"\']*)["\']', re.I),
+)
+OA_HINT_PATTERN = re.compile(
+    r'<block-rounded[^>]+class\s*=\s*["\'][^"\']*\bopenaccess\b[^"\']*["\'][^>]*>(?:(?!</block-rounded>).)*?<a[^>]+href\s*=\s*["\']([^"\']+)["\']',
+    re.I | re.S,
+)
+
+
+class Browser:
+    def __init__(self) -> None:
+        jar = http.cookiejar.CookieJar()
+        self.opener = build_opener(HTTPCookieProcessor(jar), HTTPRedirectHandler())
+
+    def open(
+        self,
+        url: str,
+        *,
+        data: bytes | None = None,
+        headers: dict[str, str] | None = None,
+    ) -> http.client.HTTPResponse:
+        req = Request(url, data=data, headers=headers or {})
+        return self.opener.open(req, timeout=TIMEOUT)
+
+
+def _headers(extra: dict[str, str] | None = None) -> dict[str, str]:
+    base = {
+        "User-Agent": UA,
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+    if extra:
+        base.update(extra)
+    return base
+
+
+def _canonicalize(url: str) -> str:
+    parts = urlsplit(url.strip().replace("\\/", "/"))
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, parts.query, ""))
+
+
+def _normalize_doi(raw: str) -> str:
+    doi = raw.strip()
+    doi = re.sub(r"^(?:doi:\s*)", "", doi, flags=re.I)
+    doi = re.sub(r"^https?://(?:dx\.)?doi\.org/", "", doi, flags=re.I)
+    return doi.strip()
+
+
+def _extract_title(html: str) -> str:
+    match = re.search(r"<title[^>]*>(.*?)</title>", html, re.I | re.S)
+    if not match:
+        return ""
+    return " ".join(match.group(1).split())
+
+
+def _iter_pdf_candidates(html: str, page_url: str) -> Iterable[str]:
+    seen: set[str] = set()
+    for pattern in PDF_PATTERNS:
+        for raw in pattern.findall(html):
+            candidate = raw.strip()
+            if not candidate:
+                continue
+            if candidate.startswith("//"):
+                candidate = f"https:{candidate}"
+            else:
+                candidate = urljoin(page_url, candidate)
+            candidate = _canonicalize(candidate)
+            if candidate in seen:
+                continue
+            seen.add(candidate)
+            yield candidate
+
+
+def _has_altcha(html: str) -> bool:
+    return bool(re.search(r"/captcha/challenge/\d+", html))
+
+
+def _hexdigest(data: str, algorithm: str) -> str:
+    digest = hashlib.new(algorithm.strip().lower().replace("-", ""))
+    digest.update(data.encode("utf-8"))
+    return digest.hexdigest()
+
+
+def _solve_altcha(browser: Browser, page_url: str, html: str) -> bool:
+    challenge_id = re.search(r"/captcha/challenge/(\d+)", html)
+    if not challenge_id:
+        return False
+    parts = urlsplit(page_url)
+    base_url = f"{parts.scheme}://{parts.netloc}"
+    challenge_url = urljoin(base_url, f"/captcha/challenge/{challenge_id.group(1)}")
+    solution_url = urljoin(base_url, f"/captcha/solution/{challenge_id.group(1)}")
+    try:
+        with browser.open(challenge_url, headers=_headers({"Accept": "application/json"})) as resp:
+            challenge = json.loads(resp.read().decode("utf-8", errors="replace"))
+        algorithm = str(challenge["algorithm"])
+        salt = str(challenge["salt"])
+        target = str(challenge["challenge"])
+        max_number = int(challenge["maxNumber"])
+    except (HTTPError, URLError, OSError, KeyError, TypeError, ValueError, json.JSONDecodeError):
+        return False
+    number = None
+    try:
+        for value in range(max_number + 1):
+            if _hexdigest(f"{salt}{value}", algorithm) == target:
+                number = value
+                break
+    except ValueError:
+        return False
+    if number is None:
+        return False
+    payload = base64.b64encode(
+        json.dumps(
+            {
+                "algorithm": algorithm,
+                "challenge": target,
+                "number": number,
+                "salt": salt,
+                "signature": challenge.get("signature", ""),
+                "took": 0,
+            },
+            separators=(",", ":"),
+        ).encode("utf-8")
+    ).decode("ascii")
+    body = json.dumps({"captcha": payload}).encode("utf-8")
+    try:
+        with browser.open(
+            solution_url,
+            data=body,
+            headers=_headers(
+                {
+                    "Content-Type": "application/json",
+                    "Accept": "application/json",
+                    "Origin": base_url,
+                    "Referer": page_url,
+                }
+            ),
+        ) as resp:
+            response = json.loads(resp.read().decode("utf-8", errors="replace"))
+    except (HTTPError, URLError, OSError, TypeError, ValueError, json.JSONDecodeError):
+        return False
+    return bool(response.get("success"))
+
+
+def _fetch_page(browser: Browser, doi_url: str) -> tuple[str, str]:
+    current_url = doi_url
+    for _ in range(3):
+        with browser.open(current_url, headers=_headers()) as resp:
+            final_url = resp.geturl()
+            html = resp.read().decode("utf-8", errors="replace")
+        if not _has_altcha(html):
+            return final_url, html
+        if not _solve_altcha(browser, final_url, html):
+            break
+        current_url = doi_url
+    return "", ""
+
+
+def _is_pdf(browser: Browser, url: str) -> bool:
+    headers = _headers(
+        {
+            "Accept": "application/pdf,*/*;q=0.8",
+            "Range": "bytes=0-7",
+        }
+    )
+    try:
+        with browser.open(url, headers=headers) as resp:
+            content_type = (resp.headers.get("Content-Type") or "").lower()
+            if "application/pdf" in content_type:
+                return True
+            prefix = resp.read(8)
+            return prefix.startswith(b"%PDF-")
+    except (HTTPError, URLError, OSError):
+        return False
+
+
+def _extract_oa_link(html: str, page_url: str) -> str:
+    match = OA_HINT_PATTERN.search(html)
+    if not match:
+        return ""
+    candidate = match.group(1).strip()
+    if not candidate:
+        return ""
+    if candidate.startswith("//"):
+        candidate = f"https:{candidate}"
+    else:
+        candidate = urljoin(page_url, candidate)
+    return _canonicalize(candidate)
+
+
+def _mirror_list() -> tuple[str, ...]:
+    raw = os.environ.get("SCIHUB_MIRRORS", "")
+    if raw.strip():
+        return tuple(item.strip().rstrip("/") for item in raw.split(",") if item.strip())
+    return DEFAULT_MIRRORS
+
+
+def resolve_pdf(doi: str) -> tuple[str, str]:
+    normalized = _normalize_doi(doi)
+    if not normalized:
+        return STATUS_INVALID_INPUT, ""
+    safe_doi = quote(normalized, safe="/:().-_")
+    saw_not_found = False
+    saw_mirror_error = False
+    oa_link = ""
+    for mirror in _mirror_list():
+        browser = Browser()
+        try:
+            page_url, html = _fetch_page(browser, f"{mirror}/{safe_doi}")
+        except (HTTPError, URLError, OSError):
+            saw_mirror_error = True
+            continue
+        if not html:
+            saw_mirror_error = True
+            continue
+        title = _extract_title(html).lower()
+        if "not available through sci-hub" in title or "no articles found" in title:
+            saw_not_found = True
+            if not oa_link:
+                oa_link = _extract_oa_link(html, page_url)
+            continue
+        for candidate in _iter_pdf_candidates(html, page_url):
+            if _is_pdf(browser, candidate):
+                return STATUS_FOUND, candidate
+        saw_mirror_error = True
+    if saw_not_found:
+        return STATUS_NOT_FOUND, oa_link
+    if saw_mirror_error:
+        return STATUS_MIRROR_ERROR, ""
+    return STATUS_NOT_FOUND, ""
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: scihub-paper-downloader.py <DOI>", file=sys.stderr)
+        sys.exit(1)
+    status, url = resolve_pdf(sys.argv[1])
+    if status == STATUS_FOUND:
+        print(url)
+        sys.exit(0)
+    print(status)
+    if status == STATUS_NOT_FOUND and url:
+        print(f"OA_LINK {url}")
+    if status == STATUS_NOT_FOUND:
+        sys.exit(1)
+    if status == STATUS_MIRROR_ERROR:
+        sys.exit(2)
+    sys.exit(3)

package/skills/literature/scripts/synthesis.py

@@ -0,0 +1,93 @@
+"""
+Unified literature synthesis helpers.
+
+This module combines general literature review summarisation with the
+preclinical extraction summary used by the legacy merged literature workflow.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Dict, List, Optional
+
+
+def classify_study_type(paper: Dict) -> str:
+    publication_types = [str(x).lower() for x in paper.get("publication_types", [])]
+    text = " ".join(publication_types)
+    if "meta-analysis" in text or "systematic review" in text:
+        return "Systematic review / meta-analysis"
+    if "randomized controlled trial" in text:
+        return "Randomized controlled trial"
+    if "clinical trial" in text:
+        return "Clinical study"
+    if paper.get("is_preprint"):
+        return "Preprint"
+
+    abstract = f"{paper.get('title', '')} {paper.get('abstract', '')}".lower()
+    if any(x in abstract for x in ["xenograft", "mouse", "mice", "in vivo"]):
+        if any(y in abstract for y in ["cell line", "in vitro", "organoid"]):
+            return "In vitro + in vivo"
+        return "In vivo"
+    if any(x in abstract for x in ["cell line", "in vitro", "organoid", "crispr"]):
+        return "In vitro"
+    return "Observational / other"
+
+
+def classify_evidence_quality(paper: Dict) -> str:
+    study_type = classify_study_type(paper)
+    citation_count = int(paper.get("citation_count") or 0)
+    if study_type in {"Systematic review / meta-analysis", "Randomized controlled trial"}:
+        return "High"
+    if study_type in {"Clinical study", "In vitro + in vivo"}:
+        return "Moderate"
+    if paper.get("is_preprint"):
+        return "Preliminary (preprint)"
+    if study_type in {"In vivo", "In vitro"}:
+        return "Moderate" if citation_count >= 20 else "Low to moderate"
+    return "Preliminary"
+
+
+def summarize_papers(papers: List[Dict]) -> Dict:
+    study_types = Counter(classify_study_type(p) for p in papers)
+    evidence = Counter(classify_evidence_quality(p) for p in papers)
+    years = [int(p.get("year")) for p in papers if str(p.get("year", "")).isdigit()]
+    return {
+        "total_papers": len(papers),
+        "study_type_breakdown": dict(study_types),
+        "evidence_quality_breakdown": dict(evidence),
+        "year_range": [min(years), max(years)] if years else None,
+    }
+
+
+def generate_narrative(papers: List[Dict], topic: str = "") -> str:
+    summary = summarize_papers(papers)
+    lead = f"Literature synthesis for **{topic}**." if topic else "Literature synthesis."
+    lines = [lead, "", f"- Papers reviewed: {summary['total_papers']}"]
+    if summary["year_range"]:
+        lines.append(f"- Year range: {summary['year_range'][0]}-{summary['year_range'][1]}")
+    if summary["study_type_breakdown"]:
+        lines.append("- Study types: " + ", ".join(f"{k} ({v})" for k, v in summary["study_type_breakdown"].items()))
+    if summary["evidence_quality_breakdown"]:
+        lines.append("- Evidence quality: " + ", ".join(f"{k} ({v})" for k, v in summary["evidence_quality_breakdown"].items()))
+
+    top_titles = [p.get("title", "Untitled") for p in papers[:5]]
+    if top_titles:
+        lines.extend(["", "Top prioritised papers:"])
+        lines.extend([f"{i + 1}. {title}" for i, title in enumerate(top_titles)])
+    return "\n".join(lines)
+
+
+def synthesize_literature(
+    papers: List[Dict],
+    experiments: Optional[List[Dict]] = None,
+    topic: str = "",
+    mode: str = "general",
+) -> Dict:
+    summary = summarize_papers(papers)
+    summary["mode"] = mode
+    summary["topic"] = topic
+    summary["narrative_markdown"] = generate_narrative(papers, topic=topic)
+    if experiments:
+        summary["experiment_type_breakdown"] = dict(Counter(e.get("experiment_type", "unclassified") for e in experiments))
+        summary["model_systems"] = dict(Counter(filter(None, [e.get("cell_lines") or e.get("animal_models") for e in experiments])))
+    return summary