@bgicli/bgicli 2.1.1 → 2.2.0

package/data/workflows/literature-preclinical/scripts/preclinical_search.py

@@ -0,0 +1,332 @@
+"""
+Preclinical Literature Search Module
+
+Search Consensus (consensus.app) for preclinical studies on a target in a disease.
+Returns structured paper metadata compatible with downstream extraction scripts.
+"""
+
+import time
+import os
+import random
+from typing import List, Dict, Optional
+import requests
+import pandas as pd
+
+
+def search_preclinical(
+    target: str,
+    disease: str,
+    max_results: int = 50,
+    years: int = 5,
+    output_dir: str = "preclinical_results"
+) -> List[Dict]:
+    """
+    Search for preclinical studies on a target in a disease context.
+
+    Uses the Consensus API (consensus.app) for fast, semantically-ranked
+    literature search. Requires CONSENSUS_API_KEY environment variable.
+
+    Parameters
+    ----------
+    target : str
+        Molecular target (e.g., "CDK4/6", "BRAF", "PD-L1")
+    disease : str
+        Disease context (e.g., "breast cancer", "melanoma")
+    max_results : int
+        Maximum results to retrieve (default: 50, API max per query: 50)
+    years : int
+        Search last N years (default: 5)
+    output_dir : str
+        Output directory for saving search results CSV
+
+    Returns
+    -------
+    List[Dict]
+        List of paper dictionaries with source, pmid, doi, title, authors,
+        journal, publication_date, abstract, keywords, url.
+
+    Verification
+    ------------
+    Prints "✓ Literature search completed successfully!"
+    """
+    print("=" * 70)
+    print("PRECLINICAL LITERATURE SEARCH")
+    print("=" * 70)
+    print(f"\n  Target:  {target}")
+    print(f"  Disease: {disease}")
+    print(f"  Years:   last {years}")
+    print(f"  Max:     {max_results} results")
+    print(f"  Backend: Consensus API (consensus.app)\n")
+
+    api_key = os.getenv("CONSENSUS_API_KEY")
+    if not api_key:
+        print("  ERROR: CONSENSUS_API_KEY environment variable not set.")
+        print("  Set it via: export CONSENSUS_API_KEY='your_key_here'")
+        print("  Get a key at: https://consensus.app/home/api/")
+        return []
+
+    import datetime
+    year_min = datetime.date.today().year - years
+
+    # Build queries — multiple phrasings to maximize coverage
+    queries = _build_queries(target, disease)
+    print(f"  Running {len(queries)} search queries...\n")
+
+    all_papers = []
+    for i, query in enumerate(queries):
+        remaining = max_results - len(all_papers)
+        if remaining <= 0:
+            break
+
+        limit = min(remaining, 50)  # API max is 50 per request
+        print(f"  Query {i+1}: \"{query}\" (limit={limit})")
+
+        papers = _query_consensus(
+            query=query,
+            api_key=api_key,
+            limit=limit,
+            year_min=year_min,
+        )
+        all_papers.extend(papers)
+        print(f"    → {len(papers)} results")
+
+    # Deduplicate
+    unique_papers = _deduplicate(all_papers)
+
+    # Sort by publication date (newest first)
+    unique_papers.sort(key=lambda x: x.get("publication_date", ""), reverse=True)
+
+    # Trim to max_results
+    unique_papers = unique_papers[:max_results]
+
+    # Save search results CSV
+    os.makedirs(output_dir, exist_ok=True)
+    _save_results_csv(unique_papers, os.path.join(output_dir, "preclinical_search_results.csv"))
+
+    print(f"\n✓ Literature search completed successfully!")
+    print(f"  Total unique papers: {len(unique_papers)}")
+    return unique_papers
+
+
+def _build_queries(target: str, disease: str) -> List[str]:
+    """Build multiple search queries for broader coverage."""
+    queries = [
+        f"{target} {disease} preclinical",
+        f"{target} {disease} in vitro in vivo",
+    ]
+    return queries
+
+
+def _query_consensus(
+    query: str,
+    api_key: str,
+    limit: int = 50,
+    year_min: Optional[int] = None,
+    max_retries: int = 3,
+) -> List[Dict]:
+    """
+    Query Consensus API and return papers in the standard format.
+
+    Parameters
+    ----------
+    query : str
+        Search query string
+    api_key : str
+        Consensus API key
+    limit : int
+        Max results (1-50)
+    year_min : int, optional
+        Exclude papers before this year
+    max_retries : int
+        Retry attempts on failure
+
+    Returns
+    -------
+    List[Dict]
+        Papers in the standard format expected by downstream scripts
+    """
+    url = "https://api.consensus.app/v1/quick_search"
+    headers = {"x-api-key": api_key, "Accept": "application/json"}
+
+    params = {
+        "query": query.strip(),
+        "limit": max(1, min(int(limit), 50)),
+    }
+    if year_min is not None:
+        params["year_min"] = year_min
+
+    delay = random.uniform(1, 3)
+
+    for attempt in range(1, max_retries + 1):
+        try:
+            response = requests.get(url, headers=headers, params=params, timeout=30)
+
+            if response.status_code == 200:
+                data = response.json()
+                if not isinstance(data, dict):
+                    print(f"    Unexpected response format")
+                    return []
+
+                results_list = data.get("results")
+                if not results_list or not isinstance(results_list, list):
+                    return []
+
+                papers = []
+                for item in results_list:
+                    if not isinstance(item, dict):
+                        continue
+                    paper = _parse_consensus_paper(item)
+                    if paper:
+                        papers.append(paper)
+                return papers
+
+            if response.status_code == 401:
+                print(f"    ERROR: Invalid or expired Consensus API key")
+                return []
+
+            if response.status_code == 429:
+                if attempt < max_retries:
+                    print(f"    Rate limited, retrying in {delay:.0f}s...")
+                    time.sleep(delay)
+                    delay = min(delay * 2, 60)
+                    continue
+                print(f"    Rate limit exceeded after {max_retries} attempts")
+                return []
+
+            print(f"    API error (HTTP {response.status_code})")
+            return []
+
+        except requests.exceptions.Timeout:
+            if attempt < max_retries:
+                print(f"    Timeout, retrying in {delay:.0f}s...")
+                time.sleep(delay)
+                delay = min(delay * 2, 60)
+                continue
+            print(f"    Timeout after {max_retries} attempts")
+            return []
+
+        except requests.exceptions.RequestException as e:
+            if attempt < max_retries:
+                time.sleep(delay)
+                delay = min(delay * 2, 60)
+                continue
+            print(f"    Connection error: {e}")
+            return []
+
+    return []
+
+
+def _parse_consensus_paper(item: dict) -> Optional[Dict]:
+    """
+    Parse a Consensus API result into the standard paper format.
+
+    Maps Consensus fields to the format expected by extract_experiments.py,
+    preclinical_synthesis.py, and other downstream scripts.
+    """
+    title = item.get("title", "")
+    if not title:
+        return None
+
+    # Authors — can be list of dicts or list of strings
+    authors_field = item.get("authors", [])
+    authors_str = ""
+    if isinstance(authors_field, list) and len(authors_field) > 0:
+        names = []
+        for a in authors_field[:10]:
+            if isinstance(a, dict):
+                name = a.get("name") or a.get("full_name") or ""
+            elif isinstance(a, str):
+                name = a
+            else:
+                name = ""
+            if name:
+                names.append(name)
+        authors_str = ", ".join(names)
+        if len(authors_field) > 10:
+            authors_str += " et al."
+
+    # Year and publication date
+    year = item.get("publish_year") or item.get("year") or ""
+    pub_date = f"{year}-01-01" if year else ""
+
+    # Journal
+    journal = item.get("journal_name") or item.get("journal") or ""
+    if journal and year:
+        journal_display = f"{journal} ({year})"
+    elif journal:
+        journal_display = journal
+    else:
+        journal_display = ""
+
+    # DOI and URL
+    doi = item.get("doi", "") or ""
+    consensus_url = item.get("url", "") or ""
+    if doi:
+        paper_url = f"https://doi.org/{doi}"
+    elif consensus_url:
+        paper_url = consensus_url
+    else:
+        paper_url = ""
+
+    # Abstract and takeaway
+    abstract = item.get("abstract", "") or ""
+    takeaway = item.get("takeaway", "") or ""
+
+    # Build keywords from study_type if available
+    keywords = item.get("study_type", "") or ""
+
+    return {
+        "source": "Consensus",
+        "pmid": "",
+        "doi": doi,
+        "title": title,
+        "authors": authors_str,
+        "journal": journal_display,
+        "publication_date": pub_date,
+        "abstract": abstract,
+        "takeaway": takeaway,
+        "keywords": keywords,
+        "url": paper_url,
+    }
+
+
+def _deduplicate(papers: List[Dict]) -> List[Dict]:
+    """Deduplicate papers by DOI and title."""
+    seen_dois = set()
+    seen_titles = set()
+    unique = []
+
+    for paper in papers:
+        doi = paper.get("doi", "").strip()
+        title = paper.get("title", "").strip().lower()
+
+        if doi and doi in seen_dois:
+            continue
+        if title and title in seen_titles:
+            continue
+
+        unique.append(paper)
+
+        if doi:
+            seen_dois.add(doi)
+        if title:
+            seen_titles.add(title)
+
+    return unique
+
+
+def _save_results_csv(results: List[Dict], output_file: str) -> None:
+    """Save search results to CSV."""
+    if not results:
+        print("  No results to save")
+        return
+
+    df = pd.DataFrame(results)
+
+    column_order = [
+        "source", "pmid", "doi", "title", "authors", "journal",
+        "publication_date", "keywords", "abstract", "takeaway", "url"
+    ]
+    df = df[[col for col in column_order if col in df.columns]]
+    df.to_csv(output_file, index=False, encoding="utf-8")
+    print(f"  Saved {len(df)} results to {output_file}")

package/data/workflows/literature-preclinical/scripts/preclinical_synthesis.py

@@ -0,0 +1,237 @@
+"""
+Preclinical Synthesis and Export Module
+
+Aggregate experiment extraction results across papers and export all outputs.
+
+Sub-modules:
+    narrative_synthesis — direction analysis, agreements/disagreements, gaps
+    report_generation   — markdown report, PDF conversion, URL helpers
+"""
+
+import os
+import pickle
+from collections import Counter
+from datetime import datetime
+from typing import List, Dict
+import pandas as pd
+
+# Import from sub-modules
+from narrative_synthesis import generate_narrative
+from report_generation import generate_markdown_report, generate_pdf_report
+
+
+def synthesize_preclinical(
+    results: List[Dict],
+    experiments: List[Dict],
+    target: str = "",
+) -> Dict:
+    """
+    Synthesize experiment extraction results across all papers.
+
+    Parameters
+    ----------
+    results : List[Dict]
+        Original search results (with abstracts for modality detection)
+    experiments : List[Dict]
+        Experiment extraction dicts from extract_all_experiments()
+    target : str
+        Target name for therapeutic direction inference
+
+    Returns
+    -------
+    Dict
+        Synthesis summary with frequency counts and breakdowns
+    """
+    print("\n" + "=" * 70)
+    print("SYNTHESIZING PRECLINICAL FINDINGS")
+    print("=" * 70)
+
+    # Experiment type breakdown
+    type_counter = Counter(e["experiment_type"] for e in experiments)
+    print(f"\n  Experiment type breakdown:")
+    for etype, count in type_counter.most_common():
+        print(f"    {etype}: {count}")
+
+    # Cell line frequency
+    cell_line_counter = Counter()
+    for e in experiments:
+        if e["cell_lines"]:
+            for cl in e["cell_lines"].split("; "):
+                cl = cl.strip()
+                if cl:
+                    cell_line_counter[cl] += 1
+
+    print(f"\n  Top cell lines ({len(cell_line_counter)} unique):")
+    for cl, count in cell_line_counter.most_common(10):
+        print(f"    {cl}: {count} papers")
+
+    # Assay frequency
+    assay_counter = Counter()
+    for e in experiments:
+        if e["assays"]:
+            for assay in e["assays"].split("; "):
+                assay = assay.strip()
+                if assay:
+                    assay_counter[assay] += 1
+
+    print(f"\n  Top assay types ({len(assay_counter)} unique):")
+    for assay, count in assay_counter.most_common(10):
+        print(f"    {assay}: {count} papers")
+
+    # Animal model frequency
+    model_counter = Counter()
+    for e in experiments:
+        if e["animal_models"]:
+            for model in e["animal_models"].split("; "):
+                model = model.strip()
+                if model:
+                    model_counter[model] += 1
+
+    print(f"\n  Top animal models ({len(model_counter)} unique):")
+    for model, count in model_counter.most_common(10):
+        print(f"    {model}: {count} papers")
+
+    # Endpoint frequency
+    endpoint_counter = Counter()
+    for e in experiments:
+        if e["endpoints"]:
+            for ep in e["endpoints"].split("; "):
+                ep = ep.strip()
+                if ep:
+                    endpoint_counter[ep] += 1
+
+    print(f"\n  Top endpoints ({len(endpoint_counter)} unique):")
+    for ep, count in endpoint_counter.most_common(10):
+        print(f"    {ep}: {count} papers")
+
+    # Papers with both in vitro and in vivo
+    both_papers = [e for e in experiments if e["experiment_type"] == "both"]
+    print(f"\n  Papers with both in vitro + in vivo: {len(both_papers)}")
+
+    # Publication year distribution
+    year_counter = Counter()
+    for e in experiments:
+        year = e.get("publication_date", "")[:4]
+        if year.isdigit():
+            year_counter[year] += 1
+
+    synthesis = {
+        "total_papers": len(experiments),
+        "experiment_type_breakdown": dict(type_counter),
+        "cell_line_frequency": dict(cell_line_counter.most_common(30)),
+        "assay_frequency": dict(assay_counter.most_common(20)),
+        "animal_model_frequency": dict(model_counter.most_common(15)),
+        "endpoint_frequency": dict(endpoint_counter.most_common(15)),
+        "year_distribution": dict(sorted(year_counter.items())),
+        "papers_with_both": len(both_papers),
+    }
+
+    # Generate narrative synthesis (delegated to narrative_synthesis module)
+    narrative = generate_narrative(synthesis, experiments, results, target)
+    synthesis["narrative"] = narrative
+
+    print(f"\n  Narrative synthesis:")
+    print(f"    Agreements:    {len(narrative['agreements'])} point(s)")
+    print(f"    Disagreements: {len(narrative['disagreements'])} point(s)")
+    print(f"    Gaps:          {len(narrative['gaps'])} point(s)")
+    td = narrative.get("therapeutic_direction", [])
+    if td:
+        print(f"    Therapeutic:   {td[0][:80]}...")
+
+    print(f"\n\u2713 Synthesis completed successfully!")
+    return synthesis
+
+
+def export_all(
+    results: List[Dict],
+    experiments: List[Dict],
+    synthesis: Dict,
+    target: str = "",
+    disease: str = "",
+    output_dir: str = "preclinical_results"
+) -> None:
+    """
+    Export all preclinical analysis results.
+
+    Saves:
+    1. preclinical_search_results.csv - All papers with metadata
+    2. experiment_extraction.csv - Per-paper experiment details
+    3. preclinical_synthesis_report.md - Structured markdown report
+    3b. preclinical_synthesis_report.pdf - PDF version (if deps available)
+    4. analysis_object.pkl - Pickle of all objects for downstream use
+
+    Parameters
+    ----------
+    results : List[Dict]
+        Original search results
+    experiments : List[Dict]
+        Experiment extraction results
+    synthesis : Dict
+        Synthesis summary
+    target : str
+        Target name for report header
+    disease : str
+        Disease name for report header
+    output_dir : str
+        Output directory
+
+    Verification
+    ------------
+    Prints "=== Export Complete ==="
+    """
+    os.makedirs(output_dir, exist_ok=True)
+
+    print("\n" + "=" * 70)
+    print("EXPORTING RESULTS")
+    print("=" * 70)
+
+    # 1. Search results CSV (may already exist from Step 1, re-save for completeness)
+    search_csv = os.path.join(output_dir, "preclinical_search_results.csv")
+    if results:
+        df_results = pd.DataFrame(results)
+        column_order = [
+            "source", "pmid", "doi", "title", "authors", "journal",
+            "publication_date", "keywords", "abstract", "url"
+        ]
+        df_results = df_results[[c for c in column_order if c in df_results.columns]]
+        df_results.to_csv(search_csv, index=False, encoding="utf-8")
+        print(f"  1. Saved: {search_csv} ({len(df_results)} papers)")
+
+    # 2. Experiment extraction CSV
+    extract_csv = os.path.join(output_dir, "experiment_extraction.csv")
+    if experiments:
+        df_exp = pd.DataFrame(experiments)
+        df_exp.to_csv(extract_csv, index=False, encoding="utf-8")
+        print(f"  2. Saved: {extract_csv} ({len(df_exp)} rows)")
+
+    # 3. Markdown synthesis report (delegated to report_generation module)
+    report_path = os.path.join(output_dir, "preclinical_synthesis_report.md")
+    generate_markdown_report(results, experiments, synthesis, target, disease, report_path)
+    print(f"  3. Saved: {report_path}")
+
+    # 3b. PDF report (delegated to report_generation module)
+    pdf_path = os.path.join(output_dir, "preclinical_synthesis_report.pdf")
+    if generate_pdf_report(report_path, pdf_path):
+        print(f"  3b. Saved: {pdf_path}")
+    else:
+        print(f"  3b. PDF skipped (install: pip install markdown weasyprint)")
+
+    # 4. Analysis object (pickle)
+    pkl_path = os.path.join(output_dir, "analysis_object.pkl")
+    analysis_object = {
+        "search_results": results,
+        "experiments": experiments,
+        "synthesis": synthesis,
+        "parameters": {
+            "target": target,
+            "disease": disease,
+            "n_papers": len(results),
+            "export_date": datetime.now().isoformat(),
+        }
+    }
+    with open(pkl_path, "wb") as f:
+        pickle.dump(analysis_object, f)
+    print(f"  4. Saved: {pkl_path}")
+    print(f"     (Load with: import pickle; obj = pickle.load(open('{pkl_path}', 'rb')))")
+
+    print(f"\n=== Export Complete ===")