@fbraza/pi-cite 0.1.0 → 0.2.0

package/skills/literature/scripts/extract_experiments.py

@@ -0,0 +1,401 @@
+"""
+Preclinical Experiment Extraction Module
+
+Parse abstracts to extract structured in vitro and in vivo experiment details.
+Uses keyword-based extraction to identify cell lines, assays, animal models,
+endpoints, and key findings from each paper.
+"""
+
+import re
+import os
+from typing import List, Dict, Tuple
+import pandas as pd
+
+
+# ---------------------------------------------------------------------------
+# Keyword dictionaries
+# ---------------------------------------------------------------------------
+
+# In vitro indicators
+IN_VITRO_KEYWORDS = [
+    "cell line", "cell lines", "cell culture", "in vitro", "cultured cells",
+    "transfect", "transduct", "knockdown", "overexpress", "overexpression",
+    "siRNA", "shRNA", "CRISPR", "sgRNA",
+    "co-culture", "monolayer", "spheroid", "organoid",
+]
+
+# Common cell line names (case-insensitive matching handled separately)
+CELL_LINE_NAMES = [
+    "MCF-7", "MCF7", "MDA-MB-231", "MDA-MB-468", "T47D", "BT-474", "BT474",
+    "BT-549", "BT549", "MDA-MB-453", "CAL-51", "HCC1937", "HCC1806",
+    "SK-BR-3", "SKBR3", "ZR-75", "4T1", "EMT6",
+    "HeLa", "HEK293", "HEK-293", "293T", "HEK293T",
+    "A549", "H1299", "H460", "H1975", "PC9", "HCC827",
+    "HCT116", "HT29", "SW480", "SW620", "LoVo", "Caco-2",
+    "U87", "U251", "T98G", "LN229",
+    "PC3", "PC-3", "LNCaP", "DU145", "22Rv1", "VCaP",
+    "K562", "HL60", "HL-60", "Jurkat", "THP-1", "U937",
+    "HepG2", "Hep3B", "Huh7", "SMMC-7721",
+    "PANC-1", "MiaPaCa-2", "BxPC-3", "AsPC-1",
+    "A375", "SK-MEL-28", "B16", "B16F10",
+    "OVCAR3", "SKOV3", "A2780",
+    "CHO", "NIH3T3", "3T3", "COS-7",
+    "Raji", "Ramos", "Daudi",
+    "SH-SY5Y", "Neuro-2a", "N2a",
+    "RAW264.7", "RAW 264.7", "J774",
+]
+
+# Assay keyword categories
+ASSAY_KEYWORDS = {
+    "viability": [
+        "viability", "MTT", "CCK-8", "CCK8", "WST", "cell counting",
+        "CellTiter", "MTS", "XTT", "alamarBlue", "resazurin",
+        "cytotoxicity", "IC50", "EC50", "dose-response",
+    ],
+    "proliferation": [
+        "proliferation", "colony formation", "clonogenic", "BrdU", "EdU",
+        "Ki-67", "Ki67", "cell growth", "growth curve", "doubling time",
+    ],
+    "apoptosis": [
+        "apoptosis", "annexin", "caspase", "TUNEL", "cell death",
+        "sub-G1", "programmed cell death", "Bcl-2", "BAX",
+        "cleaved PARP", "cytochrome c release",
+    ],
+    "migration_invasion": [
+        "migration", "invasion", "wound healing", "transwell", "Boyden",
+        "scratch assay", "chemotaxis", "Matrigel",
+    ],
+    "gene_expression": [
+        "qPCR", "RT-PCR", "real-time PCR", "qRT-PCR",
+        "mRNA expression", "RNA-seq", "RNAseq", "transcriptom",
+        "gene expression", "Northern blot",
+    ],
+    "protein_analysis": [
+        "Western blot", "immunoblot", "ELISA", "immunoprecipitation",
+        "phosphorylation", "Co-IP", "pull-down", "mass spectrometry",
+        "proteomics", "immunofluorescence",
+    ],
+    "flow_cytometry": [
+        "flow cytometry", "FACS", "cell cycle", "cell sorting",
+        "intracellular staining", "surface marker",
+    ],
+    "reporter": [
+        "luciferase", "reporter assay", "GFP", "fluorescent reporter",
+        "dual-luciferase", "beta-galactosidase",
+    ],
+    "cell_signaling": [
+        "signaling assay", "signaling pathway analysis",
+        "phospho-", "kinase activity", "kinase assay",
+        "pathway activation assay", "phosphoproteomics",
+    ],
+}
+
+# In vivo indicators
+IN_VIVO_KEYWORDS = [
+    "in vivo", "mouse", "mice", "murine", "rat", "rats",
+    "xenograft", "allograft", "PDX", "patient-derived xenograft",
+    "orthotopic", "subcutaneous", "tumor-bearing",
+    "nude mice", "BALB/c", "C57BL/6", "SCID", "NSG", "NOD",
+    "transgenic", "knockout mice", "knock-in",
+    "animal model", "animal experiment", "preclinical model",
+]
+
+# Animal model categories
+ANIMAL_MODEL_KEYWORDS = {
+    "xenograft": [
+        "xenograft", "subcutaneous tumor", "subcutaneous implant",
+        "orthotopic implant", "orthotopic model",
+        "human tumor", "nude mice xenograft",
+    ],
+    "pdx": [
+        "PDX", "patient-derived xenograft", "patient-derived model",
+    ],
+    "syngeneic": [
+        "syngeneic", "allograft", "immunocompetent",
+        "4T1", "CT26", "B16", "MC38", "LLC", "EMT6",
+    ],
+    "transgenic": [
+        "transgenic", "knockout", "knock-in", "conditional knockout",
+        "Cre-lox", "GEMM", "genetically engineered",
+    ],
+    "metastasis": [
+        "metastasis model", "tail vein injection", "intracardiac",
+        "metastatic", "lung metastasis", "liver metastasis",
+        "spontaneous metastasis",
+    ],
+}
+
+# In vivo endpoint categories
+ENDPOINT_KEYWORDS = {
+    "tumor_growth": [
+        "tumor volume", "tumor growth", "tumor weight", "tumor size",
+        "tumor regression", "tumor inhibition", "anti-tumor",
+        "tumor growth inhibition", "TGI",
+    ],
+    "survival": [
+        "survival", "overall survival", "Kaplan-Meier",
+        "median survival", "survival rate", "lifespan",
+    ],
+    "biomarker": [
+        "biomarker", "serum level", "plasma level", "circulating",
+        "pharmacodynamic", "PD marker",
+    ],
+    "imaging": [
+        "bioluminescence", "in vivo imaging", "PET", "MRI", "CT scan",
+        "IVIS", "fluorescence imaging", "ultrasound",
+    ],
+    "histology": [
+        "histology", "immunohistochemistry", "IHC", "H&E",
+        "histopathology", "tissue staining", "TUNEL staining",
+    ],
+    "pharmacokinetics": [
+        "pharmacokinetic", "PK", "half-life", "bioavailability",
+        "AUC", "Cmax", "clearance", "distribution",
+    ],
+    "toxicity": [
+        "toxicity", "body weight", "adverse", "tolerability",
+        "maximum tolerated dose", "MTD", "safety", "organ toxicity",
+    ],
+}
+
+# Finding / result keywords
+FINDING_KEYWORDS = [
+    "significantly", "inhibited", "reduced", "suppressed", "attenuated",
+    "enhanced", "increased", "promoted", "induced", "abolished",
+    "demonstrated", "showed", "revealed", "observed",
+    "decreased", "elevated", "impaired", "restored", "abrogated",
+    "potentiated", "synergistic", "additive", "antagonistic",
+]
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def extract_all_experiments(
+    results: List[Dict],
+    output_dir: str = "preclinical_results"
+) -> List[Dict]:
+    """
+    Extract in vitro and in vivo experiment details from all paper abstracts.
+
+    Parameters
+    ----------
+    results : List[Dict]
+        Search results from preclinical_search
+    output_dir : str
+        Output directory for experiment_extraction.csv
+
+    Returns
+    -------
+    List[Dict]
+        List of experiment extraction dicts, one per paper
+
+    Verification
+    ------------
+    Prints "✓ Experiment extraction completed successfully!"
+    """
+    print("\n" + "=" * 70)
+    print("EXTRACTING EXPERIMENTS FROM ABSTRACTS")
+    print("=" * 70)
+
+    experiments = []
+    in_vitro_count = 0
+    in_vivo_count = 0
+    both_count = 0
+
+    for i, paper in enumerate(results):
+        abstract = paper.get("abstract", "")
+        title = paper.get("title", "")
+        text = f"{title} {abstract}"
+
+        # Classify experiment type
+        exp_type = _classify_experiment_type(text)
+
+        # Extract details
+        in_vitro = _extract_in_vitro(text)
+        in_vivo = _extract_in_vivo(text)
+        findings = _extract_findings(text)
+
+        experiment = {
+            "pmid": paper.get("pmid", ""),
+            "doi": paper.get("doi", ""),
+            "title": paper.get("title", ""),
+            "authors": paper.get("authors", ""),
+            "publication_date": paper.get("publication_date", ""),
+            "experiment_type": exp_type,
+            # In vitro details
+            "cell_lines": "; ".join(in_vitro["cell_lines"]) if in_vitro["cell_lines"] else "",
+            "assays": "; ".join(in_vitro["assays"]) if in_vitro["assays"] else "",
+            "in_vitro_findings": " | ".join(in_vitro["findings"][:3]) if in_vitro["findings"] else "",
+            # In vivo details
+            "animal_models": "; ".join(in_vivo["animal_models"]) if in_vivo["animal_models"] else "",
+            "endpoints": "; ".join(in_vivo["endpoints"]) if in_vivo["endpoints"] else "",
+            "in_vivo_findings": " | ".join(in_vivo["findings"][:3]) if in_vivo["findings"] else "",
+            # General findings
+            "key_findings": " | ".join(findings[:3]) if findings else "",
+        }
+
+        experiments.append(experiment)
+
+        if exp_type == "in_vitro":
+            in_vitro_count += 1
+        elif exp_type == "in_vivo":
+            in_vivo_count += 1
+        elif exp_type == "both":
+            both_count += 1
+
+    # Summary
+    unclassified = len(experiments) - in_vitro_count - in_vivo_count - both_count
+    print(f"\n  Processed {len(experiments)} papers:")
+    print(f"    In vitro only:  {in_vitro_count}")
+    print(f"    In vivo only:   {in_vivo_count}")
+    print(f"    Both:           {both_count}")
+    print(f"    Unclassified:   {unclassified}")
+
+    # Save CSV
+    os.makedirs(output_dir, exist_ok=True)
+    output_file = os.path.join(output_dir, "experiment_extraction.csv")
+    df = pd.DataFrame(experiments)
+    df.to_csv(output_file, index=False, encoding="utf-8")
+    print(f"\n  Saved extraction results to {output_file}")
+
+    print(f"\n✓ Experiment extraction completed successfully!")
+    return experiments
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+def _keyword_match(text: str, keywords: list, case_sensitive: bool = False) -> bool:
+    """Check if any keyword matches as a whole word in text."""
+    for kw in keywords:
+        if case_sensitive:
+            pattern = r'\b' + re.escape(kw) + r'\b'
+            if re.search(pattern, text):
+                return True
+        else:
+            pattern = r'\b' + re.escape(kw.lower()) + r'\b'
+            if re.search(pattern, text.lower()):
+                return True
+    return False
+
+
+def _classify_experiment_type(text: str) -> str:
+    """Classify whether a paper describes in vitro, in vivo, or both experiments."""
+    has_in_vitro = _keyword_match(text, IN_VITRO_KEYWORDS)
+    has_in_vivo = _keyword_match(text, IN_VIVO_KEYWORDS)
+
+    # Also check for cell line names (case-sensitive for some)
+    if not has_in_vitro:
+        has_in_vitro = any(re.search(r'\b' + re.escape(cl) + r'\b', text) for cl in CELL_LINE_NAMES)
+
+    if has_in_vitro and has_in_vivo:
+        return "both"
+    elif has_in_vitro:
+        return "in_vitro"
+    elif has_in_vivo:
+        return "in_vivo"
+    else:
+        return "unclassified"
+
+
+def _extract_in_vitro(text: str) -> Dict:
+    """Extract in vitro experiment details from text."""
+    result = {
+        "cell_lines": [],
+        "assays": [],
+        "findings": [],
+    }
+
+    # Detect cell lines (word boundary matching to avoid partial matches)
+    found_cell_lines = set()
+    for cl_name in CELL_LINE_NAMES:
+        pattern = r'\b' + re.escape(cl_name) + r'\b'
+        if re.search(pattern, text):
+            found_cell_lines.add(cl_name)
+
+    result["cell_lines"] = sorted(found_cell_lines)
+
+    # Detect assay types
+    found_assays = set()
+    for assay_category, keywords in ASSAY_KEYWORDS.items():
+        if _keyword_match(text, keywords):
+            found_assays.add(assay_category)
+
+    result["assays"] = sorted(found_assays)
+
+    # Extract in vitro finding sentences
+    result["findings"] = _extract_finding_sentences(text, IN_VITRO_KEYWORDS)
+
+    return result
+
+
+def _extract_in_vivo(text: str) -> Dict:
+    """Extract in vivo experiment details from text."""
+    result = {
+        "animal_models": [],
+        "endpoints": [],
+        "findings": [],
+    }
+
+    # Detect animal model types
+    found_models = set()
+    for model_category, keywords in ANIMAL_MODEL_KEYWORDS.items():
+        if _keyword_match(text, keywords):
+            found_models.add(model_category)
+
+    result["animal_models"] = sorted(found_models)
+
+    # Detect endpoints
+    found_endpoints = set()
+    for endpoint_category, keywords in ENDPOINT_KEYWORDS.items():
+        if _keyword_match(text, keywords):
+            found_endpoints.add(endpoint_category)
+
+    result["endpoints"] = sorted(found_endpoints)
+
+    # Extract in vivo finding sentences
+    result["findings"] = _extract_finding_sentences(text, IN_VIVO_KEYWORDS)
+
+    return result
+
+
+def _extract_findings(text: str) -> List[str]:
+    """Extract general key finding sentences from text."""
+    sentences = re.split(r'(?<=[.!?])\s+', text)
+    findings = []
+
+    for sentence in sentences:
+        sentence = sentence.strip()
+        if len(sentence.split()) < 8:
+            continue
+
+        if _keyword_match(sentence, FINDING_KEYWORDS):
+            if len(sentence) > 300:
+                sentence = sentence[:297] + "..."
+            findings.append(sentence)
+
+    return findings[:5]
+
+
+def _extract_finding_sentences(text: str, context_keywords: List[str]) -> List[str]:
+    """Extract finding sentences that also mention context keywords."""
+    sentences = re.split(r'(?<=[.!?])\s+', text)
+    findings = []
+
+    for sentence in sentences:
+        sentence = sentence.strip()
+        if len(sentence.split()) < 8:
+            continue
+
+        has_finding = _keyword_match(sentence, FINDING_KEYWORDS)
+        has_context = _keyword_match(sentence, context_keywords)
+
+        if has_finding and has_context:
+            if len(sentence) > 300:
+                sentence = sentence[:297] + "..."
+            findings.append(sentence)
+
+    return findings[:3]

package/skills/literature/scripts/generate_table.py

@@ -0,0 +1,96 @@
+"""Generate the unified per-paper summary table for the literature skill."""
+
+from __future__ import annotations
+
+import csv
+from typing import Dict, Iterable, List
+
+from synthesis import classify_evidence_quality, classify_study_type
+
+
+def _authors_year(paper: Dict) -> str:
+    authors = paper.get("authors") or []
+    if isinstance(authors, str):
+        authors = [a.strip() for a in authors.split(";") if a.strip()]
+    lead = authors[0] if authors else "Unknown"
+    if len(authors) > 1 and "et al." not in lead:
+        lead = f"{lead} et al."
+    year = paper.get("year") or paper.get("publication_date", "")[:4] or "n.d."
+    return f"{lead} ({year})"
+
+
+def _identifier(paper: Dict) -> str:
+    if paper.get("pmid"):
+        return f"PMID:{paper['pmid']}"
+    if paper.get("doi"):
+        return paper["doi"]
+    if paper.get("s2_id"):
+        return paper["s2_id"]
+    return "NA"
+
+
+def _truncate(text: str, limit: int = 160) -> str:
+    text = " ".join(str(text or "").split())
+    if len(text) <= limit:
+        return text
+    return text[: limit - 1].rstrip() + "…"
+
+
+def build_table_rows(papers: List[Dict], experiments: List[Dict] | None = None, mode: str = "general") -> List[Dict]:
+    experiment_map = {}
+    for exp in experiments or []:
+        key = exp.get("pmid") or exp.get("doi")
+        if key:
+            experiment_map[key] = exp
+
+    rows = []
+    for idx, paper in enumerate(papers, start=1):
+        key = paper.get("pmid") or paper.get("doi")
+        exp = experiment_map.get(key, {})
+        row = {
+            "#": idx,
+            "PMID/DOI": _identifier(paper),
+            "Authors (year)": _authors_year(paper),
+            "Key Message": _truncate(paper.get("tldr") or paper.get("title") or ""),
+            "Key Results": _truncate(paper.get("abstract") or exp.get("key_findings") or ""),
+            "Key Methods": _truncate(
+                "; ".join(filter(None, [
+                    ", ".join(paper.get("publication_types", [])[:3]) if isinstance(paper.get("publication_types"), list) else "",
+                    exp.get("assays", ""),
+                    exp.get("endpoints", ""),
+                ]))
+            ),
+            "Study Type": classify_study_type(paper),
+            "Evidence Quality": classify_evidence_quality(paper),
+        }
+        if mode == "preclinical":
+            row.update({
+                "Experiment Type": exp.get("experiment_type", ""),
+                "Model System": exp.get("cell_lines") or exp.get("animal_models") or "",
+                "Assay/Endpoint": "; ".join(filter(None, [exp.get("assays", ""), exp.get("endpoints", "")])),
+                "Finding Direction": exp.get("key_findings", ""),
+            })
+        rows.append(row)
+    return rows
+
+
+def rows_to_markdown(rows: Iterable[Dict]) -> str:
+    rows = list(rows)
+    if not rows:
+        return "## Paper Summary Table\n\n_No papers available._\n"
+    headers = list(rows[0].keys())
+    out = ["## Paper Summary Table", "", "| " + " | ".join(headers) + " |", "|" + "---|" * len(headers)]
+    for row in rows:
+        out.append("| " + " | ".join(str(row.get(header, "")).replace("\n", " ") for header in headers) + " |")
+    return "\n".join(out) + "\n"
+
+
+def write_csv(rows: Iterable[Dict], output_path: str) -> None:
+    rows = list(rows)
+    if not rows:
+        return
+    headers = list(rows[0].keys())
+    with open(output_path, "w", newline="", encoding="utf-8") as handle:
+        writer = csv.DictWriter(handle, fieldnames=headers)
+        writer.writeheader()
+        writer.writerows(rows)