@bgicli/bgicli 2.1.1 → 2.2.0

package/data/workflows/chip-atlas-diff-analysis/scripts/query_chipatlas_api.py

@@ -0,0 +1,329 @@
+"""
+Query the ChIP-Atlas Diff Analysis API.
+
+Submits two sets of experiment IDs to the official ChIP-Atlas WABI API,
+which performs differential peak (DPR via edgeR) or differentially methylated
+region (DMR via metilene) analysis between the two groups.
+Results are returned as a ZIP archive containing BED and IGV files.
+
+API endpoint: POST https://dtn1.ddbj.nig.ac.jp/wabi/chipatlas/
+Reference: Zou et al. 2024 (NAR), https://chip-atlas.org
+"""
+
+import io
+import os
+import re
+import time
+import zipfile
+
+import requests
+
+# API endpoint (shared with enrichment analysis)
+WABI_API_URL = "https://dtn1.ddbj.nig.ac.jp/wabi/chipatlas/"
+
+# Valid analysis types
+VALID_ANALYSIS_TYPES = {
+    "diffbind": "Differential Peak Regions (ChIP/ATAC/DNase-seq)",
+    "dmr": "Differentially Methylated Regions (Bisulfite-seq)",
+}
+
+# Valid genomes (same as enrichment)
+VALID_GENOMES = [
+    "hg38", "hg19", "mm10", "mm9", "rn6",
+    "dm6", "dm3", "ce11", "ce10", "sacCer3",
+]
+
+# Experiment ID pattern
+EXPERIMENT_ID_PATTERN = re.compile(r"^(SRX|ERX|DRX|GSM)\d+$")
+
+
+def submit_diff_job(
+    experiments_a,
+    experiments_b,
+    genome="hg38",
+    analysis_type="diffbind",
+    title="diff_analysis",
+    description_a="group_A",
+    description_b="group_B",
+):
+    """
+    Submit differential analysis job to ChIP-Atlas API.
+
+    Parameters
+    ----------
+    experiments_a : list of str
+        Experiment IDs for group A (e.g., ['SRX123456', 'SRX123457'])
+    experiments_b : list of str
+        Experiment IDs for group B
+    genome : str
+        Genome assembly (hg38, hg19, mm10, etc.)
+    analysis_type : str
+        'diffbind' for DPR (ChIP/ATAC/DNase-seq) or 'dmr' for DMR (Bisulfite-seq)
+    title : str
+        Analysis title
+    description_a : str
+        Label for dataset A
+    description_b : str
+        Label for dataset B
+
+    Returns
+    -------
+    str
+        Request ID for polling
+
+    Raises
+    ------
+    ValueError
+        If inputs are invalid
+    RuntimeError
+        If API submission fails
+    """
+    # Validate inputs
+    if not experiments_a or len(experiments_a) < 2:
+        raise ValueError("Group A requires at least 2 experiment IDs")
+    if not experiments_b or len(experiments_b) < 2:
+        raise ValueError("Group B requires at least 2 experiment IDs")
+    if genome not in VALID_GENOMES:
+        raise ValueError(f"Invalid genome '{genome}'. Valid: {VALID_GENOMES}")
+    if analysis_type not in VALID_ANALYSIS_TYPES:
+        raise ValueError(
+            f"Invalid analysis_type '{analysis_type}'. "
+            f"Valid: {list(VALID_ANALYSIS_TYPES.keys())}"
+        )
+
+    # Validate experiment IDs
+    for group_name, ids in [("A", experiments_a), ("B", experiments_b)]:
+        for exp_id in ids:
+            if not EXPERIMENT_ID_PATTERN.match(exp_id):
+                print(f"   Warning: ID '{exp_id}' in group {group_name} "
+                      f"doesn't match expected format (SRX/ERX/DRX/GSM)")
+
+    bed_a_content = "\n".join(experiments_a)
+    bed_b_content = "\n".join(experiments_b)
+
+    data = {
+        "address": "",          # Hidden but required (from enrichment debugging)
+        "format": "text",       # Diff analysis uses text, not json
+        "result": "www",        # Hidden but required
+        "genome": genome,
+        "antigenClass": analysis_type,
+        "cellClass": "empty",   # Fixed for diff analysis
+        "threshold": "50",      # Fixed for diff analysis
+        "typeA": "srx",
+        "bedAFile": bed_a_content,
+        "typeB": "srx",
+        "bedBFile": bed_b_content,
+        "permTime": "1",
+        "title": title,
+        "descriptionA": description_a,
+        "descriptionB": description_b,
+    }
+
+    try:
+        response = requests.post(WABI_API_URL, data=data, timeout=60)
+        response.raise_for_status()
+
+        # Try JSON parsing first (in case API accepts format=text but returns JSON)
+        try:
+            result = response.json()
+            if "requestId" in result:
+                return result["requestId"]
+            error_msg = result.get("error-message", result.get("Message", ""))
+            if error_msg:
+                raise RuntimeError(f"API submission failed: {error_msg}")
+        except (ValueError, KeyError):
+            pass
+
+        # Parse text response: look for "requestId: ..." line
+        text = response.text.strip()
+        for line in text.split("\n"):
+            line = line.strip()
+            if line.startswith("requestId"):
+                # Handle both "requestId: value" and "requestId\tvalue"
+                request_id = re.split(r"[:\t]\s*", line, maxsplit=1)
+                if len(request_id) > 1 and request_id[1].strip():
+                    return request_id[1].strip()
+
+        raise RuntimeError(
+            f"Could not parse request ID from API response:\n{text[:500]}"
+        )
+
+    except requests.exceptions.RequestException as e:
+        raise RuntimeError(
+            f"Failed to connect to ChIP-Atlas API: {e}. "
+            f"Check internet connection and https://chip-atlas.org availability."
+        )
+
+
+def poll_job_status(request_id, timeout=900, poll_interval=15):
+    """
+    Poll job until finished or timeout.
+
+    Parameters
+    ----------
+    request_id : str
+        Request ID from submit_diff_job()
+    timeout : int
+        Maximum seconds to wait (default: 900 = 15 min)
+    poll_interval : int
+        Seconds between polls (default: 15)
+
+    Returns
+    -------
+    str
+        Final status ("finished")
+
+    Raises
+    ------
+    RuntimeError
+        If job fails or times out
+    """
+    url = f"{WABI_API_URL}{request_id}?info=status"
+    start_time = time.time()
+
+    while True:
+        elapsed = time.time() - start_time
+        if elapsed > timeout:
+            raise RuntimeError(
+                f"ChIP-Atlas job timed out after {timeout}s. "
+                f"Request ID: {request_id}. Try again or increase timeout."
+            )
+
+        try:
+            response = requests.get(url, timeout=30)
+            response.raise_for_status()
+            text = response.text
+
+            # Parse status from text response
+            status = None
+            for line in text.strip().split("\n"):
+                if line.startswith("status"):
+                    status = re.split(r"[:\t]\s*", line, maxsplit=1)
+                    status = status[1].strip() if len(status) > 1 else None
+                    break
+
+            if status == "finished":
+                return "finished"
+            elif status in ("error", "failed"):
+                raise RuntimeError(
+                    f"ChIP-Atlas job failed. Request ID: {request_id}. "
+                    f"Response: {text}"
+                )
+
+            # Still running
+            minutes_elapsed = elapsed / 60
+            print(f"   Job running... ({minutes_elapsed:.1f} min elapsed)")
+
+        except requests.exceptions.RequestException as e:
+            print(f"   Warning: Poll request failed ({e}), retrying...")
+
+        time.sleep(poll_interval)
+
+
+def retrieve_zip_results(request_id, output_dir):
+    """
+    Retrieve ZIP results from completed job and extract to output directory.
+
+    Parameters
+    ----------
+    request_id : str
+        Request ID from submit_diff_job()
+    output_dir : str
+        Directory to extract results into
+
+    Returns
+    -------
+    dict
+        Paths to extracted files: {'bed': path, 'igv_bed': path,
+        'log': path, 'igv_xml': path}. Keys may be absent if file
+        not found in ZIP.
+
+    Raises
+    ------
+    RuntimeError
+        If retrieval or extraction fails
+    """
+    url = f"{WABI_API_URL}{request_id}?info=result&format=zip"
+
+    try:
+        response = requests.get(url, timeout=300)
+        response.raise_for_status()
+
+        if not response.content:
+            raise RuntimeError("API returned empty ZIP response")
+
+        raw_dir = os.path.join(output_dir, "raw_results")
+        os.makedirs(raw_dir, exist_ok=True)
+
+        with zipfile.ZipFile(io.BytesIO(response.content)) as zf:
+            zf.extractall(raw_dir)
+
+        # Identify extracted files by extension
+        extracted = {}
+        for root, _dirs, files in os.walk(raw_dir):
+            for f in files:
+                full_path = os.path.join(root, f)
+                if f.endswith(".igv.bed"):
+                    extracted["igv_bed"] = full_path
+                elif f.endswith(".bed") and "igv" not in f:
+                    extracted["bed"] = full_path
+                elif f.endswith(".log"):
+                    extracted["log"] = full_path
+                elif f.endswith(".xml"):
+                    extracted["igv_xml"] = full_path
+
+        if "bed" not in extracted:
+            # List what was actually extracted for debugging
+            all_files = []
+            for root, _dirs, files in os.walk(raw_dir):
+                all_files.extend(files)
+            raise RuntimeError(
+                f"No BED file found in ZIP results. "
+                f"Extracted files: {all_files}"
+            )
+
+        return extracted
+
+    except zipfile.BadZipFile:
+        raise RuntimeError(
+            "API returned invalid ZIP data. The job may still be processing. "
+            f"Request ID: {request_id}"
+        )
+    except requests.exceptions.RequestException as e:
+        raise RuntimeError(f"Failed to retrieve results: {e}")
+
+
+if __name__ == "__main__":
+    print("Testing query_chipatlas_api module (diff analysis)...")
+    print("Note: This requires internet connection.\n")
+
+    # Test with a small example
+    test_a = ["SRX8347024", "SRX8347025"]
+    test_b = ["SRX8347026", "SRX8347027"]
+
+    print(f"Submitting diff analysis: {len(test_a)} vs {len(test_b)} experiments...")
+    request_id = submit_diff_job(
+        experiments_a=test_a,
+        experiments_b=test_b,
+        genome="mm10",
+        analysis_type="diffbind",
+        title="test_diff",
+        description_a="group_A",
+        description_b="group_B",
+    )
+    print(f"   Request ID: {request_id}")
+
+    print("Polling for completion...")
+    status = poll_job_status(request_id, timeout=900)
+    print(f"   Status: {status}")
+
+    print("Retrieving ZIP results...")
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmpdir:
+        extracted = retrieve_zip_results(request_id, tmpdir)
+        print(f"   Extracted files: {list(extracted.keys())}")
+        for key, path in extracted.items():
+            size = os.path.getsize(path)
+            print(f"   {key}: {os.path.basename(path)} ({size} bytes)")
+
+    print("\nTest completed successfully!")

package/data/workflows/chip-atlas-diff-analysis/scripts/run_diff_workflow.py

@@ -0,0 +1,256 @@
+"""
+Main diff analysis workflow orchestrator for ChIP-Atlas.
+
+Uses the official ChIP-Atlas Diff Analysis API to compare two sets of
+experiments and identify differential peak regions (DPR via edgeR) or
+differentially methylated regions (DMR via metilene).
+
+API: POST https://dtn1.ddbj.nig.ac.jp/wabi/chipatlas/
+Reference: Zou et al. 2024 (NAR), https://chip-atlas.org
+"""
+
+import os
+
+try:
+    from scripts.query_chipatlas_api import (
+        submit_diff_job,
+        poll_job_status,
+        retrieve_zip_results,
+        VALID_ANALYSIS_TYPES,
+    )
+    from scripts.parse_bed_results import parse_bed_results, summarize_regions
+    from scripts.filter_regions import (
+        filter_regions,
+        filter_standard_chromosomes,
+        filter_min_region_size,
+    )
+    from scripts.qc_checks import run_qc_checks
+except ImportError:
+    from query_chipatlas_api import (
+        submit_diff_job,
+        poll_job_status,
+        retrieve_zip_results,
+        VALID_ANALYSIS_TYPES,
+    )
+    from parse_bed_results import parse_bed_results, summarize_regions
+    from filter_regions import (
+        filter_regions,
+        filter_standard_chromosomes,
+        filter_min_region_size,
+    )
+    from qc_checks import run_qc_checks
+
+
+def run_diff_workflow(
+    experiments_a,
+    experiments_b,
+    genome="hg38",
+    analysis_type="diffbind",
+    title="diff_analysis",
+    description_a="group_A",
+    description_b="group_B",
+    min_score=0,
+    min_region_size=0,
+    direction_filter=None,
+    design_caveats=None,
+    output_dir="diff_analysis_results",
+):
+    """
+    Run complete ChIP-Atlas differential analysis workflow.
+
+    Parameters
+    ----------
+    experiments_a : list of str
+        Experiment IDs for group A (minimum 2)
+    experiments_b : list of str
+        Experiment IDs for group B (minimum 2)
+    genome : str
+        Genome assembly (hg38, hg19, mm10, mm9, rn6, dm6, dm3, ce11, ce10, sacCer3)
+    analysis_type : str
+        'diffbind' for DPR (ChIP/ATAC/DNase-seq) or 'dmr' for DMR (Bisulfite-seq)
+    title : str
+        Analysis title
+    description_a : str
+        Label for dataset A
+    description_b : str
+        Label for dataset B
+    min_score : int
+        Post-filter: minimum BED score (0-1000)
+    min_region_size : int
+        Post-filter: minimum region size in bp
+    direction_filter : str or None
+        Post-filter: 'A_enriched' or 'B_enriched'
+    output_dir : str
+        Output directory for results
+
+    Returns
+    -------
+    dict
+        {
+            'diff_regions': pd.DataFrame,
+            'experiments_a': list,
+            'experiments_b': list,
+            'raw_files': dict,
+            'log_content': str,
+            'parameters': dict,
+        }
+
+    Verification
+    ------------
+    Prints "✓ Diff analysis completed successfully!" when done
+    """
+    # Validate inputs
+    if not experiments_a or len(experiments_a) < 2:
+        raise ValueError("Group A requires at least 2 experiment IDs")
+    if not experiments_b or len(experiments_b) < 2:
+        raise ValueError("Group B requires at least 2 experiment IDs")
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    analysis_label = VALID_ANALYSIS_TYPES.get(analysis_type, analysis_type)
+
+    print("\n" + "=" * 70)
+    print("CHIP-ATLAS DIFF ANALYSIS WORKFLOW")
+    print("=" * 70 + "\n")
+
+    # Step 1: Submit to ChIP-Atlas API
+    print("Step 1: Submitting experiment sets to ChIP-Atlas Diff Analysis API...")
+    print(f"   Analysis type: {analysis_label}")
+    print(f"   Genome: {genome}")
+    print(f"   Group A ({description_a}): {len(experiments_a)} experiments")
+    for exp_id in experiments_a:
+        print(f"      - {exp_id}")
+    print(f"   Group B ({description_b}): {len(experiments_b)} experiments")
+    for exp_id in experiments_b:
+        print(f"      - {exp_id}")
+
+    request_id = submit_diff_job(
+        experiments_a=experiments_a,
+        experiments_b=experiments_b,
+        genome=genome,
+        analysis_type=analysis_type,
+        title=title,
+        description_a=description_a,
+        description_b=description_b,
+    )
+
+    print(f"   Request ID: {request_id}")
+    print("✓ Job submitted successfully")
+
+    # Step 2: Poll for completion
+    print("\nStep 2: Waiting for analysis to complete...")
+    print("   (Diff analysis typically takes 2-10 minutes depending on dataset size)")
+
+    status = poll_job_status(request_id, timeout=900, poll_interval=15)
+    print(f"✓ Job completed (status: {status})")
+
+    # Step 3: Retrieve and parse results
+    print("\nStep 3: Retrieving results...")
+
+    extracted = retrieve_zip_results(request_id, output_dir)
+    print(f"   Extracted {len(extracted)} files from ZIP archive:")
+    for key, path in extracted.items():
+        size = os.path.getsize(path)
+        print(f"      {key}: {os.path.basename(path)} ({size:,} bytes)")
+
+    # Read log content if available
+    log_content = ""
+    if "log" in extracted:
+        with open(extracted["log"], "r") as f:
+            log_content = f.read()
+
+    # Parse BED results
+    print("\n   Parsing BED results...")
+    results_df = parse_bed_results(extracted["bed"])
+    n_total_raw = len(results_df)
+    n_sig_raw = int(results_df["significant"].sum()) if "significant" in results_df.columns else 0
+    print(f"✓ Parsed {n_total_raw} differential regions ({n_sig_raw} significant at FDR < 0.05)")
+
+    # QC checks on unfiltered data
+    qc_warnings = run_qc_checks(
+        results_df,
+        n_samples_a=len(experiments_a),
+        n_samples_b=len(experiments_b),
+        analysis_type=analysis_type,
+        genome=genome,
+    )
+
+    # Preserve unfiltered data for reference
+    unfiltered_df = results_df.copy()
+    n_before_qc = len(results_df)
+
+    # Default QC filtering: remove non-standard contigs and tiny regions
+    print("\n   Applying default QC filters...")
+    results_df = filter_standard_chromosomes(results_df, genome=genome)
+    results_df = filter_min_region_size(results_df, min_size=10)
+    n_after_qc = len(results_df)
+
+    if n_before_qc != n_after_qc:
+        n_sig_after_qc = int(results_df["significant"].sum()) if "significant" in results_df.columns else 0
+        n_sig_removed = n_sig_raw - n_sig_after_qc
+        msg = f"   QC filtering: {n_before_qc} → {n_after_qc} regions"
+        if n_sig_removed > 0:
+            msg += f" ({n_sig_removed} significant regions removed)"
+        print(msg)
+
+    # Step 4: Apply user-requested post-filters
+    if min_score > 0 or min_region_size > 0 or direction_filter:
+        print("\nStep 4: Applying additional filters...")
+        results_df = filter_regions(
+            results_df,
+            min_score=min_score,
+            min_region_size=min_region_size,
+            direction_filter=direction_filter,
+        )
+        print(f"✓ Filtered to {len(results_df)} regions")
+    else:
+        print("\nStep 4: No additional user filters applied")
+
+    # Post-QC detailed summary
+    summarize_regions(results_df)
+
+    # Summary
+    n_a_enriched = (results_df["direction"] == "A_enriched").sum() if len(results_df) > 0 else 0
+    n_b_enriched = (results_df["direction"] == "B_enriched").sum() if len(results_df) > 0 else 0
+    n_sig_final = int(results_df["significant"].sum()) if len(results_df) > 0 and "significant" in results_df.columns else 0
+
+    print(f"\n   === Results Summary (post-QC) ===")
+    print(f"   Total differential regions: {len(results_df)}")
+    print(f"   Significant (FDR < 0.05): {n_sig_final}")
+    print(f"   Enriched in {description_a}: {n_a_enriched}")
+    print(f"   Enriched in {description_b}: {n_b_enriched}")
+
+    if n_sig_final > 0 and "direction" in results_df.columns:
+        sig_df = results_df[results_df["significant"]]
+        n_sig_a = int((sig_df["direction"] == "A_enriched").sum())
+        n_sig_b = int((sig_df["direction"] == "B_enriched").sum())
+        print(f"   Significant in {description_a}: {n_sig_a}")
+        print(f"   Significant in {description_b}: {n_sig_b}")
+
+    if len(results_df) > 0 and "region_size" in results_df.columns:
+        median_size = results_df["region_size"].median()
+        print(f"   Median region size: {median_size:.0f} bp")
+
+    print("\n✓ Diff analysis completed successfully!")
+    print("=" * 70 + "\n")
+
+    return {
+        "diff_regions": results_df,
+        "diff_regions_unfiltered": unfiltered_df,
+        "qc_warnings": qc_warnings,
+        "experiments_a": experiments_a,
+        "experiments_b": experiments_b,
+        "raw_files": extracted,
+        "log_content": log_content,
+        "parameters": {
+            "genome": genome,
+            "analysis_type": analysis_type,
+            "title": title,
+            "description_a": description_a,
+            "description_b": description_b,
+            "min_score": min_score,
+            "min_region_size": min_region_size,
+            "direction_filter": direction_filter,
+            "design_caveats": design_caveats or [],
+        },
+    }