opencode-knowledge-work 0.0.0-stage → 0.1.0

package/content/bio-research/skills/bio-research-nextflow-development/scripts/check_environment.py

@@ -0,0 +1,452 @@
+#!/usr/bin/env python3
+"""
+Pre-flight environment validation for nf-core pipelines.
+
+Checks Docker, Nextflow, Java, system resources, and network connectivity.
+Run this BEFORE attempting any pipeline execution.
+
+Usage:
+    python check_environment.py
+    python check_environment.py --json
+"""
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass, field, asdict
+from typing import List, Optional
+
+
+@dataclass
+class CheckResult:
+    """Result of a single environment check."""
+    name: str
+    passed: bool
+    message: str
+    details: Optional[str] = None
+    fix: Optional[str] = None
+
+
+@dataclass
+class EnvironmentReport:
+    """Complete environment validation report."""
+    ready: bool
+    checks: List[CheckResult] = field(default_factory=list)
+    recommendations: List[str] = field(default_factory=list)
+
+    def to_dict(self):
+        return {
+            "ready": self.ready,
+            "checks": [asdict(c) for c in self.checks],
+            "recommendations": self.recommendations
+        }
+
+
+def check_docker() -> CheckResult:
+    """Check Docker availability, daemon status, and permissions."""
+    if not shutil.which("docker"):
+        return CheckResult(
+            name="Docker",
+            passed=False,
+            message="Docker not found in PATH",
+            fix="Install Docker: https://docs.docker.com/get-docker/"
+        )
+
+    try:
+        result = subprocess.run(
+            ["docker", "info"],
+            capture_output=True,
+            text=True,
+            timeout=15
+        )
+
+        if result.returncode != 0:
+            stderr_lower = result.stderr.lower()
+            if "permission denied" in stderr_lower:
+                return CheckResult(
+                    name="Docker",
+                    passed=False,
+                    message="Docker permission denied",
+                    details="Cannot connect to Docker daemon",
+                    fix="sudo usermod -aG docker $USER && newgrp docker"
+                )
+            elif "cannot connect" in stderr_lower or "is the docker daemon running" in stderr_lower:
+                return CheckResult(
+                    name="Docker",
+                    passed=False,
+                    message="Docker daemon not running",
+                    details=result.stderr[:200] if result.stderr else None,
+                    fix="sudo systemctl start docker"
+                )
+            else:
+                return CheckResult(
+                    name="Docker",
+                    passed=False,
+                    message="Docker error",
+                    details=result.stderr[:200] if result.stderr else None,
+                    fix="Check Docker installation and daemon status"
+                )
+
+        return CheckResult(
+            name="Docker",
+            passed=True,
+            message="Docker is available and running"
+        )
+
+    except subprocess.TimeoutExpired:
+        return CheckResult(
+            name="Docker",
+            passed=False,
+            message="Docker command timed out",
+            fix="Check Docker daemon status: sudo systemctl status docker"
+        )
+    except Exception as e:
+        return CheckResult(
+            name="Docker",
+            passed=False,
+            message=f"Docker check failed: {str(e)}"
+        )
+
+
+def check_nextflow() -> CheckResult:
+    """Check Nextflow installation and version (requires >= 23.04)."""
+    if not shutil.which("nextflow"):
+        return CheckResult(
+            name="Nextflow",
+            passed=False,
+            message="Nextflow not found in PATH",
+            fix="curl -s https://get.nextflow.io | bash && mv nextflow ~/bin/ && export PATH=$HOME/bin:$PATH"
+        )
+
+    try:
+        result = subprocess.run(
+            ["nextflow", "-version"],
+            capture_output=True,
+            text=True,
+            timeout=30
+        )
+
+        output = result.stdout + result.stderr
+        version_line = output.strip().split('\n')[0] if output else ""
+
+        import re
+        match = re.search(r'(\d+)\.(\d+)\.(\d+)', version_line)
+
+        if match:
+            major, minor, patch = int(match.group(1)), int(match.group(2)), int(match.group(3))
+            version_str = f"{major}.{minor}.{patch}"
+
+            # Require version >= 23.04
+            if major > 23 or (major == 23 and minor >= 4):
+                return CheckResult(
+                    name="Nextflow",
+                    passed=True,
+                    message=f"Nextflow {version_str} installed",
+                    details=version_line
+                )
+            else:
+                return CheckResult(
+                    name="Nextflow",
+                    passed=False,
+                    message=f"Nextflow {version_str} is outdated (requires >= 23.04)",
+                    details=version_line,
+                    fix="nextflow self-update"
+                )
+
+        return CheckResult(
+            name="Nextflow",
+            passed=True,
+            message="Nextflow installed (version unknown)",
+            details=version_line
+        )
+
+    except subprocess.TimeoutExpired:
+        return CheckResult(
+            name="Nextflow",
+            passed=False,
+            message="Nextflow command timed out",
+            fix="Check Nextflow installation"
+        )
+    except Exception as e:
+        return CheckResult(
+            name="Nextflow",
+            passed=False,
+            message=f"Nextflow check failed: {str(e)}"
+        )
+
+
+def check_java() -> CheckResult:
+    """Check Java version (requires >= 11)."""
+    if not shutil.which("java"):
+        return CheckResult(
+            name="Java",
+            passed=False,
+            message="Java not found in PATH",
+            fix="Install Java 11+: sudo apt install openjdk-11-jdk"
+        )
+
+    try:
+        result = subprocess.run(
+            ["java", "-version"],
+            capture_output=True,
+            text=True,
+            timeout=10
+        )
+
+        # Java version is typically in stderr
+        output = result.stderr or result.stdout
+        import re
+        match = re.search(r'version "(\d+)', output)
+
+        if match:
+            version = int(match.group(1))
+            version_line = output.strip().split('\n')[0]
+
+            if version >= 11:
+                return CheckResult(
+                    name="Java",
+                    passed=True,
+                    message=f"Java {version} installed",
+                    details=version_line
+                )
+            else:
+                return CheckResult(
+                    name="Java",
+                    passed=False,
+                    message=f"Java {version} is too old (requires >= 11)",
+                    details=version_line,
+                    fix="Install Java 11+: sudo apt install openjdk-11-jdk"
+                )
+
+        return CheckResult(
+            name="Java",
+            passed=True,
+            message="Java installed",
+            details=output.strip().split('\n')[0] if output else None
+        )
+
+    except Exception as e:
+        return CheckResult(
+            name="Java",
+            passed=False,
+            message=f"Java check failed: {str(e)}"
+        )
+
+
+def check_resources() -> CheckResult:
+    """Check system resources (CPU, memory, disk)."""
+    try:
+        # CPU cores
+        cpu_count = os.cpu_count() or 1
+
+        # Memory
+        mem_gb = 0
+        try:
+            # Linux: read from /proc/meminfo
+            with open('/proc/meminfo', 'r') as f:
+                for line in f:
+                    if line.startswith('MemTotal:'):
+                        mem_kb = int(line.split()[1])
+                        mem_gb = mem_kb / (1024 * 1024)
+                        break
+        except (FileNotFoundError, PermissionError):
+            # macOS: use sysctl
+            try:
+                result = subprocess.run(
+                    ['sysctl', '-n', 'hw.memsize'],
+                    capture_output=True, text=True, timeout=5
+                )
+                if result.returncode == 0:
+                    mem_gb = int(result.stdout.strip()) / (1024**3)
+            except Exception:
+                pass
+
+        # Disk space (current directory)
+        disk_gb = 0
+        try:
+            statvfs = os.statvfs('.')
+            disk_gb = (statvfs.f_frsize * statvfs.f_bavail) / (1024**3)
+        except Exception:
+            pass
+
+        details = f"CPUs: {cpu_count}, Memory: {mem_gb:.1f}GB, Disk: {disk_gb:.1f}GB available"
+
+        # Check minimums
+        warnings = []
+        if cpu_count < 4:
+            warnings.append(f"Low CPU count ({cpu_count}). Consider --max_cpus {cpu_count}")
+        if 0 < mem_gb < 8:
+            warnings.append(f"Low memory ({mem_gb:.1f}GB). Use --max_memory '{int(mem_gb)}GB'")
+        if 0 < disk_gb < 50:
+            warnings.append(f"Low disk space ({disk_gb:.1f}GB). Pipelines need ~100GB for human data")
+
+        if warnings:
+            return CheckResult(
+                name="Resources",
+                passed=True,
+                message="Resources available (with warnings)",
+                details=details,
+                fix="; ".join(warnings)
+            )
+
+        return CheckResult(
+            name="Resources",
+            passed=True,
+            message="Sufficient resources available",
+            details=details
+        )
+
+    except Exception as e:
+        return CheckResult(
+            name="Resources",
+            passed=True,  # Don't fail on resource check errors
+            message=f"Could not fully check resources: {str(e)}"
+        )
+
+
+def check_network() -> CheckResult:
+    """Check network connectivity to Docker Hub and nf-core."""
+    try:
+        import urllib.request
+
+        # User-Agent header to avoid 403 from sites that block default Python agent
+        headers = {'User-Agent': 'nf-core-helper/1.0'}
+
+        # Try Docker Hub
+        try:
+            req = urllib.request.Request("https://hub.docker.com", headers=headers)
+            urllib.request.urlopen(req, timeout=10)
+            docker_hub_ok = True
+        except Exception:
+            docker_hub_ok = False
+
+        # Try nf-core (for pipeline downloads)
+        try:
+            req = urllib.request.Request("https://nf-co.re", headers=headers)
+            urllib.request.urlopen(req, timeout=10)
+            nfcore_ok = True
+        except Exception:
+            nfcore_ok = False
+
+        if docker_hub_ok and nfcore_ok:
+            return CheckResult(
+                name="Network",
+                passed=True,
+                message="Network connectivity OK (Docker Hub & nf-core reachable)"
+            )
+        elif docker_hub_ok:
+            return CheckResult(
+                name="Network",
+                passed=True,
+                message="Docker Hub reachable (nf-core.re not reachable)",
+                details="Pipeline downloads may still work via GitHub"
+            )
+        else:
+            return CheckResult(
+                name="Network",
+                passed=False,
+                message="Cannot reach Docker Hub",
+                fix="Check network connection. Containers require Docker Hub access."
+            )
+
+    except Exception as e:
+        return CheckResult(
+            name="Network",
+            passed=False,
+            message=f"Network check failed: {str(e)}",
+            fix="Check network connection and proxy settings"
+        )
+
+
+def run_all_checks() -> EnvironmentReport:
+    """Run all environment checks and return comprehensive report."""
+    checks = [
+        check_docker(),
+        check_nextflow(),
+        check_java(),
+        check_resources(),
+        check_network(),
+    ]
+
+    # Critical checks that must pass
+    critical_checks = ["Docker", "Nextflow", "Java"]
+    ready = all(c.passed for c in checks if c.name in critical_checks)
+
+    # Build recommendations
+    recommendations = []
+    for check in checks:
+        if not check.passed and check.fix:
+            recommendations.append(f"{check.name}: {check.fix}")
+        elif check.passed and check.fix:  # Warnings
+            recommendations.append(f"{check.name} (warning): {check.fix}")
+
+    return EnvironmentReport(
+        ready=ready,
+        checks=checks,
+        recommendations=recommendations
+    )
+
+
+def print_report(report: EnvironmentReport):
+    """Print human-readable report to stdout."""
+    print("\n" + "=" * 50)
+    print("  nf-core Environment Check")
+    print("=" * 50 + "\n")
+
+    for check in report.checks:
+        status = "\033[92m[PASS]\033[0m" if check.passed else "\033[91m[FAIL]\033[0m"
+        print(f"{status} {check.name}: {check.message}")
+
+        if check.details:
+            print(f"       {check.details}")
+
+        if not check.passed and check.fix:
+            print(f"       \033[93mFix:\033[0m {check.fix}")
+        elif check.passed and check.fix:  # Warning
+            print(f"       \033[93mWarning:\033[0m {check.fix}")
+
+    print()
+    if report.ready:
+        print("\033[92m✓ Environment is READY for nf-core pipelines.\033[0m")
+    else:
+        print("\033[91m✗ Environment is NOT READY. Please address the issues above.\033[0m")
+
+    if report.recommendations:
+        print("\n--- Recommendations ---")
+        for i, rec in enumerate(report.recommendations, 1):
+            print(f"  {i}. {rec}")
+
+    print()
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Check environment for nf-core pipeline execution",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+    python check_environment.py           # Human-readable output
+    python check_environment.py --json    # JSON output for parsing
+        """
+    )
+    parser.add_argument("--json", action="store_true",
+                        help="Output results as JSON")
+
+    args = parser.parse_args()
+
+    report = run_all_checks()
+
+    if args.json:
+        print(json.dumps(report.to_dict(), indent=2))
+    else:
+        print_report(report)
+
+    sys.exit(0 if report.ready else 1)
+
+
+if __name__ == "__main__":
+    main()

package/content/bio-research/skills/bio-research-nextflow-development/scripts/config/genomes.yaml

@@ -0,0 +1,148 @@
+# Organism to Genome Mapping for nf-core Pipelines
+# Maps organism names (as they appear in GEO/SRA) to iGenomes keys
+
+organisms:
+  # Human
+  "Homo sapiens":
+    genome: "GRCh38"
+    taxid: 9606
+    aliases: ["human", "hg38", "GRCh38"]
+    notes: "Primary human reference genome"
+
+  "Homo sapiens (legacy)":
+    genome: "GRCh37"
+    taxid: 9606
+    aliases: ["hg19", "GRCh37"]
+    notes: "Legacy human reference, still used for some clinical data"
+
+  # Mouse
+  "Mus musculus":
+    genome: "GRCm39"
+    taxid: 10090
+    aliases: ["mouse", "mm39", "GRCm39"]
+    notes: "Current mouse reference genome"
+
+  "Mus musculus (legacy)":
+    genome: "GRCm38"
+    taxid: 10090
+    aliases: ["mm10", "GRCm38"]
+    notes: "Legacy mouse reference"
+
+  # Yeast
+  "Saccharomyces cerevisiae":
+    genome: "R64-1-1"
+    taxid: 4932
+    aliases: ["yeast", "sacCer3", "S288C", "budding yeast"]
+    notes: "S288C reference strain"
+
+  # Fruit fly
+  "Drosophila melanogaster":
+    genome: "BDGP6"
+    taxid: 7227
+    aliases: ["fly", "dm6", "fruit fly", "Dmel"]
+    notes: "Berkeley Drosophila Genome Project release 6"
+
+  # Worm
+  "Caenorhabditis elegans":
+    genome: "WBcel235"
+    taxid: 6239
+    aliases: ["worm", "ce11", "C. elegans", "Cele"]
+    notes: "WormBase reference"
+
+  # Zebrafish
+  "Danio rerio":
+    genome: "GRCz11"
+    taxid: 7955
+    aliases: ["zebrafish", "danRer11", "Drer"]
+    notes: "Genome Reference Consortium Zebrafish Build 11"
+
+  # Arabidopsis
+  "Arabidopsis thaliana":
+    genome: "TAIR10"
+    taxid: 3702
+    aliases: ["arabidopsis", "thale cress", "Atha"]
+    notes: "The Arabidopsis Information Resource v10"
+
+  # Rat
+  "Rattus norvegicus":
+    genome: "Rnor_6.0"
+    taxid: 10116
+    aliases: ["rat", "rn6", "Rnor"]
+    notes: "Rnor 6.0 reference"
+
+  # Chicken
+  "Gallus gallus":
+    genome: "GRCg6a"
+    taxid: 9031
+    aliases: ["chicken", "galGal6", "Ggal"]
+    notes: "Genome Reference Consortium Chicken Build 6a"
+
+  # Pig
+  "Sus scrofa":
+    genome: "Sscrofa11.1"
+    taxid: 9823
+    aliases: ["pig", "susScr11", "Sscr"]
+    notes: "Swine genome assembly 11.1"
+
+  # Cow
+  "Bos taurus":
+    genome: "ARS-UCD1.2"
+    taxid: 9913
+    aliases: ["cow", "bosTau9", "cattle", "Btau"]
+    notes: "USDA ARS assembly"
+
+  # Dog
+  "Canis lupus familiaris":
+    genome: "CanFam3.1"
+    taxid: 9615
+    aliases: ["dog", "canFam3", "Clup"]
+    notes: "Broad Institute CanFam3.1"
+
+  # Frog
+  "Xenopus tropicalis":
+    genome: "JGI_4.2"
+    taxid: 8364
+    aliases: ["frog", "xenTro9", "Xtro"]
+    notes: "JGI assembly version 4.2"
+
+  # Maize/Corn
+  "Zea mays":
+    genome: "Zm-B73-REFERENCE-NAM-5.0"
+    taxid: 4577
+    aliases: ["maize", "corn", "Zmay"]
+    notes: "B73 reference genome v5"
+
+  # Rice
+  "Oryza sativa":
+    genome: "IRGSP-1.0"
+    taxid: 39947
+    aliases: ["rice", "Osat"]
+    notes: "International Rice Genome Sequencing Project"
+
+  # E. coli (common bacterial model)
+  "Escherichia coli":
+    genome: null
+    taxid: 562
+    aliases: ["E. coli", "Ecol"]
+    notes: "Use specific strain reference; K-12 MG1655 common"
+
+  # Fission yeast
+  "Schizosaccharomyces pombe":
+    genome: "ASM294v2"
+    taxid: 4896
+    aliases: ["fission yeast", "S. pombe", "Spom"]
+    notes: "PomBase reference"
+
+# Pipeline mapping based on library strategy
+pipeline_suggestions:
+  "RNA-SEQ": "rnaseq"
+  "ATAC-SEQ": "atacseq"
+  "CHIP-SEQ": "chipseq"
+  "WGS": "sarek"
+  "WXS": "sarek"
+  "EXOME": "sarek"
+  "AMPLICON": "ampliseq"
+  "BISULFITE-SEQ": "methylseq"
+  "HI-C": "hic"
+  "MIRNA-SEQ": "smrnaseq"
+  "RRBS": "methylseq"