opencode-knowledge-work 0.0.0-stage → 0.1.0

package/content/bio-research/skills/bio-research-nextflow-development/scripts/utils/sample_inference.py

@@ -0,0 +1,290 @@
+"""
+Sample name and metadata inference from filenames.
+
+This module extracts sample information, detects tumor/normal status,
+and matches R1/R2 read pairs from sequencing file names.
+"""
+
+import os
+import re
+from typing import Dict, List, Optional, Tuple
+
+
+# R1/R2 patterns with priority scores (higher = more confident)
+R1_PATTERNS = [
+    (r'_R1_\d{3}', 10),      # _R1_001 (Illumina standard)
+    (r'_R1[_.]', 8),         # _R1. or _R1_
+    (r'\.R1[_.]', 8),        # .R1. or .R1_
+    (r'_1[_.]', 5),          # _1. or _1_
+    (r'_R1\.f', 6),          # _R1.fastq
+    (r'_1\.f', 4),           # _1.fastq
+]
+
+R2_PATTERNS = [
+    (r'_R2_\d{3}', 10),      # _R2_001 (Illumina standard)
+    (r'_R2[_.]', 8),         # _R2. or _R2_
+    (r'\.R2[_.]', 8),        # .R2. or .R2_
+    (r'_2[_.]', 5),          # _2. or _2_
+    (r'_R2\.f', 6),          # _R2.fastq
+    (r'_2\.f', 4),           # _2.fastq
+]
+
+# Tumor/normal keywords
+TUMOR_KEYWORDS = [
+    r'\btumou?r\b',
+    r'\bmetastasis\b',
+    r'\bmet\b',
+    r'\bprimary\b',
+    r'\bcancer\b',
+    r'\bmalignant\b',
+    r'[-_]T[-_]',
+    r'[-_]T\d*$',
+    r'^T\d*[-_]',
+]
+
+NORMAL_KEYWORDS = [
+    r'\bnormal\b',
+    r'\bgermline\b',
+    r'\bblood\b',
+    r'\bpbmc\b',
+    r'\bcontrol\b',
+    r'\bhealthy\b',
+    r'\bmatched\b',
+    r'[-_]N[-_]',
+    r'[-_]N\d*$',
+    r'^N\d*[-_]',
+]
+
+# Lane pattern
+LANE_PATTERN = r'[_.]L(\d{3})[_.]'
+
+# Patient/sample extraction patterns
+PATIENT_PATTERNS = [
+    r'^(P\d+)[-_]',           # P001_sample
+    r'^(patient\d+)[-_]',     # patient1_sample
+    r'^(TCGA-\w+-\w+)',       # TCGA format
+    r'^([A-Z]{2,3}\d{3,})[-_]',  # AB123_sample
+]
+
+# Replicate patterns
+REPLICATE_PATTERNS = [
+    r'[_.]rep(\d+)',          # _rep1, .rep2
+    r'[_.]replicate(\d+)',    # _replicate1
+    r'[_.]R(\d+)[_.]',        # _R1_ (but not R1/R2 for reads!)
+    r'[-_](\d+)$',            # sample_1 (last resort)
+]
+
+
+def extract_sample_info(filepath: str) -> Dict[str, str]:
+    """
+    Extract sample metadata from filepath.
+
+    Args:
+        filepath: Path to sequencing file
+
+    Returns:
+        Dict with: sample, patient, lane (if detectable)
+    """
+    filename = os.path.basename(filepath)
+
+    # Remove extensions
+    stem = filename
+    for ext in ['.fastq.gz', '.fq.gz', '.fastq', '.fq', '.bam', '.cram', '.bai', '.crai']:
+        if stem.lower().endswith(ext):
+            stem = stem[:-len(ext)]
+            break
+
+    info = {}
+
+    # Extract lane
+    lane_match = re.search(LANE_PATTERN, stem)
+    info['lane'] = f"L{lane_match.group(1)}" if lane_match else "L001"
+
+    # Remove lane from stem
+    clean_stem = re.sub(LANE_PATTERN, '_', stem)
+
+    # Remove R1/R2 indicators and everything after
+    for pattern, _ in R1_PATTERNS + R2_PATTERNS:
+        clean_stem = re.sub(pattern + r'.*', '', clean_stem, flags=re.IGNORECASE)
+
+    # Clean up trailing/multiple underscores and dots
+    clean_stem = re.sub(r'[_.-]+$', '', clean_stem)
+    clean_stem = re.sub(r'[_.-]{2,}', '_', clean_stem)
+
+    # Try to extract patient ID
+    for pattern in PATIENT_PATTERNS:
+        match = re.match(pattern, clean_stem, re.IGNORECASE)
+        if match:
+            info['patient'] = match.group(1)
+            break
+
+    # Sample is the cleaned stem
+    info['sample'] = clean_stem if clean_stem else filename.split('.')[0]
+
+    # Default patient to sample if not extracted
+    if 'patient' not in info:
+        info['patient'] = info['sample']
+
+    return info
+
+
+def infer_tumor_normal_status(sample_name: str) -> Optional[int]:
+    """
+    Infer tumor (1) or normal (0) status from sample name.
+
+    Args:
+        sample_name: Sample identifier
+
+    Returns:
+        1 for tumor, 0 for normal, None if cannot determine
+    """
+    name_lower = sample_name.lower()
+
+    # Check tumor indicators
+    for pattern in TUMOR_KEYWORDS:
+        if re.search(pattern, name_lower, re.IGNORECASE):
+            return 1
+
+    # Check normal indicators
+    for pattern in NORMAL_KEYWORDS:
+        if re.search(pattern, name_lower, re.IGNORECASE):
+            return 0
+
+    return None
+
+
+def extract_replicate_number(sample_name: str) -> Optional[int]:
+    """
+    Extract replicate number from sample name.
+
+    Args:
+        sample_name: Sample identifier
+
+    Returns:
+        Replicate number if found, None otherwise
+    """
+    for pattern in REPLICATE_PATTERNS:
+        match = re.search(pattern, sample_name, re.IGNORECASE)
+        if match:
+            try:
+                return int(match.group(1))
+            except ValueError:
+                continue
+    return None
+
+
+def _get_pattern_score(filename: str, patterns: List[Tuple[str, int]]) -> int:
+    """Get highest matching pattern score."""
+    max_score = 0
+    for pattern, score in patterns:
+        if re.search(pattern, filename, re.IGNORECASE):
+            max_score = max(max_score, score)
+    return max_score
+
+
+def _get_sample_key(filepath: str) -> str:
+    """Generate a key for grouping related files."""
+    info = extract_sample_info(filepath)
+    sample = info['sample']
+    lane = info.get('lane', 'L001')
+
+    # Include lane in key for multi-lane samples
+    if lane != "L001":
+        return f"{sample}_{lane}"
+    return sample
+
+
+def match_read_pairs(files) -> Dict[str, Dict]:
+    """
+    Match R1/R2 read pairs using scored pattern matching.
+
+    Args:
+        files: List of FileInfo objects (from file_discovery)
+
+    Returns:
+        Dict mapping sample_key to {'r1': path, 'r2': path, 'info': dict}
+    """
+    # Classify files
+    r1_files = []
+    r2_files = []
+
+    for file in files:
+        filename = file.name if hasattr(file, 'name') else os.path.basename(str(file))
+        filepath = file.path if hasattr(file, 'path') else str(file)
+
+        r1_score = _get_pattern_score(filename, R1_PATTERNS)
+        r2_score = _get_pattern_score(filename, R2_PATTERNS)
+
+        if r2_score > r1_score and r2_score > 0:
+            r2_files.append((filepath, r2_score))
+        elif r1_score > 0:
+            r1_files.append((filepath, r1_score))
+        else:
+            # No clear indicator - assume R1 (single-end or non-standard naming)
+            r1_files.append((filepath, 0))
+
+    # Build pairs by matching sample keys
+    pairs = {}
+
+    # Process R1 files first
+    for r1_path, score in r1_files:
+        key = _get_sample_key(r1_path)
+        info = extract_sample_info(r1_path)
+
+        if key not in pairs:
+            pairs[key] = {
+                'r1': r1_path,
+                'r2': None,
+                'info': info,
+                'score': score
+            }
+        else:
+            # Multiple R1 files for same sample (should not happen)
+            pairs[key]['r1'] = r1_path
+
+    # Match R2 files
+    for r2_path, score in r2_files:
+        key = _get_sample_key(r2_path)
+        info = extract_sample_info(r2_path)
+
+        if key in pairs:
+            pairs[key]['r2'] = r2_path
+        else:
+            # R2 without matching R1
+            pairs[key] = {
+                'r1': None,
+                'r2': r2_path,
+                'info': info,
+                'score': score
+            }
+
+    return pairs
+
+
+def infer_patient_groupings(sample_names: List[str]) -> Dict[str, str]:
+    """
+    Infer patient groupings from sample names.
+
+    Groups samples that share a common prefix pattern.
+
+    Args:
+        sample_names: List of sample identifiers
+
+    Returns:
+        Dict mapping sample_name to patient_id
+    """
+    patient_map = {}
+
+    for sample in sample_names:
+        # Try to find a patient pattern
+        for pattern in PATIENT_PATTERNS:
+            match = re.match(pattern, sample, re.IGNORECASE)
+            if match:
+                patient_map[sample] = match.group(1)
+                break
+
+        if sample not in patient_map:
+            # Default: each sample is its own patient
+            patient_map[sample] = sample
+
+    return patient_map

package/content/bio-research/skills/bio-research-nextflow-development/scripts/utils/validators.py

@@ -0,0 +1,256 @@
+"""
+Samplesheet validation utilities.
+
+Validates samplesheet rows against pipeline configuration before writing,
+catching errors early with helpful messages.
+"""
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, List, Optional
+import yaml
+
+
+@dataclass
+class ValidationResult:
+    """Result of samplesheet validation."""
+    valid: bool
+    errors: List[str] = field(default_factory=list)
+    warnings: List[str] = field(default_factory=list)
+    suggestions: List[str] = field(default_factory=list)
+
+    def __bool__(self):
+        return self.valid
+
+    def summary(self) -> str:
+        """Generate human-readable summary."""
+        lines = []
+        if self.errors:
+            lines.append("Errors:")
+            for e in self.errors:
+                lines.append(f"  - {e}")
+        if self.warnings:
+            lines.append("Warnings:")
+            for w in self.warnings:
+                lines.append(f"  - {w}")
+        if self.suggestions:
+            lines.append("Suggestions:")
+            for s in self.suggestions:
+                lines.append(f"  - {s}")
+        return "\n".join(lines)
+
+
+def load_pipeline_config(pipeline: str) -> Optional[Dict]:
+    """Load pipeline configuration from YAML file."""
+    # Find config directory relative to this file
+    script_dir = Path(__file__).parent.parent.parent
+    config_path = script_dir / "config" / "pipelines" / f"{pipeline}.yaml"
+
+    if not config_path.exists():
+        return None
+
+    with open(config_path) as f:
+        return yaml.safe_load(f)
+
+
+def validate_samplesheet(
+    rows: List[Dict],
+    pipeline: str,
+    config: Optional[Dict] = None
+) -> ValidationResult:
+    """
+    Validate samplesheet rows against pipeline requirements.
+
+    Args:
+        rows: List of row dictionaries
+        pipeline: Pipeline name (e.g., 'rnaseq', 'sarek')
+        config: Optional pre-loaded config dict
+
+    Returns:
+        ValidationResult with errors, warnings, and suggestions
+    """
+    errors = []
+    warnings = []
+    suggestions = []
+
+    # Load config if not provided
+    if config is None:
+        config = load_pipeline_config(pipeline)
+
+    if config is None:
+        errors.append(f"Unknown pipeline: {pipeline}")
+        return ValidationResult(valid=False, errors=errors)
+
+    columns = config.get("samplesheet", {}).get("columns", [])
+    required_cols = [c["name"] for c in columns if c.get("required", False)]
+
+    if not rows:
+        errors.append("Samplesheet is empty - no samples found")
+        return ValidationResult(valid=False, errors=errors)
+
+    # Validate each row
+    for i, row in enumerate(rows):
+        row_num = i + 2  # Account for header row
+
+        # Check required columns
+        for col_name in required_cols:
+            col_config = next((c for c in columns if c["name"] == col_name), None)
+
+            # Skip columns with conditions that don't apply
+            if col_config and "condition" in col_config:
+                # Simple condition check - skip for now
+                # Full implementation would evaluate conditions
+                pass
+
+            if col_name not in row or row[col_name] is None or row[col_name] == "":
+                # Check if there's a default
+                if col_config and "default" in col_config:
+                    continue
+                errors.append(f"Row {row_num}: Missing required column '{col_name}'")
+
+        # Validate path columns exist
+        for col_name in ["fastq_1", "fastq_2", "bam", "bai"]:
+            if col_name in row and row[col_name]:
+                path = row[col_name]
+                if not os.path.exists(path):
+                    errors.append(f"Row {row_num}: File not found: {path}")
+                elif not os.path.isfile(path):
+                    errors.append(f"Row {row_num}: Not a file: {path}")
+
+        # Validate enum values
+        for col_config in columns:
+            col_name = col_config["name"]
+            if col_name in row and row[col_name] and "allowed" in col_config:
+                value = row[col_name]
+                allowed = col_config["allowed"]
+                if value not in allowed:
+                    errors.append(
+                        f"Row {row_num}: Invalid value '{value}' for '{col_name}'. "
+                        f"Allowed: {allowed}"
+                    )
+
+        # Check R1/R2 pairing consistency
+        r1 = row.get("fastq_1", "")
+        r2 = row.get("fastq_2", "")
+        if r1 and not r2:
+            warnings.append(f"Row {row_num}: Single-end data (no R2 file)")
+        elif r2 and not r1:
+            errors.append(f"Row {row_num}: R2 present but R1 missing")
+
+    # Check for duplicate samples
+    sample_col = "sample" if "sample" in rows[0] else "patient"
+    if sample_col in rows[0]:
+        samples = [r.get(sample_col, "") for r in rows]
+        duplicates = [s for s in set(samples) if samples.count(s) > 1]
+        if duplicates:
+            warnings.append(f"Duplicate sample names: {duplicates}")
+            suggestions.append(
+                "Duplicates may be intentional (multi-lane sequencing). "
+                "Verify sample grouping is correct."
+            )
+
+    # Pipeline-specific validation
+    if pipeline == "sarek":
+        _validate_sarek_specific(rows, errors, warnings, suggestions)
+    elif pipeline == "atacseq":
+        _validate_atacseq_specific(rows, errors, warnings, suggestions)
+
+    return ValidationResult(
+        valid=len(errors) == 0,
+        errors=errors,
+        warnings=warnings,
+        suggestions=suggestions
+    )
+
+
+def _validate_sarek_specific(
+    rows: List[Dict],
+    errors: List[str],
+    warnings: List[str],
+    suggestions: List[str]
+):
+    """Sarek-specific validation for tumor/normal pairing."""
+    # Group by patient
+    patients = {}
+    for row in rows:
+        patient = row.get("patient", "")
+        status = row.get("status")
+
+        if patient not in patients:
+            patients[patient] = {"tumor": 0, "normal": 0, "unknown": 0}
+
+        if status == 1:
+            patients[patient]["tumor"] += 1
+        elif status == 0:
+            patients[patient]["normal"] += 1
+        else:
+            patients[patient]["unknown"] += 1
+
+    # Check pairing
+    for patient, counts in patients.items():
+        if counts["tumor"] > 0 and counts["normal"] == 0:
+            warnings.append(
+                f"Patient '{patient}': Tumor sample(s) without matched normal. "
+                "Somatic calling works best with paired tumor-normal."
+            )
+            suggestions.append(
+                f"For patient '{patient}': Add a normal sample or use tumor-only mode."
+            )
+
+        if counts["unknown"] > 0:
+            warnings.append(
+                f"Patient '{patient}': {counts['unknown']} sample(s) with unknown status. "
+                "Set status column to 0 (normal) or 1 (tumor)."
+            )
+
+
+def _validate_atacseq_specific(
+    rows: List[Dict],
+    errors: List[str],
+    warnings: List[str],
+    suggestions: List[str]
+):
+    """ATAC-seq specific validation for replicates."""
+    # Group by sample (condition)
+    samples = {}
+    for row in rows:
+        sample = row.get("sample", "")
+        replicate = row.get("replicate", 1)
+
+        if sample not in samples:
+            samples[sample] = []
+
+        samples[sample].append(replicate)
+
+    # Check replicates
+    for sample, reps in samples.items():
+        if len(reps) < 2:
+            warnings.append(
+                f"Sample '{sample}': Only {len(reps)} replicate(s). "
+                "Consensus peaks require 2+ replicates."
+            )
+
+        # Check for duplicate replicate numbers
+        if len(reps) != len(set(reps)):
+            errors.append(
+                f"Sample '{sample}': Duplicate replicate numbers detected. "
+                "Each replicate must have a unique number."
+            )
+
+    # Check all samples have R2 (ATAC-seq requires paired-end)
+    for i, row in enumerate(rows):
+        if not row.get("fastq_2"):
+            errors.append(
+                f"Row {i+2}: ATAC-seq requires paired-end data. R2 file missing."
+            )
+
+
+def validate_file_exists(path: str) -> bool:
+    """Check if file exists and is accessible."""
+    return os.path.isfile(path) and os.access(path, os.R_OK)
+
+
+def validate_absolute_path(path: str) -> bool:
+    """Check if path is absolute."""
+    return os.path.isabs(path)