@brainpilot/skills 0.0.6 → 0.0.8

package/skills/06_fMRI_Neuroimaging/abcd-skill/scripts/extract_abcd_phenotype.py

@@ -0,0 +1,292 @@
+#!/usr/bin/env python3
+"""Extract and merge ABCD Study phenotype tables for downstream analysis.
+
+Reads ABCD tab-delimited phenotype files, selects columns, aligns visits,
+and optionally cross-references with imaging subject lists.
+"""
+import argparse
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional, Set
+
+try:
+    import pandas as pd
+except ImportError:
+    print("Error: pandas is required. Install with: pip install pandas", file=sys.stderr)
+    sys.exit(1)
+
+# Standard ABCD event names in temporal order
+ABCD_EVENTS = [
+    "baselineYear1Arm1",
+    "1YearFollowUpYArm1",
+    "2YearFollowUpYArm1",
+    "3YearFollowUpYArm1",
+    "4YearFollowUpYArm1",
+]
+
+# Default phenotype files to look for
+DEFAULT_PHENOTYPE_FILES = [
+    "abcd_p_tab.csv",
+    "abcd_p_tab.tsv",
+    "mental_health.csv",
+    "mental_health.tsv",
+    "cbcl.csv",
+    "cbcl.tsv",
+    "ksads.csv",
+    "ksads.tsv",
+    "nihtb.csv",
+    "nihtb.tsv",
+    "abcd_imgincl01.csv",
+    "abcd_imgincl01.tsv",
+]
+
+
+def detect_delimiter(file_path: Path) -> str:
+    """Detect whether file uses tab or comma delimiter."""
+    with open(file_path, "r", encoding="utf-8") as f:
+        first_line = f.readline()
+    if "\t" in first_line:
+        return "\t"
+    return ","
+
+
+def read_phenotype_file(file_path: Path) -> Optional["pd.DataFrame"]:
+    """Read a single ABCD phenotype file."""
+    delimiter = detect_delimiter(file_path)
+    try:
+        df = pd.read_csv(file_path, sep=delimiter, low_memory=False)
+        return df
+    except Exception as e:
+        print(f"[WARN] Failed to read {file_path}: {e}", file=sys.stderr)
+        return None
+
+
+def find_common_columns(dataframes: List["pd.DataFrame"]) -> List[str]:
+    """Find columns present in all dataframes."""
+    if not dataframes:
+        return []
+    common = set(dataframes[0].columns)
+    for df in dataframes[1:]:
+        common &= set(df.columns)
+    return sorted(common)
+
+
+def merge_phenotype_tables(
+    phenotype_dir: Path,
+    columns: Optional[List[str]] = None,
+    events: Optional[List[str]] = None,
+    imaging_ids: Optional[Set[str]] = None,
+    drop_missing_threshold: float = 0.5,
+) -> "pd.DataFrame":
+    """Merge multiple ABCD phenotype tables.
+
+    Args:
+        phenotype_dir: Directory containing phenotype CSV/TSV files.
+        columns: Specific columns to select (None = all).
+        events: ABCD event names to include (None = all).
+        imaging_ids: Set of subject IDs from imaging data to filter by.
+        drop_missing_threshold: Drop columns with > this fraction of missing values.
+
+    Returns:
+        Merged DataFrame.
+    """
+    phenotype_files = []
+    for f in sorted(phenotype_dir.iterdir()):
+        if f.is_file() and f.suffix in (".csv", ".tsv"):
+            phenotype_files.append(f)
+
+    if not phenotype_files:
+        # Try default filenames
+        for name in DEFAULT_PHENOTYPE_FILES:
+            candidate = phenotype_dir / name
+            if candidate.exists():
+                phenotype_files.append(candidate)
+
+    if not phenotype_files:
+        print(f"[ERROR] No phenotype files found in {phenotype_dir}", file=sys.stderr)
+        return pd.DataFrame()
+
+    print(f"Found {len(phenotype_files)} phenotype files:")
+    for f in phenotype_files:
+        print(f"  - {f.name}")
+
+    # Read all files
+    dataframes = []
+    for f in phenotype_files:
+        df = read_phenotype_file(f)
+        if df is not None and len(df) > 0:
+            # Standardize ID column name
+            if "src_subject_id" in df.columns:
+                df = df.rename(columns={"src_subject_id": "subject_id"})
+            elif "SUBJECTKEY" in df.columns:
+                df = df.rename(columns={"SUBJECTKEY": "subject_id"})
+            dataframes.append(df)
+
+    if not dataframes:
+        return pd.DataFrame()
+
+    # Find common columns across all tables
+    common_cols = find_common_columns(dataframes)
+    print(f"Common columns across all tables: {len(common_cols)}")
+
+    # Merge on subject_id + eventname
+    merge_keys = []
+    if "subject_id" in common_cols:
+        merge_keys.append("subject_id")
+    if "eventname" in common_cols:
+        merge_keys.append("eventname")
+
+    if not merge_keys:
+        print("[WARN] No merge keys found (subject_id/eventname). Concatenating instead.")
+        merged = pd.concat(dataframes, ignore_index=True)
+    else:
+        merged = dataframes[0]
+        for df in dataframes[1:]:
+            # Columns to merge (excluding merge keys and duplicates)
+            new_cols = [c for c in df.columns if c not in merged.columns or c in merge_keys]
+            df_subset = df[new_cols]
+            merged = pd.merge(merged, df_subset, on=merge_keys, how="outer", suffixes=("", "_dup"))
+
+    # Drop duplicate columns
+    dup_cols = [c for c in merged.columns if c.endswith("_dup")]
+    if dup_cols:
+        merged = merged.drop(columns=dup_cols)
+
+    # Filter by events
+    if events and "eventname" in merged.columns:
+        merged = merged[merged["eventname"].isin(events)]
+        print(f"Filtered to events: {events} -> {len(merged)} rows")
+
+    # Filter by imaging IDs
+    if imaging_ids and "subject_id" in merged.columns:
+        # Normalize imaging IDs for matching
+        normalized_imaging = set()
+        for sid in imaging_ids:
+            clean = sid.replace("sub-", "")
+            normalized_imaging.add(clean)
+            normalized_imaging.add(sid)
+
+        before_count = len(merged)
+        mask = merged["subject_id"].apply(
+            lambda x: str(x).strip() in normalized_imaging
+            or f"sub-{str(x).strip()}" in normalized_imaging
+        )
+        merged = merged[mask]
+        print(f"Filtered to imaging subjects: {before_count} -> {len(merged)} rows")
+
+    # Select specific columns
+    if columns:
+        available = [c for c in columns if c in merged.columns]
+        missing = [c for c in columns if c not in merged.columns]
+        if missing:
+            print(f"[WARN] Columns not found (skipped): {missing}")
+        if available:
+            merged = merged[available]
+
+    # Drop columns with too many missing values
+    if drop_missing_threshold < 1.0:
+        missing_frac = merged.isnull().mean()
+        cols_to_drop = missing_frac[missing_frac > drop_missing_threshold].index.tolist()
+        if cols_to_drop:
+            print(f"Dropping {len(cols_to_drop)} columns with >{drop_missing_threshold*100}% missing")
+            merged = merged.drop(columns=cols_to_drop)
+
+    return merged
+
+
+def load_imaging_ids(imaging_ids_file: Path) -> Set[str]:
+    """Load subject IDs from a BIDS participants.tsv or similar file."""
+    ids = set()
+    delimiter = detect_delimiter(imaging_ids_file)
+    try:
+        df = pd.read_csv(imaging_ids_file, sep=delimiter)
+        id_col = None
+        for col_name in ["participant_id", "subject_id", "SUBJECTKEY", "src_subject_id"]:
+            if col_name in df.columns:
+                id_col = col_name
+                break
+        if id_col:
+            ids = set(df[id_col].astype(str).str.strip())
+    except Exception as e:
+        print(f"[WARN] Failed to read imaging IDs: {e}", file=sys.stderr)
+    return ids
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Extract and merge ABCD Study phenotype tables."
+    )
+    parser.add_argument(
+        "--phenotype-dir",
+        required=True,
+        help="Directory containing ABCD phenotype CSV/TSV files",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Output path for merged phenotype CSV",
+    )
+    parser.add_argument(
+        "--columns",
+        help="Comma-separated list of columns to select (default: all)",
+    )
+    parser.add_argument(
+        "--events",
+        help=f"Comma-separated ABCD event names to include. Options: {', '.join(ABCD_EVENTS)}",
+    )
+    parser.add_argument(
+        "--imaging-ids",
+        help="Path to BIDS participants.tsv or file with imaging subject IDs",
+    )
+    parser.add_argument(
+        "--missing-threshold",
+        type=float,
+        default=0.5,
+        help="Drop columns with more than this fraction of missing values (default: 0.5)",
+    )
+    args = parser.parse_args()
+
+    phenotype_dir = Path(args.phenotype_dir).resolve()
+    if not phenotype_dir.exists() or not phenotype_dir.is_dir():
+        print(f"Phenotype directory does not exist: {phenotype_dir}", file=sys.stderr)
+        return 1
+
+    # Parse options
+    columns = None
+    if args.columns:
+        columns = [c.strip() for c in args.columns.split(",")]
+
+    events = None
+    if args.events:
+        events = [e.strip() for e in args.events.split(",")]
+
+    imaging_ids = None
+    if args.imaging_ids:
+        imaging_ids_path = Path(args.imaging_ids).resolve()
+        if imaging_ids_path.exists():
+            imaging_ids = load_imaging_ids(imaging_ids_path)
+            print(f"Loaded {len(imaging_ids)} imaging subject IDs")
+
+    # Merge phenotype tables
+    merged = merge_phenotype_tables(
+        phenotype_dir=phenotype_dir,
+        columns=columns,
+        events=events,
+        imaging_ids=imaging_ids,
+        drop_missing_threshold=args.missing_threshold,
+    )
+
+    if merged.empty:
+        print("[ERROR] No data after merging. Check input files.", file=sys.stderr)
+        return 1
+
+    # Write output
+    output_path = Path(args.output).resolve()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    merged.to_csv(output_path, index=False)
+    print(f"\nWrote {len(merged)} rows x {len(merged.columns)} columns to {output_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/skills/06_fMRI_Neuroimaging/abcd-skill/scripts/reorganize_abcd.py

@@ -0,0 +1,387 @@
+#!/usr/bin/env python3
+"""Reorganize ABCD Study raw data into BIDS-compliant directory structure.
+
+Handles NDAR-style subject IDs, ABCD event names, and multimodal routing
+for T1w, T2w, dMRI, rs-fMRI, and task-fMRI.
+"""
+import argparse
+import json
+import os
+import re
+import shutil
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, List, Optional, Set, Tuple
+
+# ABCD event name -> BIDS session label mapping
+ABCD_SESSION_MAP = {
+    "baselineYear1Arm1": "baselineYear1Arm1",
+    "1YearFollowUpYArm1": "1YearFollowUpYArm1",
+    "2YearFollowUpYArm1": "2YearFollowUpYArm1",
+    "3YearFollowUpYArm1": "3YearFollowUpYArm1",
+    "4YearFollowUpYArm1": "4YearFollowUpYArm1",
+}
+
+# Modality keywords -> BIDS modality mapping
+MODALITY_PATTERNS = [
+    (re.compile(r"T1w|t1w|T1|mprage", re.IGNORECASE), "T1w", "anat"),
+    (re.compile(r"T2w|t2w|T2", re.IGNORECASE), "T2w", "anat"),
+    (re.compile(r"dMRI|dmri|DWI|dwi|DTI|dti", re.IGNORECASE), "dwi", "dwi"),
+    (re.compile(r"rsfMRI|rs-fMRI|rest_bold|task-rest", re.IGNORECASE), "task-rest_bold", "func"),
+    (re.compile(r"task-fMRI|tfMRI|task-[a-zA-Z]+_bold", re.IGNORECASE), None, "func"),  # dynamic task name
+]
+
+SIDECAR_EXTENSIONS = [".json", ".bval", ".bvec", ".tsv"]
+
+
+def normalize_ndar_id(raw_id: str) -> str:
+    """Convert NDAR subject ID to BIDS-compatible label.
+
+    NDAR_INV000ABC0 -> sub-NDARINV000ABC0
+    """
+    clean = raw_id.strip()
+    if clean.startswith("sub-"):
+        clean = clean[4:]
+    # Remove any non-alphanumeric characters except underscore
+    clean = re.sub(r"[^a-zA-Z0-9_]", "", clean)
+    return f"sub-{clean}"
+
+
+def normalize_session(event_name: str) -> str:
+    """Convert ABCD event name to BIDS session label."""
+    event_name = event_name.strip()
+    if event_name.startswith("ses-"):
+        event_name = event_name[4:]
+    if event_name in ABCD_SESSION_MAP:
+        return f"ses-{ABCD_SESSION_MAP[event_name]}"
+    # Fallback: sanitize
+    clean = re.sub(r"[^a-zA-Z0-9]", "", event_name)
+    return f"ses-{clean}"
+
+
+def detect_modality(filename: str) -> Optional[Tuple[str, str, str]]:
+    """Detect BIDS modality suffix and folder from filename.
+
+    Returns (bids_suffix, bids_folder, task_name) or None.
+    """
+    for pattern, suffix, folder in MODALITY_PATTERNS:
+        if pattern.search(filename):
+            # For task-fMRI, extract task name if present
+            if folder == "func" and suffix is None:
+                task_match = re.search(r"task-([a-zA-Z]+)", filename)
+                if task_match:
+                    task_name = task_match.group(1)
+                    return (f"task-{task_name}_bold", "func", task_name)
+                return ("task-unknown_bold", "func", "unknown")
+            return (suffix, folder, "")
+    return None
+
+
+def find_nifti_files(directory: Path) -> List[Path]:
+    """Find all NIfTI files in a directory."""
+    results = []
+    for f in directory.rglob("*"):
+        if f.is_file() and (f.name.endswith(".nii") or f.name.endswith(".nii.gz")):
+            results.append(f)
+    return results
+
+
+def copy_with_sidecars(src_nifti: Path, dst_dir: Path, dst_stem: str) -> List[Path]:
+    """Copy NIfTI file and its sidecar files to destination."""
+    dst_dir.mkdir(parents=True, exist_ok=True)
+    copied = []
+
+    # Copy main NIfTI
+    ext = ".nii.gz" if src_nifti.name.endswith(".nii.gz") else ".nii"
+    dst_nifti = dst_dir / f"{dst_stem}{ext}"
+    if not dst_nifti.exists():
+        shutil.copy2(str(src_nifti), str(dst_nifti))
+    copied.append(dst_nifti)
+
+    # Copy sidecars
+    src_stem = src_nifti.name
+    if src_stem.endswith(".nii.gz"):
+        src_stem = src_stem[:-7]
+    elif src_stem.endswith(".nii"):
+        src_stem = src_stem[:-4]
+
+    for sidecar_ext in SIDECAR_EXTENSIONS:
+        src_sidecar = src_nifti.parent / f"{src_stem}{sidecar_ext}"
+        if src_sidecar.exists():
+            dst_sidecar = dst_dir / f"{dst_stem}{sidecar_ext}"
+            if not dst_sidecar.exists():
+                shutil.copy2(str(src_sidecar), str(dst_sidecar))
+            copied.append(dst_sidecar)
+
+    return copied
+
+
+def write_dataset_description(bids_root: Path, release_version: str = "5.1") -> None:
+    """Write BIDS dataset_description.json."""
+    desc = {
+        "Name": f"ABCD Study {release_version}",
+        "BIDSVersion": "1.8.0",
+        "DatasetType": "raw",
+        "GeneratedBy": [
+            {
+                "Name": "NeuroClaw abcd-skill",
+                "Description": "ABCD raw data reorganized to BIDS structure",
+                "Version": "1.0.0",
+            }
+        ],
+    }
+    desc_path = bids_root / "dataset_description.json"
+    with open(desc_path, "w", encoding="utf-8") as f:
+        json.dump(desc, f, indent=2, ensure_ascii=False)
+    print(f"[OK] Wrote {desc_path}")
+
+
+def write_participants_tsv(bids_root: Path, subject_ids: List[str]) -> None:
+    """Write BIDS participants.tsv header."""
+    tsv_path = bids_root / "participants.tsv"
+    with open(tsv_path, "w", encoding="utf-8") as f:
+        f.write("participant_id\n")
+        for sid in sorted(set(subject_ids)):
+            f.write(f"{sid}\n")
+    print(f"[OK] Wrote {tsv_path} ({len(set(subject_ids))} participants)")
+
+
+def detect_subject_layout(input_dir: Path) -> str:
+    """Detect whether input follows flat or nested ABCD layout.
+
+    Returns 'flat' if subject dirs are direct children,
+    'nested' if organized as subject/session/modality,
+    'ndar_package' if following NDA download package structure.
+    """
+    children = list(input_dir.iterdir())
+    if not children:
+        return "flat"
+
+    # Check for NDA package structure (collection directories)
+    for child in children:
+        if child.is_dir() and child.name.startswith("collection_"):
+            return "ndar_package"
+
+    # Check for subject/session/modality nesting
+    first_child = children[0]
+    if first_child.is_dir():
+        sub_children = list(first_child.iterdir())
+        if sub_children:
+            # If children have modality-like names, it's flat
+            for sc in sub_children:
+                if sc.is_dir() and detect_modality(sc.name):
+                    return "flat"
+            # If children look like session names, it's nested
+            for sc in sub_children:
+                if sc.is_dir() and any(
+                    k in sc.name for k in ["baseline", "Year", "FollowUp"]
+                ):
+                    return "nested"
+
+    return "flat"
+
+
+def process_flat_layout(
+    input_dir: Path,
+    bids_root: Path,
+    participants: List[str],
+    dry_run: bool = False,
+) -> Tuple[int, int, int]:
+    """Process flat ABCD layout: subject/modality/files."""
+    converted = 0
+    skipped = 0
+    failed = 0
+
+    subject_dirs = sorted(
+        [p for p in input_dir.iterdir() if p.is_dir() and not p.name.startswith(".")]
+    )
+
+    for subject_dir in subject_dirs:
+        sub_label = normalize_ndar_id(subject_dir.name)
+        participants.append(sub_label)
+
+        for modality_dir in sorted(subject_dir.iterdir()):
+            if not modality_dir.is_dir():
+                continue
+
+            nifti_files = find_nifti_files(modality_dir)
+            for nifti in nifti_files:
+                result = detect_modality(nifti.name)
+                if result is None:
+                    # Try parent folder name
+                    result = detect_modality(modality_dir.name)
+                if result is None:
+                    print(f"[WARN] {sub_label} | cannot detect modality: {nifti.name}")
+                    failed += 1
+                    continue
+
+                bids_suffix, bids_folder, task_name = result
+
+                # Default session
+                ses_label = "ses-baselineYear1Arm1"
+
+                bids_sub_dir = bids_root / sub_label / ses_label / bids_folder
+                dst_stem = f"{sub_label}_{ses_label}_{bids_suffix}"
+
+                if (bids_sub_dir / f"{dst_stem}.nii").exists() or (
+                    bids_sub_dir / f"{dst_stem}.nii.gz"
+                ).exists():
+                    skipped += 1
+                    continue
+
+                if dry_run:
+                    print(f"[DRY] {sub_label}/{ses_label}/{bids_folder}/{dst_stem}")
+                    converted += 1
+                else:
+                    copy_with_sidecars(nifti, bids_sub_dir, dst_stem)
+                    print(f"[OK] {sub_label}/{ses_label}/{bids_folder}/{dst_stem}")
+                    converted += 1
+
+    return converted, skipped, failed
+
+
+def process_nested_layout(
+    input_dir: Path,
+    bids_root: Path,
+    participants: List[str],
+    dry_run: bool = False,
+) -> Tuple[int, int, int]:
+    """Process nested ABCD layout: subject/session/modality/files."""
+    converted = 0
+    skipped = 0
+    failed = 0
+
+    subject_dirs = sorted(
+        [p for p in input_dir.iterdir() if p.is_dir() and not p.name.startswith(".")]
+    )
+
+    for subject_dir in subject_dirs:
+        sub_label = normalize_ndar_id(subject_dir.name)
+        participants.append(sub_label)
+
+        for session_dir in sorted(subject_dir.iterdir()):
+            if not session_dir.is_dir():
+                continue
+
+            ses_label = normalize_session(session_dir.name)
+
+            for modality_dir in sorted(session_dir.iterdir()):
+                if not modality_dir.is_dir():
+                    continue
+
+                nifti_files = find_nifti_files(modality_dir)
+                if not nifti_files:
+                    # Try detecting modality from folder name
+                    result = detect_modality(modality_dir.name)
+                    if result:
+                        nifti_files = find_nifti_files(session_dir)
+                        nifti_files = [
+                            f
+                            for f in nifti_files
+                            if f.parent == modality_dir or f.parent == session_dir
+                        ]
+
+                for nifti in nifti_files:
+                    result = detect_modality(nifti.name)
+                    if result is None:
+                        result = detect_modality(modality_dir.name)
+                    if result is None:
+                        print(
+                            f"[WARN] {sub_label}/{ses_label} | cannot detect modality: {nifti.name}"
+                        )
+                        failed += 1
+                        continue
+
+                    bids_suffix, bids_folder, task_name = result
+                    bids_sub_dir = bids_root / sub_label / ses_label / bids_folder
+                    dst_stem = f"{sub_label}_{ses_label}_{bids_suffix}"
+
+                    if (bids_sub_dir / f"{dst_stem}.nii").exists() or (
+                        bids_sub_dir / f"{dst_stem}.nii.gz"
+                    ).exists():
+                        skipped += 1
+                        continue
+
+                    if dry_run:
+                        print(
+                            f"[DRY] {sub_label}/{ses_label}/{bids_folder}/{dst_stem}"
+                        )
+                        converted += 1
+                    else:
+                        copy_with_sidecars(nifti, bids_sub_dir, dst_stem)
+                        print(
+                            f"[OK] {sub_label}/{ses_label}/{bids_folder}/{dst_stem}"
+                        )
+                        converted += 1
+
+    return converted, skipped, failed
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Reorganize ABCD Study raw data into BIDS-compliant structure."
+    )
+    parser.add_argument(
+        "--input",
+        required=True,
+        help="Path to ABCD raw data directory",
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Path to output BIDS directory",
+    )
+    parser.add_argument(
+        "--participants-file",
+        help="Optional: path to ABCD phenotype file for participant metadata",
+    )
+    parser.add_argument(
+        "--release-version",
+        default="5.1",
+        help="ABCD release version (default: 5.1)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Preview without copying files",
+    )
+    args = parser.parse_args()
+
+    input_dir = Path(args.input).resolve()
+    output_dir = Path(args.output).resolve()
+
+    if not input_dir.exists() or not input_dir.is_dir():
+        print(f"Input directory does not exist: {input_dir}", file=sys.stderr)
+        return 1
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    participants: List[str] = []
+
+    # Detect layout
+    layout = detect_subject_layout(input_dir)
+    print(f"Detected layout: {layout}")
+    print(f"Input:  {input_dir}")
+    print(f"Output: {output_dir}")
+    print(f"{'[DRY RUN] ' if args.dry_run else ''}Starting reorganization...\n")
+
+    if layout == "nested":
+        converted, skipped, failed = process_nested_layout(
+            input_dir, output_dir, participants, args.dry_run
+        )
+    else:
+        converted, skipped, failed = process_flat_layout(
+            input_dir, output_dir, participants, args.dry_run
+        )
+
+    # Write BIDS metadata
+    if not args.dry_run and converted > 0:
+        write_dataset_description(output_dir, args.release_version)
+        write_participants_tsv(output_dir, participants)
+
+    print(f"\nDone. Converted={converted}, Skipped={skipped}, Failed={failed}")
+    return 0 if failed == 0 else 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())