@brainpilot/skills 0.0.6 → 0.0.8

package/skills/06_fMRI_Neuroimaging/tcp-skill/scripts/extract_tcp_phenotype.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""Extract and merge TCP phenotype data.
+
+Reads TCP phenotype files (diagnosis, dimensional measures)
+and produces a merged phenotype table aligned with imaging subject list.
+"""
+import argparse
+import csv
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional
+
+COLUMN_MAP = {
+    "subject_id": ["subject", "Subject", "participant_id", "SubID"],
+    "age": ["age", "Age"],
+    "sex": ["sex", "Sex", "Gender"],
+    "race": ["race", "Race"],
+    "ethnicity": ["ethnicity", "Ethnicity"],
+    "handedness": ["handedness", "Handedness"],
+    # Diagnosis
+    "diagnosis": ["diagnosis", "Diagnosis", "Group", "DX"],
+    "diagnosis_detail": ["diagnosis_detail", "Diagnosis_Detail"],
+    # RDoC dimensions
+    "negative_valence": ["negative_valence", "NegativeValence"],
+    "positive_valence": ["positive_valence", "PositiveValence"],
+    "cognitive": ["cognitive", "Cognitive"],
+    "social": ["social", "Social"],
+    "arousal": ["arousal", "Arousal"],
+    # Clinical measures
+    "bdi": ["BDI", "bdi"],
+    "bai": ["BAI", "bai"],
+    "panss_positive": ["PANSS_Pos", "panss_positive"],
+    "panss_negative": ["PANSS_Neg", "panss_negative"],
+    "ymrs": ["YMRS", "ymrs"],
+}
+
+
+def load_csv(path: Path) -> List[Dict[str, str]]:
+    delimiter = "\t" if path.suffix == ".tsv" else ","
+    with open(path, "r", encoding="utf-8") as f:
+        reader = csv.DictReader(f, delimiter=delimiter)
+        return list(reader)
+
+
+def find_column(row: Dict[str, str], candidates: List[str]) -> Optional[str]:
+    for col in candidates:
+        if col in row:
+            return col
+    return None
+
+
+def extract_phenotype(
+    phenotype_files: List[Path],
+    imaging_ids: Optional[List[str]] = None,
+    columns: Optional[List[str]] = None,
+) -> List[Dict[str, str]]:
+    all_data = []
+    for fpath in phenotype_files:
+        rows = load_csv(fpath)
+        all_data.extend(rows)
+
+    if not all_data:
+        return []
+
+    target_columns = columns if columns else list(COLUMN_MAP.keys())
+    merged = {}
+
+    for row in all_data:
+        subj_col = find_column(row, COLUMN_MAP["subject_id"])
+        if subj_col is None:
+            continue
+        subj_id = row[subj_col].strip()
+        if not subj_id:
+            continue
+
+        if subj_id not in merged:
+            merged[subj_id] = {"subject_id": subj_id}
+
+        for target_col in target_columns:
+            if target_col == "subject_id":
+                continue
+            if target_col in merged[subj_id] and merged[subj_id][target_col]:
+                continue
+            candidates = COLUMN_MAP.get(target_col, [target_col])
+            src_col = find_column(row, candidates)
+            if src_col and row.get(src_col, "").strip():
+                merged[subj_id][target_col] = row[src_col].strip()
+
+    result = list(merged.values())
+    if imaging_ids:
+        imaging_set = set(imaging_ids)
+        result = [r for r in result if r["subject_id"] in imaging_set]
+    return result
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Extract TCP phenotype data.")
+    parser.add_argument("--phenotype-files", required=True, nargs="+")
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--imaging-ids")
+    parser.add_argument("--columns")
+    args = parser.parse_args()
+
+    phenotype_files = [Path(f).resolve() for f in args.phenotype_files]
+    for f in phenotype_files:
+        if not f.exists():
+            print(f"File not found: {f}", file=sys.stderr)
+            return 1
+
+    imaging_ids = None
+    if args.imaging_ids:
+        id_file = Path(args.imaging_ids).resolve()
+        if id_file.exists():
+            imaging_ids = [l.strip() for l in id_file.read_text().splitlines() if l.strip()]
+
+    columns = None
+    if args.columns:
+        columns = [c.strip() for c in args.columns.split(",")]
+
+    merged = extract_phenotype(phenotype_files, imaging_ids, columns)
+    if not merged:
+        print("[ERROR] No phenotype data extracted.", file=sys.stderr)
+        return 1
+
+    output_path = Path(args.output).resolve()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    fieldnames = list(merged[0].keys())
+    with open(output_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(merged)
+
+    diag_groups = set(r.get("diagnosis", "") for r in merged if r.get("diagnosis"))
+    print(f"Phenotype: {len(merged)} subjects, {len(fieldnames)} columns, {len(diag_groups)} groups -> {output_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/skills/06_fMRI_Neuroimaging/tcp-skill/scripts/tcp_qc_summary.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Generate per-subject QC summaries for TCP processing.
+
+Transdiagnostic: tracks diagnostic group and applies appropriate QC criteria.
+"""
+import argparse
+import csv
+import sys
+from pathlib import Path
+from typing import Dict, List
+
+
+def load_confounds(confounds_path: Path) -> Dict[str, float]:
+    metrics = {"fd_mean": float("nan"), "fd_max": float("nan")}
+    try:
+        import pandas as pd
+        df = pd.read_csv(confounds_path, sep="\t")
+        if "framewise_displacement" in df.columns:
+            fd = df["framewise_displacement"].dropna()
+            metrics["fd_mean"] = float(fd.mean())
+            metrics["fd_max"] = float(fd.max())
+    except Exception:
+        pass
+    return metrics
+
+
+def check_exclusion(metrics: Dict[str, float], fd_threshold: float = 0.3) -> List[str]:
+    reasons = []
+    if not (metrics["fd_mean"] != metrics["fd_mean"]):
+        if metrics["fd_mean"] > fd_threshold:
+            reasons.append(f"FD mean {metrics['fd_mean']:.3f} > {fd_threshold}")
+    return reasons
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Generate QC summaries for TCP processing.")
+    parser.add_argument("--fmriprep-dir", help="Path to fMRIPrep output directory")
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--exclude-output")
+    parser.add_argument("--fd-threshold", type=float, default=0.3)
+    parser.add_argument("--diagnosis-file", help="CSV with subject diagnosis info")
+    args = parser.parse_args()
+
+    output_path = Path(args.output).resolve()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    diagnosis_map = {}
+    if args.diagnosis_file:
+        diag_path = Path(args.diagnosis_file).resolve()
+        if diag_path.exists():
+            with open(diag_path, "r", encoding="utf-8") as f:
+                reader = csv.DictReader(f)
+                for row in reader:
+                    subj = row.get("subject_id") or row.get("Subject", "")
+                    diag = row.get("diagnosis") or row.get("Diagnosis") or row.get("Group", "")
+                    if subj and diag:
+                        diagnosis_map[subj] = diag
+
+    subjects = set()
+    if args.fmriprep_dir:
+        fmriprep_dir = Path(args.fmriprep_dir).resolve()
+        if fmriprep_dir.exists():
+            for d in fmriprep_dir.glob("sub-*"):
+                if d.is_dir():
+                    subjects.add(d.name)
+
+    if not subjects:
+        print("[WARN] No subjects found.", file=sys.stderr)
+        return 1
+
+    results = []
+    excluded = []
+    for subj in sorted(subjects):
+        metrics = {"subject_id": subj}
+        if diagnosis_map:
+            metrics["diagnosis"] = diagnosis_map.get(subj, "unknown")
+
+        if args.fmriprep_dir:
+            confounds_files = list(Path(args.fmriprep_dir).glob(f"{subj}/func/*_desc-confounds_timeseries.tsv"))
+            if confounds_files:
+                metrics.update(load_confounds(confounds_files[0]))
+
+        exclusion_reasons = check_exclusion(metrics, args.fd_threshold)
+        metrics["excluded"] = len(exclusion_reasons) > 0
+        metrics["exclusion_reasons"] = "; ".join(exclusion_reasons)
+
+        results.append(metrics)
+        if exclusion_reasons:
+            excluded.append({"subject_id": subj, "reasons": "; ".join(exclusion_reasons)})
+
+    if results:
+        fieldnames = list(results[0].keys())
+        with open(output_path, "w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(results)
+        print(f"QC: {len(results)} subjects, {len(excluded)} excluded -> {output_path}")
+
+    if args.exclude_output and excluded:
+        exclude_path = Path(args.exclude_output).resolve()
+        exclude_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(exclude_path, "w", newline="", encoding="utf-8") as f:
+            writer = csv.DictWriter(f, fieldnames=["subject_id", "reasons"])
+            writer.writeheader()
+            writer.writerows(excluded)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/skills/06_fMRI_Neuroimaging/tcp-skill/scripts/validate_tcp.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Validate TCP BIDS structure and generate compliance report.
+
+Checks directory structure, modality completeness, and diagnostic group labeling.
+"""
+import argparse
+import csv
+import sys
+from pathlib import Path
+from typing import Dict, List
+
+
+def validate_subject(subject_dir: Path) -> Dict[str, any]:
+    report = {
+        "subject": subject_dir.name,
+        "anat_complete": False,
+        "rs_fMRI_present": False,
+        "dwi_present": False,
+        "missing_files": [],
+        "warnings": [],
+    }
+
+    anat_dir = subject_dir / "anat"
+    if anat_dir.exists():
+        t1w_files = list(anat_dir.glob("*_T1w.nii.gz"))
+        report["anat_complete"] = len(t1w_files) > 0
+        if not t1w_files:
+            report["missing_files"].append("anat/*_T1w.nii.gz")
+    else:
+        report["missing_files"].append("anat/")
+
+    func_dir = subject_dir / "func"
+    if func_dir.exists():
+        rest_bold = list(func_dir.glob("*_task-rest_bold.nii.gz"))
+        report["rs_fMRI_present"] = len(rest_bold) > 0
+        if not rest_bold:
+            report["missing_files"].append("func/*_task-rest_bold.nii.gz")
+    else:
+        report["warnings"].append("No func directory")
+
+    dwi_dir = subject_dir / "dwi"
+    if dwi_dir.exists():
+        dwi_files = list(dwi_dir.glob("*_dwi.nii.gz"))
+        report["dwi_present"] = len(dwi_files) > 0
+    else:
+        report["warnings"].append("No dwi directory")
+
+    return report
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Validate TCP BIDS structure.")
+    parser.add_argument("--input", required=True)
+    parser.add_argument("--output", required=True)
+    args = parser.parse_args()
+
+    input_dir = Path(args.input).resolve()
+    if not input_dir.exists():
+        print(f"Input directory not found: {input_dir}", file=sys.stderr)
+        return 1
+
+    subjects = sorted([d for d in input_dir.glob("sub-*") if d.is_dir()])
+    print(f"Found {len(subjects)} subjects in {input_dir}")
+
+    if not subjects:
+        print("[ERROR] No subjects found.", file=sys.stderr)
+        return 1
+
+    results = []
+    for sub_dir in subjects:
+        report = validate_subject(sub_dir)
+        results.append(report)
+
+    output_path = Path(args.output).resolve()
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    fieldnames = ["subject", "anat_complete", "rs_fMRI_present", "dwi_present", "missing_files", "warnings"]
+    with open(output_path, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        for r in results:
+            r["missing_files"] = "; ".join(r["missing_files"])
+            r["warnings"] = "; ".join(r["warnings"])
+            writer.writerow(r)
+
+    complete = sum(1 for r in results if r["anat_complete"] and r["rs_fMRI_present"])
+    print(f"\nValidation Summary:")
+    print(f"  Total subjects: {len(results)}")
+    print(f"  Complete (anat + rs-fMRI): {complete}")
+    print(f"  With T1w: {sum(1 for r in results if r['anat_complete'])}")
+    print(f"  With rs-fMRI: {sum(1 for r in results if r['rs_fMRI_present'])}")
+    print(f"  With dMRI: {sum(1 for r in results if r['dwi_present'])}")
+    print(f"  Output: {output_path}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/skills/06_fMRI_Neuroimaging/ucla-cnp-skill/SKILL.md

@@ -0,0 +1,217 @@
+---
+name: ucla-cnp-skill
+description: "Use this skill whenever the user wants an end-to-end workflow for the UCLA CNP (Consortium for Neuropsychiatric Phenomics) dataset, including BIDS validation, multimodal processing of sMRI, task-fMRI, and dMRI, phenotype extraction, and QC integration. Triggers include: 'UCLA CNP', 'Consortium Neuropsychiatric Phenomics', 'process UCLA CNP', or any request to run the UCLA CNP multimodal pipeline."
+license: MIT License (NeuroClaw custom skill - freely modifiable within the project)
+layer: subagent
+skill_type: dataset
+dependencies:
+  - smri-skill
+  - fmri-skill
+  - dwi-skill
+  - bids-organizer
+  - claw-shell
+complementary_skills:
+  - brain-visualization
+---
+# UCLA CNP Skill (Dataset-Orchestration Layer)
+
+## Overview
+
+`ucla-cnp-skill` is the NeuroClaw orchestration skill for the **UCLA CNP (Consortium for Neuropsychiatric Phenomics)** dataset, led by Russell Poldrack and colleagues at UCLA.
+
+It strictly follows the NeuroClaw hierarchical design principles:
+- This skill **only describes WHAT needs to be done** and **which tool skill to delegate to**.
+- It contains **no implementation code or concrete commands**.
+- All concrete execution is delegated to existing base/tool skills via `claw-shell`.
+- Companion scripts in `scripts/` provide reference implementations for BIDS validation, phenotype extraction, and QC.
+
+**Core workflow (never bypassed):**
+1. Identify input UCLA CNP data and target modalities.
+2. Generate a **numbered execution plan** clearly stating WHAT needs to be done and which tool skill will handle each step.
+3. Present the full plan, estimated runtime, resource requirements, and risks to the user and wait for explicit confirmation ("YES" / "execute" / "proceed").
+4. On confirmation, delegate every step to the appropriate skill via `claw-shell`.
+5. After execution, save all outputs in a clean directory structure (`ucla_cnp_output/`).
+
+**Research use only.**
+
+---
+
+## Quick Reference
+
+| Task | What needs to be done | Delegate to | Expected output |
+|---|---|---|---|
+| BIDS validation | Validate UCLA CNP BIDS structure | `scripts/validate_ucla_cnp.py` | Validation report |
+| sMRI processing | Brain extraction, tissue segmentation | `smri-skill` | `smri_output/` derivatives |
+| task-fMRI processing | Task GLM, activation analysis | `fmri-skill` | `fmri_output/` task results |
+| dMRI processing | Diffusion preprocessing, tractography | `dwi-skill` | `dwi_output/` metrics |
+| Phenotype extraction | Diagnosis, cognitive, behavioral | `scripts/extract_ucla_cnp_phenotype.py` | Merged phenotype CSV |
+| QC summary | Per-subject quality control | `scripts/ucla_cnp_qc_summary.py` | QC summary + exclusion list |
+
+---
+
+## Dataset Characteristics
+
+- **Cohort**: ~270 participants
+  - **Healthy controls**: Age-matched
+  - **ADHD**: Attention deficit hyperactivity disorder
+  - **Bipolar disorder**: Bipolar I/II
+  - **Schizophrenia**: Schizophrenia spectrum
+- **Scanner**: 3T Siemens TIM Trio (UCLA)
+- **Modalities**: T1w sMRI, task-fMRI, dMRI/DTI
+- **Tasks**: Multiple cognitive tasks (stop-signal, spatial working memory, etc.)
+- **Clinical**: Extensive cognitive battery, diagnostic assessments
+- **Access**: OpenNeuro ds000030
+- **Format**: BIDS-compliant
+- **Reference**: Poldrack et al. (2016), Scientific Data
+
+---
+
+## Supported Modalities
+
+| Modality | Description | Tasks/Conditions |
+|---|---|---|
+| T1w | High-resolution structural MRI | 1mm isotropic |
+| task-fMRI | Task-based functional MRI | Stop-signal, spatial WM, face memory, etc. |
+| dMRI | Diffusion-weighted imaging | DTI, white matter tractography |
+
+---
+
+## UCLA CNP Task Paradigms
+
+| Task | Description | Cognitive Domain |
+|---|---|---|
+| Stop-signal | Response inhibition | Executive function |
+| Spatial working memory | Spatial WM maintenance | Working memory |
+| Face memory | Face encoding/retrieval | Episodic memory |
+| Balloon analog risk | Risk taking behavior | Decision making |
+| Monetary incentive | Reward processing | Motivation |
+
+---
+
+## UCLA CNP Diagnostic Groups
+
+| Group | Description | Typical N |
+|---|---|---|
+| Control | Healthy age-matched | ~130 |
+| ADHD | Attention deficit | ~50 |
+| Bipolar | Bipolar disorder | ~45 |
+| Schizophrenia | Schizophrenia spectrum | ~45 |
+
+---
+
+## BIDS Preparation
+
+### Script: `scripts/validate_ucla_cnp.py`
+
+Validates UCLA CNP BIDS structure and generates a compliance report.
+
+```bash
+python skills/ucla-cnp-skill/scripts/validate_ucla_cnp.py \
+  --input /path/to/UCLA-CNP/bids \
+  --output /path/to/ucla_cnp_output/qc/bids_validation.csv
+```
+
+Features:
+- BIDS directory structure validation
+- Diagnostic group completeness check
+- Modality completeness (T1w, task-fMRI, dMRI)
+- Task paradigm presence verification
+
+---
+
+## Core Workflow (Never Bypassed)
+
+1. Identify user target: full UCLA CNP processing, imaging subset, phenotype extraction, or BIDS validation only.
+2. Generate a numbered plan with tools, outputs, runtime, storage, and risks.
+3. Wait for explicit confirmation (`YES` / `execute` / `proceed`).
+4. On confirmation, run BIDS validation using `scripts/validate_ucla_cnp.py`.
+5. Delegate to `smri-skill` for structural MRI processing.
+6. Delegate to `fmri-skill` for task-fMRI processing.
+7. Delegate to `dwi-skill` for dMRI processing.
+8. If phenotype extraction is requested, run `scripts/extract_ucla_cnp_phenotype.py`.
+9. If QC summary is requested, run `scripts/ucla_cnp_qc_summary.py`.
+10. Save outputs into `ucla_cnp_output/`.
+
+---
+
+## Modality Processing Delegation
+
+| Modality | Delegated skill | Typical tasks | Main outputs |
+|---|---|---|---|
+| sMRI (T1w) | `smri-skill` | brain extraction, tissue segmentation | `smri_output/` derivatives |
+| task-fMRI | `fmri-skill` | task GLM, activation analysis | `fmri_output/` task results |
+| dMRI | `dwi-skill` | diffusion preprocessing, tensor metrics | `dwi_output/` metrics |
+
+---
+
+## Standard Output Layout
+
+```
+ucla_cnp_output/
+├── bids/                   # BIDS-staged data (or validation report)
+├── smri/                   # Structural MRI derivatives
+├── fmri/                   # Functional MRI derivatives (task-fMRI)
+├── dwi/                    # Diffusion MRI derivatives
+├── phenotype/              # Merged phenotype tables (diagnosis, cognitive)
+├── qc/                     # QC summaries and exclusion lists
+└── logs/                   # Processing logs
+```
+
+---
+
+## Benchmark Adapter Guidance
+
+For benchmark-style prompts, do not force the full orchestration when the task only asks for local UCLA CNP data validation.
+
+- If the task starts from UCLA CNP data already present on disk and only asks for BIDS validation:
+  - Skip the download stage
+  - Default to the narrow path `local UCLA CNP discovery -> BIDS validation -> report`
+- In benchmark mode, do not require explicit confirmation before presenting the validation solution.
+
+---
+
+## Safety and Execution Policy
+- No execution before explicit plan confirmation.
+- All execution must be routed via `claw-shell`.
+- Missing dependencies must be resolved by `dependency-planner` before running.
+
+---
+
+## Important Notes and Limitations
+- UCLA CNP has 4 diagnostic groups; group comparisons should account for sample size differences.
+- Multiple cognitive tasks enable rich cognitive phenotyping.
+- OpenNeuro ds000030 is a widely used benchmark dataset for BIDS tools.
+- Extensive cognitive battery enables linking brain structure/function to behavior.
+- `ucla-cnp-skill` is orchestration-only; detailed preprocessing logic remains in modality skills.
+
+---
+
+## When to Call This Skill
+- User asks for end-to-end UCLA CNP workflow.
+- User asks to process UCLA CNP neuroimaging data.
+- User needs BIDS validation for UCLA CNP data.
+- User asks to extract UCLA CNP phenotype data (diagnosis, cognitive).
+- User asks for multi-disorder neuroimaging analysis.
+
+---
+
+## Complementary / Related Skills
+- `smri-skill` → structural MRI preprocessing
+- `fmri-skill` → functional MRI preprocessing and analysis
+- `dwi-skill` → diffusion MRI preprocessing
+- `bids-organizer` → BIDS validation and organization
+- `brain-visualization` → visualization of derivatives
+- `dependency-planner` → dependency resolution
+- `conda-env-manager` → environment management
+- `claw-shell` → command execution
+
+---
+
+## Reference
+- UCLA CNP: https://www.humanconnectome.org/
+- Poldrack et al. (2016): A phenomic analysis of the UCLA Consortium for Neuropsychiatric Phenomics LA5c Study. Scientific Data.
+- OpenNeuro ds000030
+
+Created At: 2026-05-06 14:21 HKT
+Last Updated At: 2026-05-06 14:21 HKT
+Author: chengwang96