opencode-knowledge-work 0.0.0-stage → 0.1.1

package/content/bio-research/skills/bio-research-scvi-tools/scripts/transfer_labels.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+"""
+Transfer cell type labels from reference to query using scANVI.
+
+Maps query cells to a pre-trained reference model and predicts cell types.
+
+Usage:
+    python transfer_labels.py reference_model/ query.h5ad output_dir/
+    python transfer_labels.py reference_model/ query.h5ad output_dir/ --confidence 0.7
+"""
+
+import argparse
+import os
+import sys
+
+
+def transfer_labels(
+    reference_model,
+    adata_query,
+    max_epochs=100,
+    confidence_threshold=0.5
+):
+    """
+    Transfer labels from reference to query.
+
+    Parameters
+    ----------
+    reference_model : SCANVI model
+        Pre-trained scANVI model
+    adata_query : AnnData
+        Query data to annotate
+    max_epochs : int
+        Fine-tuning epochs
+    confidence_threshold : float
+        Minimum confidence for predictions
+
+    Returns
+    -------
+    AnnData with predictions
+    """
+    import scvi
+    import numpy as np
+
+    # Get reference genes
+    ref_genes = reference_model.adata.var_names
+    print(f"Reference genes: {len(ref_genes)}")
+
+    # Check gene overlap
+    query_genes = adata_query.var_names
+    common = ref_genes.intersection(query_genes)
+    print(f"Query genes: {len(query_genes)}")
+    print(f"Common genes: {len(common)} ({len(common)/len(ref_genes)*100:.1f}%)")
+
+    if len(common) < len(ref_genes) * 0.5:
+        print("Warning: Less than 50% gene overlap. Results may be unreliable.")
+
+    # Subset query to reference genes
+    # Missing genes will be filled with zeros
+    adata_query = adata_query[:, adata_query.var_names.isin(ref_genes)].copy()
+
+    # Ensure counts layer
+    if "counts" not in adata_query.layers:
+        adata_query.layers["counts"] = adata_query.X.copy()
+
+    # Prepare query for mapping
+    print("Preparing query data...")
+    scvi.model.SCANVI.prepare_query_anndata(adata_query, reference_model)
+
+    # Create query model
+    print("Creating query model...")
+    query_model = scvi.model.SCANVI.load_query_data(
+        adata_query,
+        reference_model
+    )
+
+    # Fine-tune
+    print(f"Fine-tuning ({max_epochs} epochs)...")
+    query_model.train(
+        max_epochs=max_epochs,
+        plan_kwargs={"weight_decay": 0.0}
+    )
+
+    # Get predictions
+    print("Getting predictions...")
+    predictions = query_model.predict()
+    soft_predictions = query_model.predict(soft=True)
+
+    adata_query.obs["predicted_cell_type"] = predictions
+    adata_query.obs["prediction_confidence"] = soft_predictions.max(axis=1)
+    adata_query.obs["confident_prediction"] = adata_query.obs["prediction_confidence"] >= confidence_threshold
+
+    # Get latent representation
+    adata_query.obsm["X_scANVI"] = query_model.get_latent_representation()
+
+    # Stats
+    n_confident = adata_query.obs["confident_prediction"].sum()
+    print(f"\nPrediction summary:")
+    print(f"  Total cells: {adata_query.n_obs}")
+    print(f"  Confident (>= {confidence_threshold}): {n_confident} ({n_confident/adata_query.n_obs*100:.1f}%)")
+    print(f"  Mean confidence: {adata_query.obs['prediction_confidence'].mean():.3f}")
+
+    print("\nPredicted cell types:")
+    print(adata_query.obs["predicted_cell_type"].value_counts())
+
+    return adata_query, query_model
+
+
+def plot_predictions(adata, output_dir):
+    """Plot prediction results."""
+    import scanpy as sc
+    import matplotlib.pyplot as plt
+
+    # Compute UMAP if needed
+    if "X_umap" not in adata.obsm:
+        sc.pp.neighbors(adata, use_rep="X_scANVI")
+        sc.tl.umap(adata)
+
+    # Plot
+    fig, axes = plt.subplots(1, 3, figsize=(15, 4))
+
+    sc.pl.umap(adata, color="predicted_cell_type", ax=axes[0], show=False,
+               title="Predicted Cell Type")
+    sc.pl.umap(adata, color="prediction_confidence", ax=axes[1], show=False,
+               title="Prediction Confidence", cmap="viridis")
+    sc.pl.umap(adata, color="confident_prediction", ax=axes[2], show=False,
+               title="Confident Predictions")
+
+    plt.tight_layout()
+    plot_path = os.path.join(output_dir, "predictions.png")
+    plt.savefig(plot_path, dpi=150, bbox_inches="tight")
+    plt.close()
+    print(f"Prediction plot saved to {plot_path}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Transfer cell type labels using scANVI",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+    # Basic label transfer
+    python transfer_labels.py reference_model/ query.h5ad results/
+
+    # With confidence threshold
+    python transfer_labels.py reference_model/ query.h5ad results/ --confidence 0.7
+
+    # More fine-tuning
+    python transfer_labels.py reference_model/ query.h5ad results/ --max-epochs 200
+        """
+    )
+    parser.add_argument("model_dir", help="Directory containing reference scANVI model")
+    parser.add_argument("query", help="Query h5ad file to annotate")
+    parser.add_argument("output_dir", help="Output directory")
+    parser.add_argument("--reference-adata", help="Reference adata used for training (if not saved with model)")
+    parser.add_argument("--max-epochs", type=int, default=100,
+                        help="Fine-tuning epochs (default: 100)")
+    parser.add_argument("--confidence", type=float, default=0.5,
+                        help="Confidence threshold (default: 0.5)")
+
+    args = parser.parse_args()
+
+    try:
+        import scvi
+        import scanpy as sc
+    except ImportError:
+        print("Error: scvi-tools and scanpy required")
+        sys.exit(1)
+
+    # Create output directory
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    # Load query data
+    print(f"Loading query data: {args.query}")
+    adata_query = sc.read_h5ad(args.query)
+    print(f"Query: {adata_query.shape}")
+
+    # Load reference model
+    print(f"Loading reference model: {args.model_dir}")
+    if args.reference_adata:
+        ref_adata = sc.read_h5ad(args.reference_adata)
+        reference_model = scvi.model.SCANVI.load(args.model_dir, adata=ref_adata)
+    else:
+        # Try loading without adata (works if model was saved with adata)
+        try:
+            reference_model = scvi.model.SCANVI.load(args.model_dir)
+        except ValueError as e:
+            if "no saved anndata" in str(e).lower():
+                print("Error: Model was saved without adata. Please provide --reference-adata")
+                sys.exit(1)
+            raise
+    print(f"Reference: {reference_model.adata.shape}")
+
+    # Transfer labels
+    adata_annotated, query_model = transfer_labels(
+        reference_model,
+        adata_query,
+        max_epochs=args.max_epochs,
+        confidence_threshold=args.confidence
+    )
+
+    # Save results
+    adata_path = os.path.join(args.output_dir, "query_annotated.h5ad")
+    adata_annotated.write_h5ad(adata_path)
+    print(f"\nAnnotated data saved to {adata_path}")
+
+    # Save query model
+    model_path = os.path.join(args.output_dir, "query_model")
+    query_model.save(model_path)
+    print(f"Query model saved to {model_path}")
+
+    # Save predictions CSV
+    pred_df = adata_annotated.obs[["predicted_cell_type", "prediction_confidence", "confident_prediction"]]
+    pred_path = os.path.join(args.output_dir, "predictions.csv")
+    pred_df.to_csv(pred_path)
+    print(f"Predictions saved to {pred_path}")
+
+    # Plot
+    plot_predictions(adata_annotated, args.output_dir)
+
+    print("\nDone!")
+
+
+if __name__ == "__main__":
+    main()

package/content/bio-research/skills/bio-research-scvi-tools/scripts/validate_adata.py

@@ -0,0 +1,397 @@
+#!/usr/bin/env python3
+"""
+Validation utilities for checking AnnData compatibility with scvi-tools.
+
+Usage:
+    python validate_adata.py data.h5ad
+
+    # Or import as module
+    from validate_adata import validate_for_scvi, ValidationResult
+    result = validate_for_scvi(adata)
+    print(result.summary())
+"""
+
+import argparse
+import sys
+from dataclasses import dataclass, field
+from typing import Optional, List, Dict, Any
+import warnings
+
+
+@dataclass
+class ValidationResult:
+    """Results from AnnData validation."""
+
+    is_valid: bool = True
+    errors: List[str] = field(default_factory=list)
+    warnings: List[str] = field(default_factory=list)
+    info: Dict[str, Any] = field(default_factory=dict)
+    recommendations: List[str] = field(default_factory=list)
+
+    def add_error(self, msg: str):
+        """Add an error (makes validation fail)."""
+        self.errors.append(msg)
+        self.is_valid = False
+
+    def add_warning(self, msg: str):
+        """Add a warning (doesn't fail validation)."""
+        self.warnings.append(msg)
+
+    def add_recommendation(self, msg: str):
+        """Add a recommendation for improvement."""
+        self.recommendations.append(msg)
+
+    def summary(self) -> str:
+        """Generate summary report."""
+        lines = []
+        lines.append("=" * 60)
+        lines.append("scvi-tools AnnData Validation Report")
+        lines.append("=" * 60)
+
+        # Status
+        status = "PASSED" if self.is_valid else "FAILED"
+        lines.append(f"\nStatus: {status}")
+
+        # Info
+        if self.info:
+            lines.append("\n--- Data Summary ---")
+            for key, value in self.info.items():
+                lines.append(f"  {key}: {value}")
+
+        # Errors
+        if self.errors:
+            lines.append(f"\n--- Errors ({len(self.errors)}) ---")
+            for i, err in enumerate(self.errors, 1):
+                lines.append(f"  {i}. {err}")
+
+        # Warnings
+        if self.warnings:
+            lines.append(f"\n--- Warnings ({len(self.warnings)}) ---")
+            for i, warn in enumerate(self.warnings, 1):
+                lines.append(f"  {i}. {warn}")
+
+        # Recommendations
+        if self.recommendations:
+            lines.append(f"\n--- Recommendations ({len(self.recommendations)}) ---")
+            for i, rec in enumerate(self.recommendations, 1):
+                lines.append(f"  {i}. {rec}")
+
+        lines.append("\n" + "=" * 60)
+        return "\n".join(lines)
+
+
+def validate_for_scvi(
+    adata,
+    layer: Optional[str] = None,
+    batch_key: Optional[str] = None,
+    labels_key: Optional[str] = None,
+    check_hvg: bool = True
+) -> ValidationResult:
+    """
+    Validate AnnData for scvi-tools compatibility.
+
+    Parameters
+    ----------
+    adata : AnnData
+        Data to validate
+    layer : str, optional
+        Layer containing counts (if None, checks X)
+    batch_key : str, optional
+        Expected batch column in obs
+    labels_key : str, optional
+        Expected labels column in obs
+    check_hvg : bool
+        Check for highly variable genes
+
+    Returns
+    -------
+    ValidationResult with errors, warnings, and recommendations
+    """
+    import numpy as np
+    from scipy.sparse import issparse
+
+    result = ValidationResult()
+
+    # Basic info
+    result.info["shape"] = f"{adata.n_obs} cells x {adata.n_vars} genes"
+    result.info["layers"] = list(adata.layers.keys()) if adata.layers else "None"
+
+    # Get data matrix to check
+    if layer is not None:
+        if layer not in adata.layers:
+            result.add_error(f"Layer '{layer}' not found. Available: {list(adata.layers.keys())}")
+            return result
+        X = adata.layers[layer]
+        result.info["checking"] = f"layer '{layer}'"
+    else:
+        X = adata.X
+        result.info["checking"] = "adata.X"
+
+    # Check for None or empty
+    if X is None:
+        result.add_error("Data matrix is None")
+        return result
+
+    if X.shape[0] == 0 or X.shape[1] == 0:
+        result.add_error(f"Data matrix is empty: shape {X.shape}")
+        return result
+
+    # Convert to array for checking
+    if issparse(X):
+        result.info["sparse"] = True
+        X_check = X.data  # Just check non-zero values
+    else:
+        result.info["sparse"] = False
+        X_check = X.flatten()
+
+    # Check for raw counts (integers)
+    if len(X_check) > 0:
+        is_integer = np.allclose(X_check, X_check.astype(int))
+        result.info["contains_integers"] = is_integer
+
+        if not is_integer:
+            result.add_error(
+                "Data does not contain integers (raw counts required). "
+                "Found float values - data may be normalized."
+            )
+            result.add_recommendation(
+                "Use adata.raw.to_adata() to recover raw counts, "
+                "or specify a layer with raw counts"
+            )
+
+    # Check for negative values
+    min_val = X.min()
+    if min_val < 0:
+        result.add_error(f"Data contains negative values (min={min_val})")
+
+    # Check for NaN/Inf
+    if issparse(X):
+        has_nan = np.isnan(X.data).any()
+        has_inf = np.isinf(X.data).any()
+    else:
+        has_nan = np.isnan(X).any()
+        has_inf = np.isinf(X).any()
+
+    if has_nan:
+        result.add_error("Data contains NaN values")
+    if has_inf:
+        result.add_error("Data contains Inf values")
+
+    # Check data range
+    max_val = X.max()
+    result.info["value_range"] = f"[{min_val}, {max_val}]"
+
+    if max_val < 10:
+        result.add_warning(
+            f"Maximum value is {max_val}, which is very low. "
+            "Data may be log-transformed or normalized."
+        )
+
+    # Check sparsity
+    if issparse(X):
+        sparsity = 1 - (X.nnz / (X.shape[0] * X.shape[1]))
+        result.info["sparsity"] = f"{sparsity:.1%}"
+
+        if sparsity < 0.5:
+            result.add_warning(
+                f"Data is only {sparsity:.1%} sparse. "
+                "Consider if this is expected for your data type."
+            )
+
+    # Check batch key
+    if batch_key is not None:
+        if batch_key not in adata.obs.columns:
+            result.add_error(
+                f"batch_key '{batch_key}' not found in obs. "
+                f"Available columns: {list(adata.obs.columns)}"
+            )
+        else:
+            n_batches = adata.obs[batch_key].nunique()
+            result.info["n_batches"] = n_batches
+
+            if n_batches == 1:
+                result.add_warning(
+                    "Only 1 batch found. Batch correction may not be needed."
+                )
+
+            # Check for small batches
+            batch_counts = adata.obs[batch_key].value_counts()
+            small_batches = batch_counts[batch_counts < 50]
+            if len(small_batches) > 0:
+                result.add_warning(
+                    f"{len(small_batches)} batches have fewer than 50 cells. "
+                    "Consider merging small batches."
+                )
+
+    # Check labels key
+    if labels_key is not None:
+        if labels_key not in adata.obs.columns:
+            result.add_error(
+                f"labels_key '{labels_key}' not found in obs. "
+                f"Available columns: {list(adata.obs.columns)}"
+            )
+        else:
+            n_labels = adata.obs[labels_key].nunique()
+            result.info["n_labels"] = n_labels
+
+            # Check for rare labels
+            label_counts = adata.obs[labels_key].value_counts()
+            rare_labels = label_counts[label_counts < 30]
+            if len(rare_labels) > 0:
+                result.add_warning(
+                    f"{len(rare_labels)} cell types have fewer than 30 cells. "
+                    "Rare types may not be well learned."
+                )
+
+    # Check HVG
+    if check_hvg:
+        if 'highly_variable' not in adata.var.columns:
+            result.add_recommendation(
+                "No highly variable genes found. Run sc.pp.highly_variable_genes() "
+                "and subset to HVGs for better performance."
+            )
+        else:
+            n_hvg = adata.var['highly_variable'].sum()
+            result.info["n_hvg"] = n_hvg
+
+            if n_hvg < 1000:
+                result.add_warning(
+                    f"Only {n_hvg} HVGs selected. Consider using 2000-4000 for best results."
+                )
+            elif n_hvg > 5000:
+                result.add_warning(
+                    f"{n_hvg} HVGs selected. Consider reducing to 2000-4000 "
+                    "for efficiency."
+                )
+
+    # Check gene count
+    if adata.n_vars > 30000:
+        result.add_recommendation(
+            f"Dataset has {adata.n_vars} genes. Subset to HVGs (2000-4000) "
+            "for faster training and better results."
+        )
+
+    # Check cell count
+    if adata.n_obs < 1000:
+        result.add_warning(
+            f"Dataset has only {adata.n_obs} cells. "
+            "Deep learning models work best with >5000 cells."
+        )
+
+    # Check for counts layer
+    if layer is None and 'counts' not in adata.layers:
+        result.add_recommendation(
+            "Store raw counts in adata.layers['counts'] before any normalization. "
+            "This preserves the original data for scvi-tools."
+        )
+
+    # Check for raw attribute
+    if adata.raw is not None:
+        result.info["has_raw"] = True
+        result.add_recommendation(
+            "adata.raw exists. If X is normalized, use adata.raw.to_adata() "
+            "to recover raw counts."
+        )
+    else:
+        result.info["has_raw"] = False
+
+    return result
+
+
+def suggest_model(adata, result: ValidationResult) -> str:
+    """
+    Suggest appropriate scvi-tools model based on data.
+
+    Parameters
+    ----------
+    adata : AnnData
+        Data to analyze
+    result : ValidationResult
+        Validation result with info
+
+    Returns
+    -------
+    String with model suggestion
+    """
+    suggestions = []
+
+    # Check for multi-modal data
+    if 'protein_expression' in adata.obsm:
+        suggestions.append("totalVI: CITE-seq data detected (protein + RNA)")
+
+    if 'spliced' in adata.layers and 'unspliced' in adata.layers:
+        suggestions.append("veloVI: RNA velocity data detected (spliced + unspliced)")
+
+    # Check for labels
+    has_labels = result.info.get('n_labels', 0) > 0
+    has_batches = result.info.get('n_batches', 0) > 1
+
+    if has_batches:
+        if has_labels:
+            suggestions.append(
+                "scANVI: Integration with cell type labels (recommended for label transfer)"
+            )
+        else:
+            suggestions.append(
+                "scVI: Unsupervised batch integration"
+            )
+    else:
+        suggestions.append(
+            "scVI: Dimensionality reduction and differential expression"
+        )
+
+    if not suggestions:
+        suggestions.append("scVI: General-purpose single-cell analysis")
+
+    return "\n".join([f"  - {s}" for s in suggestions])
+
+
+def main():
+    """Command-line interface."""
+    parser = argparse.ArgumentParser(
+        description="Validate AnnData for scvi-tools compatibility"
+    )
+    parser.add_argument("file", help="Path to h5ad file")
+    parser.add_argument("--layer", help="Layer to check (default: X)")
+    parser.add_argument("--batch-key", help="Batch column to check")
+    parser.add_argument("--labels-key", help="Labels column to check")
+    parser.add_argument("--suggest", action="store_true", help="Suggest model type")
+
+    args = parser.parse_args()
+
+    try:
+        import scanpy as sc
+    except ImportError:
+        print("Error: scanpy is required. Install with: pip install scanpy")
+        sys.exit(1)
+
+    # Load data
+    print(f"Loading {args.file}...")
+    try:
+        adata = sc.read_h5ad(args.file)
+    except Exception as e:
+        print(f"Error loading file: {e}")
+        sys.exit(1)
+
+    # Validate
+    result = validate_for_scvi(
+        adata,
+        layer=args.layer,
+        batch_key=args.batch_key,
+        labels_key=args.labels_key
+    )
+
+    # Print report
+    print(result.summary())
+
+    # Suggest model
+    if args.suggest:
+        print("\nSuggested models:")
+        print(suggest_model(adata, result))
+
+    # Exit code
+    sys.exit(0 if result.is_valid else 1)
+
+
+if __name__ == "__main__":
+    main()