@bgicli/bgicli 2.1.1 → 2.2.0

package/data/workflows/bulk-omics-clustering/scripts/density_clustering.py

@@ -0,0 +1,321 @@
+"""
+Density-based clustering (DBSCAN and HDBSCAN).
+
+This module provides density-based clustering methods that can find
+arbitrary-shaped clusters and identify noise/outliers.
+"""
+
+import numpy as np
+from sklearn.cluster import DBSCAN
+from typing import Tuple, Optional
+import warnings
+
+# HDBSCAN is optional but recommended
+try:
+    import hdbscan
+    HDBSCAN_AVAILABLE = True
+except ImportError:
+    HDBSCAN_AVAILABLE = False
+    warnings.warn("HDBSCAN not available. Install with: pip install hdbscan")
+
+
+def hdbscan_clustering(
+    data: np.ndarray,
+    min_cluster_size: int = 10,
+    min_samples: Optional[int] = None,
+    metric: str = "euclidean",
+    cluster_selection_method: str = "eom",
+    plot_hierarchy: bool = False
+) -> Tuple[np.ndarray, np.ndarray, int]:
+    """
+    Perform HDBSCAN (Hierarchical Density-Based Spatial Clustering).
+
+    HDBSCAN advantages:
+    - Automatically determines number of clusters
+    - Finds arbitrary-shaped clusters
+    - Identifies noise/outliers (label = -1)
+    - More stable than DBSCAN
+    - Provides cluster membership probabilities
+
+    Parameters
+    ----------
+    data : np.ndarray
+        Data matrix (samples × features)
+    min_cluster_size : int, default=10
+        Minimum number of samples in a cluster
+        Smaller = more clusters; Larger = fewer, denser clusters
+    min_samples : int, optional
+        Number of samples in neighborhood for core point
+        If None, uses min_cluster_size
+        Higher = more conservative (denser clusters)
+    metric : str, default="euclidean"
+        Distance metric
+    cluster_selection_method : str, default="eom"
+        Method for selecting clusters from hierarchy:
+        - "eom": Excess of Mass (default, good general choice)
+        - "leaf": Selects leaf clusters (more granular)
+    plot_hierarchy : bool, default=False
+        If True, plot cluster hierarchy
+
+    Returns
+    -------
+    cluster_labels : np.ndarray
+        Cluster assignments (noise points labeled as -1)
+    probabilities : np.ndarray
+        Cluster membership probabilities (0-1)
+    n_clusters : int
+        Number of clusters found (excluding noise)
+    """
+
+    if not HDBSCAN_AVAILABLE:
+        raise ImportError("HDBSCAN not installed. Install with: pip install hdbscan")
+
+    print(f"Performing HDBSCAN clustering...")
+    print(f"  min_cluster_size={min_cluster_size}, min_samples={min_samples}")
+
+    if min_samples is None:
+        min_samples = min_cluster_size
+
+    clusterer = hdbscan.HDBSCAN(
+        min_cluster_size=min_cluster_size,
+        min_samples=min_samples,
+        metric=metric,
+        cluster_selection_method=cluster_selection_method,
+        prediction_data=True  # Enable prediction on new data
+    )
+
+    cluster_labels = clusterer.fit_predict(data)
+    probabilities = clusterer.probabilities_
+
+    # Count clusters (excluding noise = -1)
+    n_clusters = len(set(cluster_labels)) - (1 if -1 in cluster_labels else 0)
+    n_noise = np.sum(cluster_labels == -1)
+
+    print(f"\nFound {n_clusters} clusters")
+    print(f"Noise points: {n_noise} ({n_noise / len(cluster_labels) * 100:.1f}%)")
+
+    # Report cluster sizes
+    if n_clusters > 0:
+        unique_labels = np.unique(cluster_labels[cluster_labels >= 0])
+        sizes = [np.sum(cluster_labels == label) for label in unique_labels]
+        print("Cluster sizes:", dict(zip(unique_labels, sizes)))
+
+    # Plot hierarchy if requested
+    if plot_hierarchy and hasattr(clusterer, 'condensed_tree_'):
+        try:
+            import matplotlib.pyplot as plt
+            clusterer.condensed_tree_.plot(select_clusters=True, selection_palette=plt.cm.viridis)
+            plt.title('HDBSCAN Cluster Hierarchy')
+            plt.show()
+        except Exception as e:
+            print(f"Could not plot hierarchy: {e}")
+
+    return cluster_labels, probabilities, n_clusters
+
+
+def dbscan_clustering(
+    data: np.ndarray,
+    eps: float,
+    min_samples: int = 5,
+    metric: str = "euclidean"
+) -> Tuple[np.ndarray, int]:
+    """
+    Perform DBSCAN (Density-Based Spatial Clustering of Applications with Noise).
+
+    Note: HDBSCAN is generally preferred as it doesn't require eps parameter.
+
+    Parameters
+    ----------
+    data : np.ndarray
+        Data matrix (samples × features)
+    eps : float
+        Maximum distance between samples to be considered neighbors
+        Critical parameter - requires tuning
+    min_samples : int, default=5
+        Minimum samples in neighborhood for core point
+    metric : str, default="euclidean"
+        Distance metric
+
+    Returns
+    -------
+    cluster_labels : np.ndarray
+        Cluster assignments (noise points labeled as -1)
+    n_clusters : int
+        Number of clusters found (excluding noise)
+    """
+
+    print(f"Performing DBSCAN clustering (eps={eps}, min_samples={min_samples})...")
+
+    clusterer = DBSCAN(
+        eps=eps,
+        min_samples=min_samples,
+        metric=metric
+    )
+
+    cluster_labels = clusterer.fit_predict(data)
+
+    # Count clusters (excluding noise = -1)
+    n_clusters = len(set(cluster_labels)) - (1 if -1 in cluster_labels else 0)
+    n_noise = np.sum(cluster_labels == -1)
+
+    print(f"\nFound {n_clusters} clusters")
+    print(f"Noise points: {n_noise} ({n_noise / len(cluster_labels) * 100:.1f}%)")
+
+    if n_clusters > 0:
+        unique_labels = np.unique(cluster_labels[cluster_labels >= 0])
+        sizes = [np.sum(cluster_labels == label) for label in unique_labels]
+        print("Cluster sizes:", dict(zip(unique_labels, sizes)))
+
+    return cluster_labels, n_clusters
+
+
+def estimate_dbscan_eps(
+    data: np.ndarray,
+    min_samples: int = 5,
+    percentile: float = 90
+) -> float:
+    """
+    Estimate eps parameter for DBSCAN using k-distance plot.
+
+    Parameters
+    ----------
+    data : np.ndarray
+        Data matrix (samples × features)
+    min_samples : int, default=5
+        min_samples parameter to use
+    percentile : float, default=90
+        Percentile of k-distances to use as eps estimate
+
+    Returns
+    -------
+    eps : float
+        Estimated eps parameter
+    """
+
+    from sklearn.neighbors import NearestNeighbors
+
+    print(f"Estimating DBSCAN eps parameter (k={min_samples})...")
+
+    # Fit k-nearest neighbors
+    nbrs = NearestNeighbors(n_neighbors=min_samples)
+    nbrs.fit(data)
+
+    # Get distances to k-th nearest neighbor
+    distances, _ = nbrs.kneighbors(data)
+    k_distances = np.sort(distances[:, -1])
+
+    # Use specified percentile as eps estimate
+    eps = np.percentile(k_distances, percentile)
+
+    print(f"Suggested eps: {eps:.4f} ({percentile}th percentile of k-distances)")
+    print(f"k-distance range: [{k_distances.min():.4f}, {k_distances.max():.4f}]")
+
+    return eps
+
+
+def tune_hdbscan_min_cluster_size(
+    data: np.ndarray,
+    min_cluster_sizes: list,
+    metric: str = "euclidean"
+) -> dict:
+    """
+    Try different min_cluster_size values to find optimal setting.
+
+    Parameters
+    ----------
+    data : np.ndarray
+        Data matrix (samples × features)
+    min_cluster_sizes : list
+        List of min_cluster_size values to try
+    metric : str, default="euclidean"
+        Distance metric
+
+    Returns
+    -------
+    results : dict
+        Dictionary with results for each min_cluster_size:
+        - "n_clusters": number of clusters found
+        - "n_noise": number of noise points
+        - "cluster_labels": cluster assignments
+    """
+
+    if not HDBSCAN_AVAILABLE:
+        raise ImportError("HDBSCAN not installed")
+
+    print(f"\nTuning HDBSCAN min_cluster_size parameter...")
+
+    results = {}
+
+    for mcs in min_cluster_sizes:
+        labels, probs, n_clusters = hdbscan_clustering(
+            data,
+            min_cluster_size=mcs,
+            metric=metric,
+            plot_hierarchy=False
+        )
+
+        n_noise = np.sum(labels == -1)
+
+        results[mcs] = {
+            "n_clusters": n_clusters,
+            "n_noise": n_noise,
+            "noise_pct": n_noise / len(labels) * 100,
+            "cluster_labels": labels,
+            "probabilities": probs
+        }
+
+        print(f"  min_cluster_size={mcs}: {n_clusters} clusters, {n_noise} noise ({n_noise/len(labels)*100:.1f}%)")
+
+    return results
+
+
+def get_cluster_persistence(
+    data: np.ndarray,
+    cluster_labels: np.ndarray,
+    probabilities: np.ndarray,
+    threshold: float = 0.5
+) -> dict:
+    """
+    Analyze cluster persistence/stability from HDBSCAN probabilities.
+
+    Parameters
+    ----------
+    data : np.ndarray
+        Data matrix
+    cluster_labels : np.ndarray
+        Cluster assignments from HDBSCAN
+    probabilities : np.ndarray
+        Cluster membership probabilities from HDBSCAN
+    threshold : float, default=0.5
+        Probability threshold for "confident" assignments
+
+    Returns
+    -------
+    persistence_info : dict
+        Dictionary with persistence statistics per cluster
+    """
+
+    unique_labels = np.unique(cluster_labels[cluster_labels >= 0])
+
+    persistence_info = {}
+
+    for label in unique_labels:
+        mask = cluster_labels == label
+        cluster_probs = probabilities[mask]
+
+        persistence_info[int(label)] = {
+            "size": mask.sum(),
+            "mean_probability": cluster_probs.mean(),
+            "median_probability": np.median(cluster_probs),
+            "min_probability": cluster_probs.min(),
+            "n_confident": np.sum(cluster_probs >= threshold),
+            "pct_confident": np.sum(cluster_probs >= threshold) / len(cluster_probs) * 100
+        }
+
+    print("\nCluster Persistence Analysis:")
+    print("Cluster\tSize\tMean Prob\tConfident %")
+    print("-" * 50)
+    for label, info in persistence_info.items():
+        print(f"{label}\t{info['size']}\t{info['mean_probability']:.3f}\t\t{info['pct_confident']:.1f}%")
+
+    return persistence_info

package/data/workflows/bulk-omics-clustering/scripts/dimensionality_reduction.py

@@ -0,0 +1,328 @@
+"""
+Dimensionality reduction for clustering preprocessing.
+
+This module provides PCA, UMAP, and t-SNE implementations for
+reducing dimensionality before clustering or for visualization.
+"""
+
+import numpy as np
+from sklearn.decomposition import PCA
+from sklearn.manifold import TSNE
+from typing import Tuple, Optional, Union
+import warnings
+
+# UMAP is optional
+try:
+    import umap
+    UMAP_AVAILABLE = True
+except ImportError:
+    UMAP_AVAILABLE = False
+    warnings.warn("UMAP not available. Install with: pip install umap-learn")
+
+
+def apply_pca(
+    data: np.ndarray,
+    n_components: Optional[int] = None,
+    variance_threshold: Optional[float] = None,
+    plot_variance: bool = False,
+    random_state: int = 42
+) -> Tuple[np.ndarray, PCA, np.ndarray]:
+    """
+    Apply Principal Component Analysis (PCA) for dimensionality reduction.
+
+    Parameters
+    ----------
+    data : np.ndarray
+        Data matrix (samples × features)
+    n_components : int, optional
+        Number of components to keep. If None, uses variance_threshold
+    variance_threshold : float, optional
+        Keep components explaining this fraction of variance (e.g., 0.95)
+        Only used if n_components is None
+    plot_variance : bool, default=False
+        If True, display variance explained information
+    random_state : int, default=42
+        Random state for reproducibility
+
+    Returns
+    -------
+    pca_data : np.ndarray
+        Transformed data (samples × n_components)
+    pca_model : PCA
+        Fitted PCA model
+    explained_variance : np.ndarray
+        Fraction of variance explained by each component
+    """
+
+    print("Applying PCA...")
+
+    if n_components is None and variance_threshold is None:
+        # Default: keep 95% variance
+        variance_threshold = 0.95
+
+    if n_components is not None:
+        # Use specified number of components
+        pca = PCA(n_components=n_components, random_state=random_state)
+    else:
+        # Use variance threshold
+        pca = PCA(n_components=variance_threshold, random_state=random_state)
+
+    pca_data = pca.fit_transform(data)
+
+    explained_variance = pca.explained_variance_ratio_
+
+    print(f"Reduced from {data.shape[1]} to {pca_data.shape[1]} dimensions")
+    print(f"Total variance explained: {explained_variance.sum():.2%}")
+
+    if plot_variance:
+        _print_variance_summary(explained_variance)
+
+    return pca_data, pca, explained_variance
+
+
+def apply_umap(
+    data: np.ndarray,
+    n_neighbors: int = 15,
+    min_dist: float = 0.1,
+    n_components: int = 2,
+    metric: str = "euclidean",
+    random_state: int = 42
+) -> np.ndarray:
+    """
+    Apply UMAP (Uniform Manifold Approximation and Projection).
+
+    UMAP is useful for:
+    - Non-linear dimensionality reduction
+    - Visualization (n_components=2 or 3)
+    - Can also be used for clustering (e.g., n_components=10-50)
+
+    Parameters
+    ----------
+    data : np.ndarray
+        Data matrix (samples × features)
+    n_neighbors : int, default=15
+        Size of local neighborhood (larger = more global structure)
+    min_dist : float, default=0.1
+        Minimum distance between points in embedding (smaller = more clustered)
+    n_components : int, default=2
+        Number of dimensions in output (2 for visualization, 10-50 for clustering)
+    metric : str, default="euclidean"
+        Distance metric (euclidean, manhattan, cosine, correlation)
+    random_state : int, default=42
+        Random state for reproducibility
+
+    Returns
+    -------
+    umap_embedding : np.ndarray
+        UMAP-transformed data (samples × n_components)
+    """
+
+    if not UMAP_AVAILABLE:
+        raise ImportError("UMAP not installed. Install with: pip install umap-learn")
+
+    print(f"Applying UMAP (n_components={n_components}, n_neighbors={n_neighbors})...")
+
+    reducer = umap.UMAP(
+        n_neighbors=n_neighbors,
+        min_dist=min_dist,
+        n_components=n_components,
+        metric=metric,
+        random_state=random_state
+    )
+
+    umap_embedding = reducer.fit_transform(data)
+
+    print(f"Reduced from {data.shape[1]} to {umap_embedding.shape[1]} dimensions")
+
+    return umap_embedding
+
+
+def apply_tsne(
+    data: np.ndarray,
+    n_components: int = 2,
+    perplexity: float = 30.0,
+    learning_rate: Union[float, str] = "auto",
+    n_iter: int = 1000,
+    metric: str = "euclidean",
+    random_state: int = 42
+) -> np.ndarray:
+    """
+    Apply t-SNE (t-Distributed Stochastic Neighbor Embedding).
+
+    Note: t-SNE is primarily for visualization (2D/3D), not for clustering.
+    Distances in t-SNE space are not meaningful.
+
+    Parameters
+    ----------
+    data : np.ndarray
+        Data matrix (samples × features)
+    n_components : int, default=2
+        Number of dimensions in output (typically 2 or 3)
+    perplexity : float, default=30.0
+        Related to number of nearest neighbors (5-50 typical range)
+        Larger datasets can use larger perplexity
+    learning_rate : float or "auto", default="auto"
+        Learning rate for optimization
+    n_iter : int, default=1000
+        Number of iterations
+    metric : str, default="euclidean"
+        Distance metric
+    random_state : int, default=42
+        Random state for reproducibility
+
+    Returns
+    -------
+    tsne_embedding : np.ndarray
+        t-SNE embedding (samples × n_components)
+    """
+
+    print(f"Applying t-SNE (perplexity={perplexity}, n_iter={n_iter})...")
+    print("Note: t-SNE is for visualization only, not for clustering input")
+
+    # Recommend PCA preprocessing for large feature spaces
+    if data.shape[1] > 50:
+        print(f"Warning: {data.shape[1]} features. Consider PCA preprocessing first.")
+
+    tsne = TSNE(
+        n_components=n_components,
+        perplexity=perplexity,
+        learning_rate=learning_rate,
+        n_iter=n_iter,
+        metric=metric,
+        random_state=random_state,
+        verbose=0
+    )
+
+    tsne_embedding = tsne.fit_transform(data)
+
+    print(f"Reduced from {data.shape[1]} to {tsne_embedding.shape[1]} dimensions")
+
+    return tsne_embedding
+
+
+def _print_variance_summary(explained_variance: np.ndarray, top_n: int = 10):
+    """Print summary of variance explained by top components."""
+
+    cumulative_variance = np.cumsum(explained_variance)
+
+    print("\nVariance explained by principal components:")
+    print("PC\tIndividual\tCumulative")
+    print("-" * 35)
+
+    n_show = min(top_n, len(explained_variance))
+
+    for i in range(n_show):
+        print(f"{i+1}\t{explained_variance[i]:.4f}\t\t{cumulative_variance[i]:.4f}")
+
+    if len(explained_variance) > top_n:
+        print("...")
+        last_idx = len(explained_variance) - 1
+        print(f"{last_idx+1}\t{explained_variance[last_idx]:.4f}\t\t{cumulative_variance[last_idx]:.4f}")
+
+
+def determine_optimal_pca_components(
+    data: np.ndarray,
+    max_components: Optional[int] = None,
+    variance_thresholds: list = [0.80, 0.90, 0.95, 0.99]
+) -> dict:
+    """
+    Determine how many PCA components are needed for different variance thresholds.
+
+    Parameters
+    ----------
+    data : np.ndarray
+        Data matrix (samples × features)
+    max_components : int, optional
+        Maximum number of components to test. If None, uses min(n_samples, n_features)
+    variance_thresholds : list, default=[0.80, 0.90, 0.95, 0.99]
+        Variance thresholds to report
+
+    Returns
+    -------
+    results : dict
+        Dictionary with keys:
+        - "n_components": number of components for each threshold
+        - "explained_variance": variance explained by each component
+        - "cumulative_variance": cumulative variance explained
+    """
+
+    if max_components is None:
+        max_components = min(data.shape[0], data.shape[1])
+
+    print(f"Analyzing PCA components (max={max_components})...")
+
+    pca = PCA(n_components=max_components)
+    pca.fit(data)
+
+    explained_variance = pca.explained_variance_ratio_
+    cumulative_variance = np.cumsum(explained_variance)
+
+    # Find number of components for each threshold
+    n_components_dict = {}
+    for threshold in variance_thresholds:
+        n_comp = np.argmax(cumulative_variance >= threshold) + 1
+        n_components_dict[threshold] = n_comp
+
+    print("\nComponents needed for variance thresholds:")
+    for threshold, n_comp in n_components_dict.items():
+        print(f"  {threshold:.0%} variance: {n_comp} components")
+
+    results = {
+        "n_components": n_components_dict,
+        "explained_variance": explained_variance,
+        "cumulative_variance": cumulative_variance
+    }
+
+    return results
+
+
+def recommend_dimensionality_reduction(
+    n_samples: int,
+    n_features: int,
+    purpose: str = "clustering"
+) -> str:
+    """
+    Recommend dimensionality reduction strategy based on data size and purpose.
+
+    Parameters
+    ----------
+    n_samples : int
+        Number of samples
+    n_features : int
+        Number of features
+    purpose : str, default="clustering"
+        Purpose of reduction: "clustering" or "visualization"
+
+    Returns
+    -------
+    recommendation : str
+        Text recommendation
+    """
+
+    recommendations = []
+
+    if purpose == "clustering":
+        if n_features > 1000:
+            recommendations.append("✓ Use PCA before clustering (keep 80-95% variance)")
+            recommendations.append(f"  Suggested: {int(min(50, n_samples * 0.5))} components")
+
+        elif n_features > 100:
+            recommendations.append("✓ Consider PCA for efficiency (optional)")
+            recommendations.append(f"  Suggested: {int(min(30, n_features * 0.5))} components")
+
+        else:
+            recommendations.append("✓ Can cluster in original feature space")
+            recommendations.append("  PCA not required (<100 features)")
+
+    elif purpose == "visualization":
+        recommendations.append("✓ Use UMAP or t-SNE for 2D visualization")
+
+        if n_features > 50:
+            recommendations.append("  Preprocess with PCA (50 components) before UMAP/t-SNE")
+        else:
+            recommendations.append("  Can apply UMAP/t-SNE directly")
+
+        if n_samples > 5000:
+            recommendations.append("  Note: t-SNE slow on large data, prefer UMAP")
+
+    return "\n".join(recommendations)