@synsci/cli-darwin-x64-baseline 1.1.76 → 1.1.78

package/bin/skills/anndata/references/concatenation.md

@@ -0,0 +1,396 @@
+# Concatenating AnnData Objects
+
+Combine multiple AnnData objects along either observations or variables axis.
+
+## Basic Concatenation
+
+### Concatenate along observations (stack cells/samples)
+```python
+import anndata as ad
+import numpy as np
+
+# Create multiple AnnData objects
+adata1 = ad.AnnData(X=np.random.rand(100, 50))
+adata2 = ad.AnnData(X=np.random.rand(150, 50))
+adata3 = ad.AnnData(X=np.random.rand(200, 50))
+
+# Concatenate along observations (axis=0, default)
+adata_combined = ad.concat([adata1, adata2, adata3], axis=0)
+
+print(adata_combined.shape)  # (450, 50)
+```
+
+### Concatenate along variables (stack genes/features)
+```python
+# Create objects with same observations, different variables
+adata1 = ad.AnnData(X=np.random.rand(100, 50))
+adata2 = ad.AnnData(X=np.random.rand(100, 30))
+adata3 = ad.AnnData(X=np.random.rand(100, 70))
+
+# Concatenate along variables (axis=1)
+adata_combined = ad.concat([adata1, adata2, adata3], axis=1)
+
+print(adata_combined.shape)  # (100, 150)
+```
+
+## Join Types
+
+### Inner join (intersection)
+Keep only variables/observations present in all objects.
+
+```python
+import pandas as pd
+
+# Create objects with different variables
+adata1 = ad.AnnData(
+    X=np.random.rand(100, 50),
+    var=pd.DataFrame(index=[f'Gene_{i}' for i in range(50)])
+)
+adata2 = ad.AnnData(
+    X=np.random.rand(150, 60),
+    var=pd.DataFrame(index=[f'Gene_{i}' for i in range(10, 70)])
+)
+
+# Inner join: only genes 10-49 are kept (overlap)
+adata_inner = ad.concat([adata1, adata2], join='inner')
+print(adata_inner.n_vars)  # 40 genes (overlap)
+```
+
+### Outer join (union)
+Keep all variables/observations, filling missing values.
+
+```python
+# Outer join: all genes are kept
+adata_outer = ad.concat([adata1, adata2], join='outer')
+print(adata_outer.n_vars)  # 70 genes (union)
+
+# Missing values are filled with appropriate defaults:
+# - 0 for sparse matrices
+# - NaN for dense matrices
+```
+
+### Fill values for outer joins
+```python
+# Specify fill value for missing data
+adata_filled = ad.concat([adata1, adata2], join='outer', fill_value=0)
+```
+
+## Tracking Data Sources
+
+### Add batch labels
+```python
+# Label which object each observation came from
+adata_combined = ad.concat(
+    [adata1, adata2, adata3],
+    label='batch',  # Column name for labels
+    keys=['batch1', 'batch2', 'batch3']  # Labels for each object
+)
+
+print(adata_combined.obs['batch'].value_counts())
+# batch1    100
+# batch2    150
+# batch3    200
+```
+
+### Automatic batch labels
+```python
+# If keys not provided, uses integer indices
+adata_combined = ad.concat(
+    [adata1, adata2, adata3],
+    label='dataset'
+)
+# dataset column contains: 0, 1, 2
+```
+
+## Merge Strategies
+
+Control how metadata from different objects is combined using the `merge` parameter.
+
+### merge=None (default for observations)
+Exclude metadata on non-concatenation axis.
+
+```python
+# When concatenating observations, var metadata must match
+adata1.var['gene_type'] = 'protein_coding'
+adata2.var['gene_type'] = 'protein_coding'
+
+# var is kept only if identical across all objects
+adata_combined = ad.concat([adata1, adata2], merge=None)
+```
+
+### merge='same'
+Keep metadata that is identical across all objects.
+
+```python
+adata1.var['chromosome'] = ['chr1'] * 25 + ['chr2'] * 25
+adata2.var['chromosome'] = ['chr1'] * 25 + ['chr2'] * 25
+adata1.var['type'] = 'protein_coding'
+adata2.var['type'] = 'lncRNA'  # Different
+
+# 'chromosome' is kept (same), 'type' is excluded (different)
+adata_combined = ad.concat([adata1, adata2], merge='same')
+```
+
+### merge='unique'
+Keep metadata columns where each key has exactly one value.
+
+```python
+adata1.var['gene_id'] = [f'ENSG{i:05d}' for i in range(50)]
+adata2.var['gene_id'] = [f'ENSG{i:05d}' for i in range(50)]
+
+# gene_id is kept (unique values for each key)
+adata_combined = ad.concat([adata1, adata2], merge='unique')
+```
+
+### merge='first'
+Take values from the first object containing each key.
+
+```python
+adata1.var['description'] = ['Desc1'] * 50
+adata2.var['description'] = ['Desc2'] * 50
+
+# Uses descriptions from adata1
+adata_combined = ad.concat([adata1, adata2], merge='first')
+```
+
+### merge='only'
+Keep metadata that appears in only one object.
+
+```python
+adata1.var['adata1_specific'] = [1] * 50
+adata2.var['adata2_specific'] = [2] * 50
+
+# Both metadata columns are kept
+adata_combined = ad.concat([adata1, adata2], merge='only')
+```
+
+## Handling Index Conflicts
+
+### Make indices unique
+```python
+import pandas as pd
+
+# Create objects with overlapping observation names
+adata1 = ad.AnnData(
+    X=np.random.rand(3, 10),
+    obs=pd.DataFrame(index=['cell_1', 'cell_2', 'cell_3'])
+)
+adata2 = ad.AnnData(
+    X=np.random.rand(3, 10),
+    obs=pd.DataFrame(index=['cell_1', 'cell_2', 'cell_3'])
+)
+
+# Make indices unique by appending batch keys
+adata_combined = ad.concat(
+    [adata1, adata2],
+    label='batch',
+    keys=['batch1', 'batch2'],
+    index_unique='_'  # Separator for making indices unique
+)
+
+print(adata_combined.obs_names)
+# ['cell_1_batch1', 'cell_2_batch1', 'cell_3_batch1',
+#  'cell_1_batch2', 'cell_2_batch2', 'cell_3_batch2']
+```
+
+## Concatenating Layers
+
+```python
+# Objects with layers
+adata1 = ad.AnnData(X=np.random.rand(100, 50))
+adata1.layers['normalized'] = np.random.rand(100, 50)
+adata1.layers['scaled'] = np.random.rand(100, 50)
+
+adata2 = ad.AnnData(X=np.random.rand(150, 50))
+adata2.layers['normalized'] = np.random.rand(150, 50)
+adata2.layers['scaled'] = np.random.rand(150, 50)
+
+# Layers are concatenated automatically if present in all objects
+adata_combined = ad.concat([adata1, adata2])
+
+print(adata_combined.layers.keys())
+# dict_keys(['normalized', 'scaled'])
+```
+
+## Concatenating Multi-dimensional Annotations
+
+### obsm/varm
+```python
+# Objects with embeddings
+adata1.obsm['X_pca'] = np.random.rand(100, 50)
+adata2.obsm['X_pca'] = np.random.rand(150, 50)
+
+# obsm is concatenated along observation axis
+adata_combined = ad.concat([adata1, adata2])
+print(adata_combined.obsm['X_pca'].shape)  # (250, 50)
+```
+
+### obsp/varp (pairwise annotations)
+```python
+from scipy.sparse import csr_matrix
+
+# Pairwise matrices
+adata1.obsp['connectivities'] = csr_matrix((100, 100))
+adata2.obsp['connectivities'] = csr_matrix((150, 150))
+
+# By default, obsp is NOT concatenated (set pairwise=True to include)
+adata_combined = ad.concat([adata1, adata2])
+# adata_combined.obsp is empty
+
+# Include pairwise data (creates block diagonal matrix)
+adata_combined = ad.concat([adata1, adata2], pairwise=True)
+print(adata_combined.obsp['connectivities'].shape)  # (250, 250)
+```
+
+## Concatenating uns (unstructured)
+
+Unstructured metadata is merged recursively:
+
+```python
+adata1.uns['experiment'] = {'date': '2025-01-01', 'batch': 'A'}
+adata2.uns['experiment'] = {'date': '2025-01-01', 'batch': 'B'}
+
+# Using merge='unique' for uns
+adata_combined = ad.concat([adata1, adata2], uns_merge='unique')
+# 'date' is kept (same value), 'batch' might be excluded (different values)
+```
+
+## Lazy Concatenation (AnnCollection)
+
+For very large datasets, use lazy concatenation that doesn't load all data:
+
+```python
+from anndata.experimental import AnnCollection
+
+# Create collection from file paths (doesn't load data)
+files = ['data1.h5ad', 'data2.h5ad', 'data3.h5ad']
+collection = AnnCollection(
+    files,
+    join_obs='outer',
+    join_vars='inner',
+    label='dataset',
+    keys=['dataset1', 'dataset2', 'dataset3']
+)
+
+# Access data lazily
+print(collection.n_obs)  # Total observations
+print(collection.obs.head())  # Metadata loaded, not X
+
+# Convert to regular AnnData when needed (loads all data)
+adata = collection.to_adata()
+```
+
+### Working with AnnCollection
+```python
+# Subset without loading data
+subset = collection[collection.obs['cell_type'] == 'T cell']
+
+# Iterate through datasets
+for adata in collection:
+    print(adata.shape)
+
+# Access specific dataset
+first_dataset = collection[0]
+```
+
+## Concatenation on Disk
+
+For datasets too large for memory, concatenate directly on disk:
+
+```python
+from anndata.experimental import concat_on_disk
+
+# Concatenate without loading into memory
+concat_on_disk(
+    ['data1.h5ad', 'data2.h5ad', 'data3.h5ad'],
+    'combined.h5ad',
+    join='outer'
+)
+
+# Load result in backed mode
+adata = ad.read_h5ad('combined.h5ad', backed='r')
+```
+
+## Common Concatenation Patterns
+
+### Combine technical replicates
+```python
+# Multiple runs of the same samples
+replicates = [adata_run1, adata_run2, adata_run3]
+adata_combined = ad.concat(
+    replicates,
+    label='technical_replicate',
+    keys=['rep1', 'rep2', 'rep3'],
+    join='inner'  # Keep only genes measured in all runs
+)
+```
+
+### Combine batches from experiment
+```python
+# Different experimental batches
+batches = [adata_batch1, adata_batch2, adata_batch3]
+adata_combined = ad.concat(
+    batches,
+    label='batch',
+    keys=['batch1', 'batch2', 'batch3'],
+    join='outer'  # Keep all genes
+)
+
+# Later: apply batch correction
+```
+
+### Merge multi-modal data
+```python
+# Different measurement modalities (e.g., RNA + protein)
+adata_rna = ad.AnnData(X=np.random.rand(100, 2000))
+adata_protein = ad.AnnData(X=np.random.rand(100, 50))
+
+# Concatenate along variables to combine modalities
+adata_multimodal = ad.concat([adata_rna, adata_protein], axis=1)
+
+# Add labels to distinguish modalities
+adata_multimodal.var['modality'] = ['RNA'] * 2000 + ['protein'] * 50
+```
+
+## Best Practices
+
+1. **Check compatibility before concatenating**
+```python
+# Verify shapes are compatible
+print([adata.n_vars for adata in [adata1, adata2, adata3]])
+
+# Check variable names match
+print([set(adata.var_names) for adata in [adata1, adata2, adata3]])
+```
+
+2. **Use appropriate join type**
+- `inner`: When you need the same features across all samples (most stringent)
+- `outer`: When you want to preserve all features (most inclusive)
+
+3. **Track data sources**
+Always use `label` and `keys` to track which observations came from which dataset.
+
+4. **Consider memory usage**
+- For large datasets, use `AnnCollection` or `concat_on_disk`
+- Consider backed mode for the result
+
+5. **Handle batch effects**
+Concatenation combines data but doesn't correct for batch effects. Apply batch correction after concatenation:
+```python
+# After concatenation, apply batch correction
+import scanpy as sc
+sc.pp.combat(adata_combined, key='batch')
+```
+
+6. **Validate results**
+```python
+# Check dimensions
+print(adata_combined.shape)
+
+# Check batch distribution
+print(adata_combined.obs['batch'].value_counts())
+
+# Verify metadata integrity
+print(adata_combined.var.head())
+print(adata_combined.obs.head())
+```

package/bin/skills/anndata/references/data_structure.md

@@ -0,0 +1,314 @@
+# AnnData Object Structure
+
+The AnnData object stores a data matrix with associated annotations, providing a flexible framework for managing experimental data and metadata.
+
+## Core Components
+
+### X (Data Matrix)
+The primary data matrix with shape (n_obs, n_vars) storing experimental measurements.
+
+```python
+import anndata as ad
+import numpy as np
+
+# Create with dense array
+adata = ad.AnnData(X=np.random.rand(100, 2000))
+
+# Create with sparse matrix (recommended for large, sparse data)
+from scipy.sparse import csr_matrix
+sparse_data = csr_matrix(np.random.rand(100, 2000))
+adata = ad.AnnData(X=sparse_data)
+```
+
+Access data:
+```python
+# Full matrix (caution with large datasets)
+full_data = adata.X
+
+# Single observation
+obs_data = adata.X[0, :]
+
+# Single variable across all observations
+var_data = adata.X[:, 0]
+```
+
+### obs (Observation Annotations)
+DataFrame storing metadata about observations (rows). Each row corresponds to one observation in X.
+
+```python
+import pandas as pd
+
+# Create AnnData with observation metadata
+obs_df = pd.DataFrame({
+    'cell_type': ['T cell', 'B cell', 'Monocyte'],
+    'treatment': ['control', 'treated', 'control'],
+    'timepoint': [0, 24, 24]
+}, index=['cell_1', 'cell_2', 'cell_3'])
+
+adata = ad.AnnData(X=np.random.rand(3, 100), obs=obs_df)
+
+# Access observation metadata
+print(adata.obs['cell_type'])
+print(adata.obs.loc['cell_1'])
+```
+
+### var (Variable Annotations)
+DataFrame storing metadata about variables (columns). Each row corresponds to one variable in X.
+
+```python
+# Create AnnData with variable metadata
+var_df = pd.DataFrame({
+    'gene_name': ['ACTB', 'GAPDH', 'TP53'],
+    'chromosome': ['7', '12', '17'],
+    'highly_variable': [True, False, True]
+}, index=['ENSG00001', 'ENSG00002', 'ENSG00003'])
+
+adata = ad.AnnData(X=np.random.rand(100, 3), var=var_df)
+
+# Access variable metadata
+print(adata.var['gene_name'])
+print(adata.var.loc['ENSG00001'])
+```
+
+### layers (Alternative Data Representations)
+Dictionary storing alternative matrices with the same dimensions as X.
+
+```python
+# Store raw counts, normalized data, and scaled data
+adata = ad.AnnData(X=np.random.rand(100, 2000))
+adata.layers['raw_counts'] = np.random.randint(0, 100, (100, 2000))
+adata.layers['normalized'] = adata.X / np.sum(adata.X, axis=1, keepdims=True)
+adata.layers['scaled'] = (adata.X - adata.X.mean()) / adata.X.std()
+
+# Access layers
+raw_data = adata.layers['raw_counts']
+normalized_data = adata.layers['normalized']
+```
+
+Common layer uses:
+- `raw_counts`: Original count data before normalization
+- `normalized`: Log-normalized or TPM values
+- `scaled`: Z-scored values for analysis
+- `imputed`: Data after imputation
+
+### obsm (Multi-dimensional Observation Annotations)
+Dictionary storing multi-dimensional arrays aligned to observations.
+
+```python
+# Store PCA coordinates and UMAP embeddings
+adata.obsm['X_pca'] = np.random.rand(100, 50)  # 50 principal components
+adata.obsm['X_umap'] = np.random.rand(100, 2)  # 2D UMAP coordinates
+adata.obsm['X_tsne'] = np.random.rand(100, 2)  # 2D t-SNE coordinates
+
+# Access embeddings
+pca_coords = adata.obsm['X_pca']
+umap_coords = adata.obsm['X_umap']
+```
+
+Common obsm uses:
+- `X_pca`: Principal component coordinates
+- `X_umap`: UMAP embedding coordinates
+- `X_tsne`: t-SNE embedding coordinates
+- `X_diffmap`: Diffusion map coordinates
+- `protein_expression`: Protein abundance measurements (CITE-seq)
+
+### varm (Multi-dimensional Variable Annotations)
+Dictionary storing multi-dimensional arrays aligned to variables.
+
+```python
+# Store PCA loadings
+adata.varm['PCs'] = np.random.rand(2000, 50)  # Loadings for 50 components
+adata.varm['gene_modules'] = np.random.rand(2000, 10)  # Gene module scores
+
+# Access loadings
+pc_loadings = adata.varm['PCs']
+```
+
+Common varm uses:
+- `PCs`: Principal component loadings
+- `gene_modules`: Gene co-expression module assignments
+
+### obsp (Pairwise Observation Relationships)
+Dictionary storing sparse matrices representing relationships between observations.
+
+```python
+from scipy.sparse import csr_matrix
+
+# Store k-nearest neighbor graph
+n_obs = 100
+knn_graph = csr_matrix(np.random.rand(n_obs, n_obs) > 0.95)
+adata.obsp['connectivities'] = knn_graph
+adata.obsp['distances'] = csr_matrix(np.random.rand(n_obs, n_obs))
+
+# Access graphs
+knn_connections = adata.obsp['connectivities']
+distances = adata.obsp['distances']
+```
+
+Common obsp uses:
+- `connectivities`: Cell-cell neighborhood graph
+- `distances`: Pairwise distances between cells
+
+### varp (Pairwise Variable Relationships)
+Dictionary storing sparse matrices representing relationships between variables.
+
+```python
+# Store gene-gene correlation matrix
+n_vars = 2000
+gene_corr = csr_matrix(np.random.rand(n_vars, n_vars) > 0.99)
+adata.varp['correlations'] = gene_corr
+
+# Access correlations
+gene_correlations = adata.varp['correlations']
+```
+
+### uns (Unstructured Annotations)
+Dictionary storing arbitrary unstructured metadata.
+
+```python
+# Store analysis parameters and results
+adata.uns['experiment_date'] = '2025-11-03'
+adata.uns['pca'] = {
+    'variance_ratio': [0.15, 0.10, 0.08],
+    'params': {'n_comps': 50}
+}
+adata.uns['neighbors'] = {
+    'params': {'n_neighbors': 15, 'method': 'umap'},
+    'connectivities_key': 'connectivities'
+}
+
+# Access unstructured data
+exp_date = adata.uns['experiment_date']
+pca_params = adata.uns['pca']['params']
+```
+
+Common uns uses:
+- Analysis parameters and settings
+- Color palettes for plotting
+- Cluster information
+- Tool-specific metadata
+
+### raw (Original Data Snapshot)
+Optional attribute preserving the original data matrix and variable annotations before filtering.
+
+```python
+# Create AnnData and store raw state
+adata = ad.AnnData(X=np.random.rand(100, 5000))
+adata.var['gene_name'] = [f'Gene_{i}' for i in range(5000)]
+
+# Store raw state before filtering
+adata.raw = adata.copy()
+
+# Filter to highly variable genes
+highly_variable_mask = np.random.rand(5000) > 0.5
+adata = adata[:, highly_variable_mask]
+
+# Access original data
+original_matrix = adata.raw.X
+original_var = adata.raw.var
+```
+
+## Object Properties
+
+```python
+# Dimensions
+n_observations = adata.n_obs
+n_variables = adata.n_vars
+shape = adata.shape  # (n_obs, n_vars)
+
+# Index information
+obs_names = adata.obs_names  # Observation identifiers
+var_names = adata.var_names  # Variable identifiers
+
+# Storage mode
+is_view = adata.is_view  # True if this is a view of another object
+is_backed = adata.isbacked  # True if backed by on-disk storage
+filename = adata.filename  # Path to backing file (if backed)
+```
+
+## Creating AnnData Objects
+
+### From arrays and DataFrames
+```python
+import anndata as ad
+import numpy as np
+import pandas as pd
+
+# Minimal creation
+X = np.random.rand(100, 2000)
+adata = ad.AnnData(X)
+
+# With metadata
+obs = pd.DataFrame({'cell_type': ['A', 'B'] * 50}, index=[f'cell_{i}' for i in range(100)])
+var = pd.DataFrame({'gene_name': [f'Gene_{i}' for i in range(2000)]}, index=[f'ENSG{i:05d}' for i in range(2000)])
+adata = ad.AnnData(X=X, obs=obs, var=var)
+
+# With all components
+adata = ad.AnnData(
+    X=X,
+    obs=obs,
+    var=var,
+    layers={'raw': np.random.randint(0, 100, (100, 2000))},
+    obsm={'X_pca': np.random.rand(100, 50)},
+    uns={'experiment': 'test'}
+)
+```
+
+### From DataFrame
+```python
+# Create from pandas DataFrame (genes as columns, cells as rows)
+df = pd.DataFrame(
+    np.random.rand(100, 50),
+    columns=[f'Gene_{i}' for i in range(50)],
+    index=[f'Cell_{i}' for i in range(100)]
+)
+adata = ad.AnnData(df)
+```
+
+## Data Access Patterns
+
+### Vector extraction
+```python
+# Get observation annotation as array
+cell_types = adata.obs_vector('cell_type')
+
+# Get variable values across observations
+gene_expression = adata.obs_vector('ACTB')  # If ACTB is in var_names
+
+# Get variable annotation as array
+gene_names = adata.var_vector('gene_name')
+```
+
+### Subsetting
+```python
+# By index
+subset = adata[0:10, 0:100]  # First 10 obs, first 100 vars
+
+# By name
+subset = adata[['cell_1', 'cell_2'], ['ACTB', 'GAPDH']]
+
+# By boolean mask
+high_count_cells = adata.obs['total_counts'] > 1000
+subset = adata[high_count_cells, :]
+
+# By observation metadata
+t_cells = adata[adata.obs['cell_type'] == 'T cell']
+```
+
+## Memory Considerations
+
+The AnnData structure is designed for memory efficiency:
+- Sparse matrices reduce memory for sparse data
+- Views avoid copying data when possible
+- Backed mode enables working with data larger than RAM
+- Categorical annotations reduce memory for discrete values
+
+```python
+# Convert strings to categoricals (more memory efficient)
+adata.obs['cell_type'] = adata.obs['cell_type'].astype('category')
+adata.strings_to_categoricals()
+
+# Check if object is a view (doesn't own data)
+if adata.is_view:
+    adata = adata.copy()  # Create independent copy
+```