@synsci/cli-darwin-x64-baseline 1.1.76 → 1.1.78

package/bin/skills/pytdc/scripts/benchmark_evaluation.py

@@ -0,0 +1,327 @@
+#!/usr/bin/env python3
+"""
+TDC Benchmark Group Evaluation Template
+
+This script demonstrates how to use TDC benchmark groups for systematic
+model evaluation following the required 5-seed protocol.
+
+Usage:
+    python benchmark_evaluation.py
+"""
+
+from tdc.benchmark_group import admet_group
+from tdc import Evaluator
+import numpy as np
+import pandas as pd
+
+
+def load_benchmark_group():
+    """
+    Load the ADMET benchmark group
+    """
+    print("=" * 60)
+    print("Loading ADMET Benchmark Group")
+    print("=" * 60)
+
+    # Initialize benchmark group
+    group = admet_group(path='data/')
+
+    # Get available benchmarks
+    print("\nAvailable benchmarks in ADMET group:")
+    benchmark_names = group.dataset_names
+    print(f"Total: {len(benchmark_names)} datasets")
+
+    for i, name in enumerate(benchmark_names[:10], 1):
+        print(f"  {i}. {name}")
+
+    if len(benchmark_names) > 10:
+        print(f"  ... and {len(benchmark_names) - 10} more")
+
+    return group
+
+
+def single_dataset_evaluation(group, dataset_name='Caco2_Wang'):
+    """
+    Example: Evaluate on a single dataset with 5-seed protocol
+    """
+    print("\n" + "=" * 60)
+    print(f"Example 1: Single Dataset Evaluation ({dataset_name})")
+    print("=" * 60)
+
+    # Get dataset benchmarks
+    benchmark = group.get(dataset_name)
+
+    print(f"\nBenchmark structure:")
+    print(f"  Seeds: {list(benchmark.keys())}")
+
+    # Required: Evaluate with 5 different seeds
+    predictions = {}
+
+    for seed in [1, 2, 3, 4, 5]:
+        print(f"\n--- Seed {seed} ---")
+
+        # Get train/valid data for this seed
+        train = benchmark[seed]['train']
+        valid = benchmark[seed]['valid']
+
+        print(f"Train size: {len(train)}")
+        print(f"Valid size: {len(valid)}")
+
+        # TODO: Replace with your model training
+        # model = YourModel()
+        # model.fit(train['Drug'], train['Y'])
+
+        # For demonstration, create dummy predictions
+        # Replace with: predictions[seed] = model.predict(benchmark[seed]['test'])
+        test = benchmark[seed]['test']
+        y_true = test['Y'].values
+
+        # Simulate predictions (add controlled noise)
+        np.random.seed(seed)
+        y_pred = y_true + np.random.normal(0, 0.3, len(y_true))
+
+        predictions[seed] = y_pred
+
+        # Evaluate this seed
+        evaluator = Evaluator(name='MAE')
+        score = evaluator(y_true, y_pred)
+        print(f"MAE for seed {seed}: {score:.4f}")
+
+    # Evaluate across all seeds
+    print("\n--- Overall Evaluation ---")
+    results = group.evaluate(predictions)
+
+    print(f"\nResults for {dataset_name}:")
+    mean_score, std_score = results[dataset_name]
+    print(f"  Mean MAE: {mean_score:.4f}")
+    print(f"  Std MAE: {std_score:.4f}")
+
+    return predictions, results
+
+
+def multiple_datasets_evaluation(group):
+    """
+    Example: Evaluate on multiple datasets
+    """
+    print("\n" + "=" * 60)
+    print("Example 2: Multiple Datasets Evaluation")
+    print("=" * 60)
+
+    # Select a subset of datasets for demonstration
+    selected_datasets = ['Caco2_Wang', 'HIA_Hou', 'Bioavailability_Ma']
+
+    all_predictions = {}
+    all_results = {}
+
+    for dataset_name in selected_datasets:
+        print(f"\n{'='*40}")
+        print(f"Evaluating: {dataset_name}")
+        print(f"{'='*40}")
+
+        benchmark = group.get(dataset_name)
+        predictions = {}
+
+        # Train and predict for each seed
+        for seed in [1, 2, 3, 4, 5]:
+            train = benchmark[seed]['train']
+            test = benchmark[seed]['test']
+
+            # TODO: Replace with your model
+            # model = YourModel()
+            # model.fit(train['Drug'], train['Y'])
+            # predictions[seed] = model.predict(test['Drug'])
+
+            # Dummy predictions for demonstration
+            np.random.seed(seed)
+            y_true = test['Y'].values
+            y_pred = y_true + np.random.normal(0, 0.3, len(y_true))
+            predictions[seed] = y_pred
+
+        all_predictions[dataset_name] = predictions
+
+        # Evaluate this dataset
+        results = group.evaluate({dataset_name: predictions})
+        all_results[dataset_name] = results[dataset_name]
+
+        mean_score, std_score = results[dataset_name]
+        print(f"  {dataset_name}: {mean_score:.4f} ± {std_score:.4f}")
+
+    # Summary
+    print("\n" + "=" * 60)
+    print("Summary of Results")
+    print("=" * 60)
+
+    results_df = pd.DataFrame([
+        {
+            'Dataset': name,
+            'Mean MAE': f"{mean:.4f}",
+            'Std MAE': f"{std:.4f}"
+        }
+        for name, (mean, std) in all_results.items()
+    ])
+
+    print(results_df.to_string(index=False))
+
+    return all_predictions, all_results
+
+
+def custom_model_template():
+    """
+    Template for integrating your own model with TDC benchmarks
+    """
+    print("\n" + "=" * 60)
+    print("Example 3: Custom Model Template")
+    print("=" * 60)
+
+    code_template = '''
+# Template for using your own model with TDC benchmarks
+
+from tdc.benchmark_group import admet_group
+from your_library import YourModel  # Replace with your model
+
+# Initialize benchmark group
+group = admet_group(path='data/')
+benchmark = group.get('Caco2_Wang')
+
+predictions = {}
+
+for seed in [1, 2, 3, 4, 5]:
+    # Get data for this seed
+    train = benchmark[seed]['train']
+    valid = benchmark[seed]['valid']
+    test = benchmark[seed]['test']
+
+    # Extract features and labels
+    X_train, y_train = train['Drug'], train['Y']
+    X_valid, y_valid = valid['Drug'], valid['Y']
+    X_test = test['Drug']
+
+    # Initialize and train model
+    model = YourModel(random_state=seed)
+    model.fit(X_train, y_train)
+
+    # Optionally use validation set for early stopping
+    # model.fit(X_train, y_train, validation_data=(X_valid, y_valid))
+
+    # Make predictions on test set
+    predictions[seed] = model.predict(X_test)
+
+# Evaluate with TDC
+results = group.evaluate(predictions)
+print(f"Results: {results}")
+'''
+
+    print("\nCustom Model Integration Template:")
+    print("=" * 60)
+    print(code_template)
+
+    return code_template
+
+
+def multi_seed_statistics(predictions_dict):
+    """
+    Example: Analyzing multi-seed prediction statistics
+    """
+    print("\n" + "=" * 60)
+    print("Example 4: Multi-Seed Statistics Analysis")
+    print("=" * 60)
+
+    # Analyze prediction variability across seeds
+    all_preds = np.array([predictions_dict[seed] for seed in [1, 2, 3, 4, 5]])
+
+    print("\nPrediction statistics across 5 seeds:")
+    print(f"  Shape: {all_preds.shape}")
+    print(f"  Mean prediction: {all_preds.mean():.4f}")
+    print(f"  Std across seeds: {all_preds.std(axis=0).mean():.4f}")
+    print(f"  Min prediction: {all_preds.min():.4f}")
+    print(f"  Max prediction: {all_preds.max():.4f}")
+
+    # Per-sample variance
+    per_sample_std = all_preds.std(axis=0)
+    print(f"\nPer-sample prediction std:")
+    print(f"  Mean: {per_sample_std.mean():.4f}")
+    print(f"  Median: {np.median(per_sample_std):.4f}")
+    print(f"  Max: {per_sample_std.max():.4f}")
+
+
+def leaderboard_submission_guide():
+    """
+    Guide for submitting to TDC leaderboards
+    """
+    print("\n" + "=" * 60)
+    print("Example 5: Leaderboard Submission Guide")
+    print("=" * 60)
+
+    guide = """
+To submit results to TDC leaderboards:
+
+1. Evaluate your model following the 5-seed protocol:
+   - Use seeds [1, 2, 3, 4, 5] exactly as provided
+   - Do not modify the train/valid/test splits
+   - Report mean ± std across all 5 seeds
+
+2. Format your results:
+   results = group.evaluate(predictions)
+   # Returns: {'dataset_name': [mean_score, std_score]}
+
+3. Submit to leaderboard:
+   - Visit: https://tdcommons.ai/benchmark/admet_group/
+   - Click on your dataset of interest
+   - Submit your results with:
+     * Model name and description
+     * Mean score ± standard deviation
+     * Reference to paper/code (if available)
+
+4. Best practices:
+   - Report all datasets in the benchmark group
+   - Include model hyperparameters
+   - Share code for reproducibility
+   - Compare against baseline models
+
+5. Evaluation metrics:
+   - ADMET Group uses MAE by default
+   - Other groups may use different metrics
+   - Check benchmark-specific requirements
+"""
+
+    print(guide)
+
+
+def main():
+    """
+    Main function to run all benchmark evaluation examples
+    """
+    print("\n" + "=" * 60)
+    print("TDC Benchmark Group Evaluation Examples")
+    print("=" * 60)
+
+    # Load benchmark group
+    group = load_benchmark_group()
+
+    # Example 1: Single dataset evaluation
+    predictions, results = single_dataset_evaluation(group)
+
+    # Example 2: Multiple datasets evaluation
+    all_predictions, all_results = multiple_datasets_evaluation(group)
+
+    # Example 3: Custom model template
+    custom_model_template()
+
+    # Example 4: Multi-seed statistics
+    multi_seed_statistics(predictions)
+
+    # Example 5: Leaderboard submission guide
+    leaderboard_submission_guide()
+
+    print("\n" + "=" * 60)
+    print("Benchmark evaluation examples completed!")
+    print("=" * 60)
+    print("\nNext steps:")
+    print("1. Replace dummy predictions with your model")
+    print("2. Run full evaluation on all benchmark datasets")
+    print("3. Submit results to TDC leaderboard")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

package/bin/skills/pytdc/scripts/load_and_split_data.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+"""
+TDC Data Loading and Splitting Template
+
+This script demonstrates how to load TDC datasets and apply different
+splitting strategies for model training and evaluation.
+
+Usage:
+    python load_and_split_data.py
+"""
+
+from tdc.single_pred import ADME
+from tdc.multi_pred import DTI
+from tdc import Evaluator
+import pandas as pd
+
+
+def load_single_pred_example():
+    """
+    Example: Loading and splitting a single-prediction dataset (ADME)
+    """
+    print("=" * 60)
+    print("Example 1: Single-Prediction Task (ADME)")
+    print("=" * 60)
+
+    # Load Caco2 dataset (intestinal permeability)
+    print("\nLoading Caco2_Wang dataset...")
+    data = ADME(name='Caco2_Wang')
+
+    # Get basic dataset info
+    print(f"\nDataset size: {len(data.get_data())} molecules")
+    data.print_stats()
+
+    # Method 1: Scaffold split (default, recommended)
+    print("\n--- Scaffold Split ---")
+    split = data.get_split(method='scaffold', seed=42, frac=[0.7, 0.1, 0.2])
+
+    train = split['train']
+    valid = split['valid']
+    test = split['test']
+
+    print(f"Train: {len(train)} molecules")
+    print(f"Valid: {len(valid)} molecules")
+    print(f"Test: {len(test)} molecules")
+
+    # Display sample data
+    print("\nSample training data:")
+    print(train.head(3))
+
+    # Method 2: Random split
+    print("\n--- Random Split ---")
+    split_random = data.get_split(method='random', seed=42, frac=[0.8, 0.1, 0.1])
+    print(f"Train: {len(split_random['train'])} molecules")
+    print(f"Valid: {len(split_random['valid'])} molecules")
+    print(f"Test: {len(split_random['test'])} molecules")
+
+    return split
+
+
+def load_multi_pred_example():
+    """
+    Example: Loading and splitting a multi-prediction dataset (DTI)
+    """
+    print("\n" + "=" * 60)
+    print("Example 2: Multi-Prediction Task (DTI)")
+    print("=" * 60)
+
+    # Load BindingDB Kd dataset (drug-target interactions)
+    print("\nLoading BindingDB_Kd dataset...")
+    data = DTI(name='BindingDB_Kd')
+
+    # Get basic dataset info
+    full_data = data.get_data()
+    print(f"\nDataset size: {len(full_data)} drug-target pairs")
+    print(f"Unique drugs: {full_data['Drug_ID'].nunique()}")
+    print(f"Unique targets: {full_data['Target_ID'].nunique()}")
+
+    # Method 1: Random split
+    print("\n--- Random Split ---")
+    split_random = data.get_split(method='random', seed=42)
+    print(f"Train: {len(split_random['train'])} pairs")
+    print(f"Valid: {len(split_random['valid'])} pairs")
+    print(f"Test: {len(split_random['test'])} pairs")
+
+    # Method 2: Cold drug split (unseen drugs in test)
+    print("\n--- Cold Drug Split ---")
+    split_cold_drug = data.get_split(method='cold_drug', seed=42)
+
+    train = split_cold_drug['train']
+    test = split_cold_drug['test']
+
+    # Verify no drug overlap
+    train_drugs = set(train['Drug_ID'])
+    test_drugs = set(test['Drug_ID'])
+    overlap = train_drugs & test_drugs
+
+    print(f"Train: {len(train)} pairs, {len(train_drugs)} unique drugs")
+    print(f"Test: {len(test)} pairs, {len(test_drugs)} unique drugs")
+    print(f"Drug overlap: {len(overlap)} (should be 0)")
+
+    # Method 3: Cold target split (unseen targets in test)
+    print("\n--- Cold Target Split ---")
+    split_cold_target = data.get_split(method='cold_target', seed=42)
+
+    train = split_cold_target['train']
+    test = split_cold_target['test']
+
+    train_targets = set(train['Target_ID'])
+    test_targets = set(test['Target_ID'])
+    overlap = train_targets & test_targets
+
+    print(f"Train: {len(train)} pairs, {len(train_targets)} unique targets")
+    print(f"Test: {len(test)} pairs, {len(test_targets)} unique targets")
+    print(f"Target overlap: {len(overlap)} (should be 0)")
+
+    # Display sample data
+    print("\nSample DTI data:")
+    print(full_data.head(3))
+
+    return split_cold_drug
+
+
+def evaluation_example(split):
+    """
+    Example: Evaluating model predictions with TDC evaluators
+    """
+    print("\n" + "=" * 60)
+    print("Example 3: Model Evaluation")
+    print("=" * 60)
+
+    test = split['test']
+
+    # For demonstration, create dummy predictions
+    # In practice, replace with your model's predictions
+    import numpy as np
+    np.random.seed(42)
+
+    # Simulate predictions (replace with model.predict(test['Drug']))
+    y_true = test['Y'].values
+    y_pred = y_true + np.random.normal(0, 0.5, len(y_true))  # Add noise
+
+    # Evaluate with different metrics
+    print("\nEvaluating predictions...")
+
+    # Regression metrics
+    mae_evaluator = Evaluator(name='MAE')
+    mae = mae_evaluator(y_true, y_pred)
+    print(f"MAE: {mae:.4f}")
+
+    rmse_evaluator = Evaluator(name='RMSE')
+    rmse = rmse_evaluator(y_true, y_pred)
+    print(f"RMSE: {rmse:.4f}")
+
+    r2_evaluator = Evaluator(name='R2')
+    r2 = r2_evaluator(y_true, y_pred)
+    print(f"R²: {r2:.4f}")
+
+    spearman_evaluator = Evaluator(name='Spearman')
+    spearman = spearman_evaluator(y_true, y_pred)
+    print(f"Spearman: {spearman:.4f}")
+
+
+def custom_split_example():
+    """
+    Example: Creating custom splits with different fractions
+    """
+    print("\n" + "=" * 60)
+    print("Example 4: Custom Split Fractions")
+    print("=" * 60)
+
+    data = ADME(name='HIA_Hou')
+
+    # Custom split fractions
+    custom_fracs = [
+        ([0.6, 0.2, 0.2], "60/20/20 split"),
+        ([0.8, 0.1, 0.1], "80/10/10 split"),
+        ([0.7, 0.15, 0.15], "70/15/15 split")
+    ]
+
+    for frac, description in custom_fracs:
+        split = data.get_split(method='scaffold', seed=42, frac=frac)
+        print(f"\n{description}:")
+        print(f"  Train: {len(split['train'])} ({frac[0]*100:.0f}%)")
+        print(f"  Valid: {len(split['valid'])} ({frac[1]*100:.0f}%)")
+        print(f"  Test: {len(split['test'])} ({frac[2]*100:.0f}%)")
+
+
+def main():
+    """
+    Main function to run all examples
+    """
+    print("\n" + "=" * 60)
+    print("TDC Data Loading and Splitting Examples")
+    print("=" * 60)
+
+    # Example 1: Single prediction with scaffold split
+    split = load_single_pred_example()
+
+    # Example 2: Multi prediction with cold splits
+    dti_split = load_multi_pred_example()
+
+    # Example 3: Model evaluation
+    evaluation_example(split)
+
+    # Example 4: Custom split fractions
+    custom_split_example()
+
+    print("\n" + "=" * 60)
+    print("Examples completed!")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()