clusterkit 0.1.0.pre.1

data/examples/pca_example.rb

@@ -0,0 +1,114 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'clusterkit'
+
+puts "PCA Example - Dimensionality Reduction and Variance Analysis"
+puts "=" * 60
+
+# Generate sample data with clear structure
+# High variance in first 2 dimensions, low variance in others
+def generate_structured_data(n_samples: 100, n_features: 20)
+  data = []
+  
+  n_samples.times do
+    point = []
+    
+    # First dimension: high variance (range ~10)
+    point << rand * 10
+    
+    # Second dimension: medium variance (range ~5)
+    point << rand * 5
+    
+    # Third dimension: some variance (range ~2)
+    point << rand * 2
+    
+    # Remaining dimensions: very low variance (noise)
+    (n_features - 3).times do
+      point << rand * 0.1
+    end
+    
+    data << point
+  end
+  
+  data
+end
+
+# Generate data
+data = generate_structured_data(n_samples: 100, n_features: 20)
+puts "\nGenerated #{data.size} samples with #{data.first.size} features"
+
+# Perform PCA with different numbers of components
+[2, 3, 5, 10].each do |n_components|
+  puts "\n" + "-" * 40
+  puts "PCA with #{n_components} components:"
+  
+  pca = ClusterKit::PCA.new(n_components: n_components)
+  transformed = pca.fit_transform(data)
+  
+  puts "  Transformed shape: #{transformed.size} x #{transformed.first.size}"
+  
+  # Show explained variance for each component
+  puts "  Explained variance ratio:"
+  pca.explained_variance_ratio.each_with_index do |ratio, i|
+    puts "    PC#{i+1}: #{(ratio * 100).round(2)}%"
+  end
+  
+  # Show cumulative explained variance
+  cumulative = pca.cumulative_explained_variance_ratio[-1]
+  puts "  Total variance explained: #{(cumulative * 100).round(2)}%"
+end
+
+# Demonstrate reconstruction
+puts "\n" + "=" * 60
+puts "Reconstruction Example:"
+puts "-" * 40
+
+# Use 2 components (should capture most variance)
+pca_2 = ClusterKit::PCA.new(n_components: 2)
+compressed = pca_2.fit_transform(data)
+reconstructed = pca_2.inverse_transform(compressed)
+
+# Calculate reconstruction error
+sample_idx = 0
+original = data[sample_idx]
+recon = reconstructed[sample_idx]
+
+puts "\nOriginal data point (first 5 features):"
+puts "  #{original[0..4].map { |v| v.round(3) }.join(', ')}"
+
+puts "\nReconstructed from 2 components (first 5 features):"
+puts "  #{recon[0..4].map { |v| v.round(3) }.join(', ')}"
+
+# Calculate mean squared error
+mse = original.zip(recon).map { |o, r| (o - r) ** 2 }.sum / original.size
+puts "\nReconstruction MSE: #{mse.round(4)}"
+
+# Demonstrate data compression ratio
+original_size = data.size * data.first.size
+compressed_size = compressed.size * compressed.first.size
+compression_ratio = (1 - compressed_size.to_f / original_size) * 100
+
+puts "\nData Compression:"
+puts "  Original size: #{original_size} values"
+puts "  Compressed size: #{compressed_size} values"
+puts "  Compression ratio: #{compression_ratio.round(1)}%"
+puts "  Variance retained: #{(pca_2.cumulative_explained_variance_ratio[-1] * 100).round(1)}%"
+
+# Compare with SVD
+puts "\n" + "=" * 60
+puts "PCA vs SVD Comparison:"
+puts "-" * 40
+
+# PCA (with mean centering)
+pca = ClusterKit::PCA.new(n_components: 2)
+pca_result = pca.fit_transform(data)
+
+# SVD (without mean centering)
+u, s, vt = ClusterKit.svd(data, 2)
+svd_result = u
+
+puts "PCA result (first point): #{pca_result[0].map { |v| v.round(3) }}"
+puts "SVD result (first point): #{svd_result[0].map { |v| v.round(3) }}"
+puts "\nNote: PCA centers the data (subtracts mean), SVD does not."
+puts "This makes PCA better for finding principal components of variation."

data/examples/reproducible_umap.rb

@@ -0,0 +1,99 @@
+#!/usr/bin/env ruby
+# Example: Achieving reproducibility with UMAP despite random seed issues
+
+require_relative '../lib/clusterkit'
+require 'json'
+
+# Due to upstream limitations, UMAP doesn't give perfectly reproducible results
+# even with a fixed random_seed. Here are workarounds:
+
+# Generate sample data
+srand(42)
+data = []
+3.times do |cluster|
+  center = Array.new(50) { rand * 0.1 + cluster * 2.0 }
+  30.times do
+    point = center.map { |c| c + (rand - 0.5) * 0.3 }
+    data << point
+  end
+end
+
+puts "Workaround 1: Cache transformed results"
+puts "=" * 60
+
+# First run: transform and save results
+cache_file = "umap_results_cache.json"
+if File.exist?(cache_file)
+  puts "Loading cached results from #{cache_file}"
+  embedded = JSON.parse(File.read(cache_file))
+else
+  puts "No cache found, running UMAP..."
+  umap = ClusterKit::Dimensionality::UMAP.new(
+    n_components: 2,
+    n_neighbors: 5,
+    random_seed: 42  # Still use for *some* consistency
+  )
+  embedded = umap.fit_transform(data)
+  
+  # Save results for reproducibility
+  File.write(cache_file, JSON.pretty_generate(embedded))
+  puts "Results cached to #{cache_file}"
+end
+
+puts "First 3 points:"
+embedded[0..2].each_with_index do |point, i|
+  puts "  Point #{i}: [#{point[0].round(3)}, #{point[1].round(3)}]"
+end
+
+puts "\nWorkaround 2: Save and load fitted models"
+puts "=" * 60
+
+model_file = "umap_model.bin"
+
+# Train and save model once
+if File.exist?(model_file)
+  puts "Loading existing model from #{model_file}"
+  umap = ClusterKit::Dimensionality::UMAP.load(model_file)
+else
+  puts "Training new model..."
+  umap = ClusterKit::Dimensionality::UMAP.new(
+    n_components: 2,
+    n_neighbors: 5,
+    random_seed: 42
+  )
+  umap.fit(data)
+  umap.save(model_file)
+  puts "Model saved to #{model_file}"
+end
+
+# Now transform new data with the same model
+new_data = data[0..9]  # Take first 10 points as "new" data
+transformed = umap.transform(new_data)
+puts "Transformed 10 new points with saved model"
+puts "First 3 transformed points:"
+transformed[0..2].each_with_index do |point, i|
+  puts "  Point #{i}: [#{point[0].round(3)}, #{point[1].round(3)}]"
+end
+
+puts "\nWorkaround 3: Use PCA for deterministic reduction"
+puts "=" * 60
+
+# PCA is deterministic - same input always gives same output
+pca = ClusterKit::Dimensionality::PCA.new(n_components: 2)
+pca_result1 = pca.fit_transform(data)
+pca_result2 = pca.fit_transform(data)  # Do it again
+
+puts "PCA results are identical: #{pca_result1[0] == pca_result2[0]}"
+puts "First point from run 1: [#{pca_result1[0][0].round(3)}, #{pca_result1[0][1].round(3)}]"
+puts "First point from run 2: [#{pca_result2[0][0].round(3)}, #{pca_result2[0][1].round(3)}]"
+
+puts "\nRecommendations:"
+puts "-" * 40
+puts "1. For production pipelines, cache UMAP results"
+puts "2. For model deployment, save fitted models and reuse them"
+puts "3. For testing/CI, use PCA or cached test data"
+puts "4. Accept small variations in UMAP results as normal"
+
+# Clean up example files (uncomment to remove)
+# File.delete(cache_file) if File.exist?(cache_file)
+# File.delete(model_file) if File.exist?(model_file)

data/examples/verbose_control.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+# Example demonstrating how to control verbose output from clusterkit
+
+require 'bundler/setup'
+require 'clusterkit'
+
+# Generate some random test data
+data = Array.new(50) { Array.new(20) { rand } }
+
+puts "=" * 60
+puts "clusterkit Verbose Output Control Demo"
+puts "=" * 60
+
+puts "\n1. Default behavior (quiet mode):"
+puts "-" * 40
+umap1 = ClusterKit::UMAP.new(n_components: 2, n_neighbors: 10)
+result1 = umap1.fit_transform(data)
+puts "✓ UMAP completed silently"
+puts "  Result shape: #{result1.length} x #{result1.first.length}"
+
+puts "\n2. Enable verbose output:"
+puts "-" * 40
+ClusterKit.configure do |config|
+  config.verbose = true
+end
+
+umap2 = ClusterKit::UMAP.new(n_components: 2, n_neighbors: 10)
+puts "Running UMAP with verbose output enabled..."
+result2 = umap2.fit_transform(data)
+puts "✓ UMAP completed with debug output"
+
+puts "\n3. Back to quiet mode:"
+puts "-" * 40
+ClusterKit.configuration.verbose = false
+
+umap3 = ClusterKit::UMAP.new(n_components: 2, n_neighbors: 10)
+result3 = umap3.fit_transform(data)
+puts "✓ UMAP completed silently again"
+
+puts "\n" + "=" * 60
+puts "You can also set verbose mode via environment variable:"
+puts "  ANNEMBED_VERBOSE=true ruby your_script.rb"
+puts "=" * 60

data/ext/clusterkit/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "clusterkit"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+magnus = { version = "0.6", features = ["embed"] }
+annembed = { git = "https://github.com/cpetersen/annembed", tag = "clusterkit-0.1.0" }
+hnsw_rs = { git = "https://github.com/cpetersen/hnswlib-rs", tag = "clusterkit-0.1.0" }
+hdbscan = "0.11"
+ndarray = "0.16"
+num-traits = "0.2"
+rayon = "1.7"
+serde = { version = "1.0", features = ["derive"] }
+bincode = "1.3"
+rand = "0.8"
+
+[features]
+default = ["openblas-static"]
+openblas-static = ["annembed/openblas-static"]
+openblas-system = ["annembed/openblas-system"]
+intel-mkl-static = ["annembed/intel-mkl-static"]

data/ext/clusterkit/extconf.rb

@@ -0,0 +1,4 @@
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("clusterkit/clusterkit")

data/ext/clusterkit/src/clustering/hdbscan_wrapper.rs

@@ -0,0 +1,115 @@
+use magnus::{function, prelude::*, Error, Value, RArray, RHash, Integer, TryConvert};
+use hdbscan::{Hdbscan, HdbscanHyperParams};
+
+/// Perform HDBSCAN clustering
+/// Returns a hash with labels and basic statistics
+pub fn hdbscan_fit(
+    data: Value,
+    min_samples: usize,
+    min_cluster_size: usize,
+    metric: String,
+) -> Result<RHash, Error> {
+    // Convert Ruby array to ndarray
+    let rarray: RArray = TryConvert::try_convert(data)?;
+    let n_samples = rarray.len();
+    
+    if n_samples == 0 {
+        return Err(Error::new(
+            magnus::exception::arg_error(),
+            "Data cannot be empty",
+        ));
+    }
+    
+    // Get dimensions
+    let first_row: RArray = rarray.entry::<RArray>(0)?;
+    let n_features = first_row.len();
+    
+    // Convert to Vec<Vec<f64>> format expected by hdbscan crate
+    let mut data_vec: Vec<Vec<f64>> = Vec::with_capacity(n_samples);
+    for i in 0..n_samples {
+        let row: RArray = rarray.entry(i as isize)?;
+        let mut row_vec: Vec<f64> = Vec::with_capacity(n_features);
+        for j in 0..n_features {
+            let val: f64 = row.entry(j as isize)?;
+            row_vec.push(val);
+        }
+        data_vec.push(row_vec);
+    }
+    
+    // Note: hdbscan crate doesn't support custom metrics directly
+    // We'll use the default Euclidean distance for now
+    if metric != "euclidean" && metric != "l2" {
+        eprintln!("Warning: Current hdbscan version only supports Euclidean distance. Using Euclidean.");
+    }
+    
+    // Adjust parameters to avoid index out of bounds errors
+    // The hdbscan crate has issues when min_samples >= n_samples
+    let adjusted_min_samples = min_samples.min(n_samples.saturating_sub(1)).max(1);
+    let adjusted_min_cluster_size = min_cluster_size.min(n_samples).max(2);
+    
+    // Create hyperparameters
+    let hyper_params = HdbscanHyperParams::builder()
+        .min_cluster_size(adjusted_min_cluster_size)
+        .min_samples(adjusted_min_samples)
+        .build();
+    
+    // Create HDBSCAN instance and run clustering
+    let clusterer = Hdbscan::new(&data_vec, hyper_params);
+    
+    // Run the clustering algorithm - cluster() returns Result<Vec<i32>, HdbscanError>
+    let labels = clusterer.cluster().map_err(|e| {
+        Error::new(
+            magnus::exception::runtime_error(),
+            format!("HDBSCAN clustering failed: {:?}", e)
+        )
+    })?;
+    
+    // Convert results to Ruby types
+    let ruby = magnus::Ruby::get().unwrap();
+    let result = RHash::new();
+    
+    // Convert labels (i32 to Ruby Integer, -1 for noise)
+    let labels_array = RArray::new();
+    for &label in labels.iter() {
+        labels_array.push(Integer::from_value(
+            ruby.eval(&format!("{}", label)).unwrap()
+        ).unwrap())?;
+    }
+    result.aset("labels", labels_array)?;
+    
+    // For now, we'll create dummy probabilities and outlier scores
+    // since the basic hdbscan crate doesn't provide these
+    // In the future, we could calculate these ourselves or use a more advanced implementation
+    
+    // Create probabilities array (all 1.0 for clustered points, 0.0 for noise)
+    let probs_array = RArray::new();
+    for &label in labels.iter() {
+        let prob = if label == -1 { 0.0 } else { 1.0 };
+        probs_array.push(prob)?;
+    }
+    result.aset("probabilities", probs_array)?;
+    
+    // Create outlier scores array (0.0 for clustered points, 1.0 for noise)
+    let outlier_array = RArray::new();
+    for &label in labels.iter() {
+        let score = if label == -1 { 1.0 } else { 0.0 };
+        outlier_array.push(score)?;
+    }
+    result.aset("outlier_scores", outlier_array)?;
+    
+    // Create empty cluster persistence hash for now
+    let persistence_hash = RHash::new();
+    result.aset("cluster_persistence", persistence_hash)?;
+    
+    Ok(result)
+}
+
+/// Initialize HDBSCAN module functions
+pub fn init(clustering_module: &magnus::RModule) -> Result<(), Error> {
+    clustering_module.define_singleton_method(
+        "hdbscan_rust",
+        function!(hdbscan_fit, 4),
+    )?;
+    
+    Ok(())
+}

data/ext/clusterkit/src/clustering.rs

@@ -0,0 +1,267 @@
+use magnus::{function, prelude::*, Error, Value, RArray, Integer, TryConvert};
+use ndarray::{Array1, Array2, ArrayView1, Axis};
+use rand::prelude::*;
+
+mod hdbscan_wrapper;
+
+pub fn init(parent: &magnus::RModule) -> Result<(), Error> {
+    let clustering_module = parent.define_module("Clustering")?;
+    
+    clustering_module.define_singleton_method(
+        "kmeans_rust",
+        function!(kmeans, 3),
+    )?;
+    
+    clustering_module.define_singleton_method(
+        "kmeans_predict_rust",
+        function!(kmeans_predict, 2),
+    )?;
+    
+    // Initialize HDBSCAN functions
+    hdbscan_wrapper::init(&clustering_module)?;
+    
+    Ok(())
+}
+
+/// Perform K-means clustering
+/// Returns (labels, centroids, inertia)
+fn kmeans(data: Value, k: usize, max_iter: usize) -> Result<(RArray, RArray, f64), Error> {
+    // Convert Ruby array to ndarray
+    let rarray: RArray = TryConvert::try_convert(data)?;
+    let n_samples = rarray.len();
+    
+    if n_samples == 0 {
+        return Err(Error::new(
+            magnus::exception::arg_error(),
+            "Data cannot be empty",
+        ));
+    }
+    
+    // Get dimensions
+    let first_row: RArray = rarray.entry::<RArray>(0)?;
+    let n_features = first_row.len();
+    
+    if k > n_samples {
+        return Err(Error::new(
+            magnus::exception::arg_error(),
+            format!("k ({}) cannot be larger than number of samples ({})", k, n_samples),
+        ));
+    }
+    
+    // Convert to ndarray
+    let mut data_array = Array2::<f64>::zeros((n_samples, n_features));
+    for i in 0..n_samples {
+        let row: RArray = rarray.entry(i as isize)?;
+        for j in 0..n_features {
+            let val: f64 = row.entry(j as isize)?;
+            data_array[[i, j]] = val;
+        }
+    }
+    
+    // Initialize centroids using K-means++
+    let mut centroids = kmeans_plusplus(&data_array, k)?;
+    let mut labels = vec![0usize; n_samples];
+    let mut prev_labels = vec![0usize; n_samples];
+    
+    // K-means iterations
+    for iteration in 0..max_iter {
+        // Assign points to nearest centroid
+        let mut changed = false;
+        for i in 0..n_samples {
+            let point = data_array.row(i);
+            let mut min_dist = f64::INFINITY;
+            let mut best_cluster = 0;
+            
+            for (j, centroid) in centroids.axis_iter(Axis(0)).enumerate() {
+                let dist = euclidean_distance(&point, &centroid);
+                if dist < min_dist {
+                    min_dist = dist;
+                    best_cluster = j;
+                }
+            }
+            
+            if labels[i] != best_cluster {
+                changed = true;
+            }
+            labels[i] = best_cluster;
+        }
+        
+        // Check for convergence
+        if !changed && iteration > 0 {
+            break;
+        }
+        
+        // Update centroids
+        for j in 0..k {
+            let mut sum = Array1::<f64>::zeros(n_features);
+            let mut count = 0;
+            
+            for i in 0..n_samples {
+                if labels[i] == j {
+                    sum += &data_array.row(i);
+                    count += 1;
+                }
+            }
+            
+            if count > 0 {
+                centroids.row_mut(j).assign(&(sum / count as f64));
+            }
+        }
+        
+        prev_labels.clone_from(&labels);
+    }
+    
+    // Calculate inertia (sum of squared distances to nearest centroid)
+    let mut inertia = 0.0;
+    for i in 0..n_samples {
+        let point = data_array.row(i);
+        let centroid = centroids.row(labels[i]);
+        inertia += euclidean_distance(&point, &centroid).powi(2);
+    }
+    
+    // Convert results to Ruby arrays
+    let ruby = magnus::Ruby::get().unwrap();
+    let labels_array = RArray::new();
+    for label in labels {
+        labels_array.push(Integer::from_value(ruby.eval(&format!("{}", label)).unwrap()).unwrap())?;
+    }
+    
+    let centroids_array = RArray::new();
+    for i in 0..k {
+        let row_array = RArray::new();
+        for j in 0..n_features {
+            row_array.push(centroids[[i, j]])?;
+        }
+        centroids_array.push(row_array)?;
+    }
+    
+    Ok((labels_array, centroids_array, inertia))
+}
+
+/// Predict cluster labels for new data given centroids
+fn kmeans_predict(data: Value, centroids: Value) -> Result<RArray, Error> {
+    // Convert inputs
+    let data_array: RArray = TryConvert::try_convert(data)?;
+    let centroids_array: RArray = TryConvert::try_convert(centroids)?;
+    
+    let n_samples = data_array.len();
+    let k = centroids_array.len();
+    
+    if n_samples == 0 {
+        return Err(Error::new(
+            magnus::exception::arg_error(),
+            "Data cannot be empty",
+        ));
+    }
+    
+    // Get dimensions
+    let first_row: RArray = data_array.entry::<RArray>(0)?;
+    let n_features = first_row.len();
+    
+    // Convert data to ndarray
+    let mut data_matrix = Array2::<f64>::zeros((n_samples, n_features));
+    for i in 0..n_samples {
+        let row: RArray = data_array.entry(i as isize)?;
+        for j in 0..n_features {
+            let val: f64 = row.entry(j as isize)?;
+            data_matrix[[i, j]] = val;
+        }
+    }
+    
+    // Convert centroids to ndarray
+    let mut centroids_matrix = Array2::<f64>::zeros((k, n_features));
+    for i in 0..k {
+        let row: RArray = centroids_array.entry(i as isize)?;
+        for j in 0..n_features {
+            let val: f64 = row.entry(j as isize)?;
+            centroids_matrix[[i, j]] = val;
+        }
+    }
+    
+    // Predict labels
+    let ruby = magnus::Ruby::get().unwrap();
+    let labels_array = RArray::new();
+    
+    for i in 0..n_samples {
+        let point = data_matrix.row(i);
+        let mut min_dist = f64::INFINITY;
+        let mut best_cluster = 0;
+        
+        for (j, centroid) in centroids_matrix.axis_iter(Axis(0)).enumerate() {
+            let dist = euclidean_distance(&point, &centroid);
+            if dist < min_dist {
+                min_dist = dist;
+                best_cluster = j;
+            }
+        }
+        
+        labels_array.push(Integer::from_value(ruby.eval(&format!("{}", best_cluster)).unwrap()).unwrap())?;
+    }
+    
+    Ok(labels_array)
+}
+
+/// K-means++ initialization
+fn kmeans_plusplus(data: &Array2<f64>, k: usize) -> Result<Array2<f64>, Error> {
+    let n_samples = data.nrows();
+    let n_features = data.ncols();
+    let mut rng = thread_rng();
+    
+    let mut centroids = Array2::<f64>::zeros((k, n_features));
+    
+    // Choose first centroid randomly
+    let first_idx = rng.gen_range(0..n_samples);
+    centroids.row_mut(0).assign(&data.row(first_idx));
+    
+    // Choose remaining centroids
+    for i in 1..k {
+        let mut distances = vec![f64::INFINITY; n_samples];
+        
+        // Calculate distance to nearest centroid for each point
+        for j in 0..n_samples {
+            for c in 0..i {
+                let dist = euclidean_distance(&data.row(j), &centroids.row(c));
+                if dist < distances[j] {
+                    distances[j] = dist;
+                }
+            }
+        }
+        
+        // Convert distances to probabilities
+        let total: f64 = distances.iter().map(|d| d * d).sum();
+        if total == 0.0 {
+            // All points are identical or we've selected duplicates
+            // Just use sequential points as centroids
+            if i < n_samples {
+                centroids.row_mut(i).assign(&data.row(i));
+            } else {
+                // Reuse first point if we run out
+                centroids.row_mut(i).assign(&data.row(0));
+            }
+            continue;
+        }
+        
+        // Choose next centroid with probability proportional to squared distance
+        let mut cumsum = 0.0;
+        let rand_val: f64 = rng.gen::<f64>() * total;
+        
+        for j in 0..n_samples {
+            cumsum += distances[j] * distances[j];
+            if cumsum >= rand_val {
+                centroids.row_mut(i).assign(&data.row(j));
+                break;
+            }
+        }
+    }
+    
+    Ok(centroids)
+}
+
+/// Calculate Euclidean distance between two points
+fn euclidean_distance(a: &ArrayView1<f64>, b: &ArrayView1<f64>) -> f64 {
+    a.iter()
+        .zip(b.iter())
+        .map(|(x, y)| (x - y).powi(2))
+        .sum::<f64>()
+        .sqrt()
+}