clusterkit 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ebad40c2aac3fa3569357eedf740336a3d463ecc5c038d2771d3fd266d414b1e
-  data.tar.gz: 0ab2851e0adab583567460e469d4073e755514854e57bca464d5a835c1534bea
+  metadata.gz: ed8756bb9b2d2cf253c849f529b18aa57c0043fe0e0b65997fffc62522d21fd6
+  data.tar.gz: adcb4a5186e042a0be6d9e712b8dbff5751c469295ccbef9034bf40688658c21
 SHA512:
-  metadata.gz: 16412a2db10bf55593c778c7a02813b8db9652e39479d17fa3cd10da1e87298d5d95f4afccd8f5da10bb89bc950b314faeb9a8f0149ecd2dd3c3305121c1b5b2
-  data.tar.gz: edf73f2d0ce8f73441c07975a5cd4da31faaece9b216d908a743b9c1bd3f2d8e213d1b85a7e82f2f397c60f8d2fd2c4a0109a5a337963e008ecf069b3fa40266
+  metadata.gz: 3f16b15661cf7db9a42f080dc5cb8835ded9f36518f0f3e095ea50bf7c93555906a1b3233744b11551cb2d45c1b3a9b1d2e5a64051d770b277e6a0669e0698b2
+  data.tar.gz: 36e4df00fbfb48b44bdcacbcb29e9a077c5711cc5cbc79d0ae9091c604db489bea06fa1c354ad89a87aa50e5c7dc8fe2b75f426e1942ae01d46887dda72b5ab1

data/clusterkit.gemspec

@@ -30,16 +30,16 @@ Gem::Specification.new do |spec|
   # Runtime dependencies
   # Numo is optional but recommended for better performance
   # spec.add_dependency "numo-narray", "~> 0.9"
+  spec.add_dependency "rb_sys", "~> 0.9"
 
   # Development dependencies
   spec.add_development_dependency "csv"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rake-compiler", "~> 1.2"
-  spec.add_development_dependency "rb_sys", "~> 0.9"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "simplecov", "~> 0.22"
   spec.add_development_dependency "yard", "~> 0.9"
 
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html
-end
+end

data/ext/clusterkit/src/clustering/hdbscan_wrapper.rs

@@ -1,5 +1,6 @@
-use magnus::{function, prelude::*, Error, Value, RArray, RHash, Integer, TryConvert};
+use magnus::{function, prelude::*, Error, Value, RArray, RHash, Integer};
 use hdbscan::{Hdbscan, HdbscanHyperParams};
+use crate::utils::ruby_array_to_vec_vec_f64;
 
 /// Perform HDBSCAN clustering
 /// Returns a hash with labels and basic statistics
@@ -9,32 +10,9 @@ pub fn hdbscan_fit(
     min_cluster_size: usize,
     metric: String,
 ) -> Result<RHash, Error> {
-    // Convert Ruby array to ndarray
-    let rarray: RArray = TryConvert::try_convert(data)?;
-    let n_samples = rarray.len();
-    
-    if n_samples == 0 {
-        return Err(Error::new(
-            magnus::exception::arg_error(),
-            "Data cannot be empty",
-        ));
-    }
-    
-    // Get dimensions
-    let first_row: RArray = rarray.entry::<RArray>(0)?;
-    let n_features = first_row.len();
-    
-    // Convert to Vec<Vec<f64>> format expected by hdbscan crate
-    let mut data_vec: Vec<Vec<f64>> = Vec::with_capacity(n_samples);
-    for i in 0..n_samples {
-        let row: RArray = rarray.entry(i as isize)?;
-        let mut row_vec: Vec<f64> = Vec::with_capacity(n_features);
-        for j in 0..n_features {
-            let val: f64 = row.entry(j as isize)?;
-            row_vec.push(val);
-        }
-        data_vec.push(row_vec);
-    }
+    // Convert Ruby array to Vec<Vec<f64>> using shared helper
+    let data_vec = ruby_array_to_vec_vec_f64(data)?;
+    let n_samples = data_vec.len();
     
     // Note: hdbscan crate doesn't support custom metrics directly
     // We'll use the default Euclidean distance for now

data/ext/clusterkit/src/clustering.rs

@@ -1,6 +1,9 @@
-use magnus::{function, prelude::*, Error, Value, RArray, Integer, TryConvert};
+use magnus::{function, prelude::*, Error, Value, RArray, Integer};
 use ndarray::{Array1, Array2, ArrayView1, Axis};
 use rand::prelude::*;
+use rand::rngs::StdRng;
+use rand::SeedableRng;
+use crate::utils::{ruby_array_to_ndarray2};
 
 mod hdbscan_wrapper;
 
@@ -9,7 +12,7 @@ pub fn init(parent: &magnus::RModule) -> Result<(), Error> {
     
     clustering_module.define_singleton_method(
         "kmeans_rust",
-        function!(kmeans, 3),
+        function!(kmeans, 4),
     )?;
     
     clustering_module.define_singleton_method(
@@ -25,21 +28,10 @@ pub fn init(parent: &magnus::RModule) -> Result<(), Error> {
 
 /// Perform K-means clustering
 /// Returns (labels, centroids, inertia)
-fn kmeans(data: Value, k: usize, max_iter: usize) -> Result<(RArray, RArray, f64), Error> {
-    // Convert Ruby array to ndarray
-    let rarray: RArray = TryConvert::try_convert(data)?;
-    let n_samples = rarray.len();
-    
-    if n_samples == 0 {
-        return Err(Error::new(
-            magnus::exception::arg_error(),
-            "Data cannot be empty",
-        ));
-    }
-    
-    // Get dimensions
-    let first_row: RArray = rarray.entry::<RArray>(0)?;
-    let n_features = first_row.len();
+fn kmeans(data: Value, k: usize, max_iter: usize, random_seed: Option<i64>) -> Result<(RArray, RArray, f64), Error> {
+    // Convert Ruby array to ndarray using shared helper
+    let data_array = ruby_array_to_ndarray2(data)?;
+    let (n_samples, n_features) = data_array.dim();
     
     if k > n_samples {
         return Err(Error::new(
@@ -48,18 +40,8 @@ fn kmeans(data: Value, k: usize, max_iter: usize) -> Result<(RArray, RArray, f64
         ));
     }
     
-    // Convert to ndarray
-    let mut data_array = Array2::<f64>::zeros((n_samples, n_features));
-    for i in 0..n_samples {
-        let row: RArray = rarray.entry(i as isize)?;
-        for j in 0..n_features {
-            let val: f64 = row.entry(j as isize)?;
-            data_array[[i, j]] = val;
-        }
-    }
-    
     // Initialize centroids using K-means++
-    let mut centroids = kmeans_plusplus(&data_array, k)?;
+    let mut centroids = kmeans_plusplus(&data_array, k, random_seed)?;
     let mut labels = vec![0usize; n_samples];
     let mut prev_labels = vec![0usize; n_samples];
     
@@ -140,43 +122,12 @@ fn kmeans(data: Value, k: usize, max_iter: usize) -> Result<(RArray, RArray, f64
 
 /// Predict cluster labels for new data given centroids
 fn kmeans_predict(data: Value, centroids: Value) -> Result<RArray, Error> {
-    // Convert inputs
-    let data_array: RArray = TryConvert::try_convert(data)?;
-    let centroids_array: RArray = TryConvert::try_convert(centroids)?;
-    
-    let n_samples = data_array.len();
-    let k = centroids_array.len();
-    
-    if n_samples == 0 {
-        return Err(Error::new(
-            magnus::exception::arg_error(),
-            "Data cannot be empty",
-        ));
-    }
-    
-    // Get dimensions
-    let first_row: RArray = data_array.entry::<RArray>(0)?;
-    let n_features = first_row.len();
-    
-    // Convert data to ndarray
-    let mut data_matrix = Array2::<f64>::zeros((n_samples, n_features));
-    for i in 0..n_samples {
-        let row: RArray = data_array.entry(i as isize)?;
-        for j in 0..n_features {
-            let val: f64 = row.entry(j as isize)?;
-            data_matrix[[i, j]] = val;
-        }
-    }
+    // Convert inputs using shared helpers
+    let data_matrix = ruby_array_to_ndarray2(data)?;
+    let centroids_matrix = ruby_array_to_ndarray2(centroids)?;
     
-    // Convert centroids to ndarray
-    let mut centroids_matrix = Array2::<f64>::zeros((k, n_features));
-    for i in 0..k {
-        let row: RArray = centroids_array.entry(i as isize)?;
-        for j in 0..n_features {
-            let val: f64 = row.entry(j as isize)?;
-            centroids_matrix[[i, j]] = val;
-        }
-    }
+    let (n_samples, _) = data_matrix.dim();
+    let (_k, _) = centroids_matrix.dim();
     
     // Predict labels
     let ruby = magnus::Ruby::get().unwrap();
@@ -202,10 +153,19 @@ fn kmeans_predict(data: Value, centroids: Value) -> Result<RArray, Error> {
 }
 
 /// K-means++ initialization
-fn kmeans_plusplus(data: &Array2<f64>, k: usize) -> Result<Array2<f64>, Error> {
+fn kmeans_plusplus(data: &Array2<f64>, k: usize, random_seed: Option<i64>) -> Result<Array2<f64>, Error> {
     let n_samples = data.nrows();
     let n_features = data.ncols();
-    let mut rng = thread_rng();
+    
+    // Use seeded RNG if seed is provided, otherwise use thread_rng
+    let mut rng: Box<dyn RngCore> = match random_seed {
+        Some(seed) => {
+            // Convert i64 to u64 for seeding (negative numbers wrap around)
+            let seed_u64 = seed as u64;
+            Box::new(StdRng::seed_from_u64(seed_u64))
+        },
+        None => Box::new(thread_rng()),
+    };
     
     let mut centroids = Array2::<f64>::zeros((k, n_features));
     

data/ext/clusterkit/src/embedder.rs

@@ -1,4 +1,4 @@
-use magnus::{Error, RArray, RHash, Value, TryConvert, Integer, Float, Module, Object};
+use magnus::{Error, RArray, RHash, Value, TryConvert, Integer, Module, Object};
 use magnus::value::ReprValue;
 use hnsw_rs::prelude::*;
 use annembed::prelude::*;
@@ -7,6 +7,7 @@ use std::io::{Write, Read};
 use std::cell::RefCell;
 use bincode;
 use serde::{Serialize, Deserialize};
+use crate::utils::ruby_array_to_vec_vec_f32;
 
 // Simple struct to serialize UMAP results
 #[derive(Serialize, Deserialize)]
@@ -124,61 +125,8 @@ impl RustUMAP {
     }
 
     fn fit_transform(&self, data: Value) -> Result<RArray, Error> {
-        // Convert Ruby array to Rust Vec<Vec<f64>>
-        let ruby_array = RArray::try_convert(data)?;
-        let mut rust_data: Vec<Vec<f64>> = Vec::new();
-
-        // Get array length
-        let array_len = ruby_array.len();
-
-        for i in 0..array_len {
-            let row = ruby_array.entry::<Value>(i as isize)?;
-            let row_array = RArray::try_convert(row).map_err(|_| {
-                Error::new(
-                    magnus::exception::type_error(),
-                    "Expected array of arrays (2D array)",
-                )
-            })?;
-
-            let mut rust_row: Vec<f64> = Vec::new();
-            let row_len = row_array.len();
-
-            for j in 0..row_len {
-                let val = row_array.entry::<Value>(j as isize)?;
-                let float_val = if let Ok(f) = Float::try_convert(val) {
-                    f.to_f64()
-                } else if let Ok(i) = Integer::try_convert(val) {
-                    i.to_i64()? as f64
-                } else {
-                    return Err(Error::new(
-                        magnus::exception::type_error(),
-                        "All values must be numeric",
-                    ));
-                };
-                rust_row.push(float_val);
-            }
-
-            if !rust_data.is_empty() && rust_row.len() != rust_data[0].len() {
-                return Err(Error::new(
-                    magnus::exception::arg_error(),
-                    "All rows must have the same length",
-                ));
-            }
-
-            rust_data.push(rust_row);
-        }
-
-        if rust_data.is_empty() {
-            return Err(Error::new(
-                magnus::exception::arg_error(),
-                "Input data cannot be empty",
-            ));
-        }
-
-        // Convert to Vec<Vec<f32>> for HNSW
-        let data_f32: Vec<Vec<f32>> = rust_data.iter()
-            .map(|row| row.iter().map(|&x| x as f32).collect())
-            .collect();
+        // Convert Ruby array to Rust Vec<Vec<f32>> using shared helper
+        let data_f32 = ruby_array_to_vec_vec_f32(data)?;
 
         // Build HNSW graph
         let ef_c = 50;
@@ -331,31 +279,8 @@ impl RustUMAP {
         let training_embeddings_ref = training_embeddings.as_ref()
             .ok_or_else(|| Error::new(magnus::exception::runtime_error(), "No embeddings available."))?;
 
-        // Convert input data to Rust format
-        let ruby_array = RArray::try_convert(data)?;
-        let mut new_data: Vec<Vec<f32>> = Vec::new();
-
-        for i in 0..ruby_array.len() {
-            let row = ruby_array.entry::<Value>(i as isize)?;
-            let row_array = RArray::try_convert(row)?;
-            let mut rust_row: Vec<f32> = Vec::new();
-
-            for j in 0..row_array.len() {
-                let val = row_array.entry::<Value>(j as isize)?;
-                let float_val = if let Ok(f) = Float::try_convert(val) {
-                    f.to_f64() as f32
-                } else if let Ok(i) = Integer::try_convert(val) {
-                    i.to_i64()? as f32
-                } else {
-                    return Err(Error::new(
-                        magnus::exception::type_error(),
-                        "All values must be numeric",
-                    ));
-                };
-                rust_row.push(float_val);
-            }
-            new_data.push(rust_row);
-        }
+        // Convert input data to Rust format using shared helper
+        let new_data = ruby_array_to_vec_vec_f32(data)?;
 
         // For each new point, find k nearest neighbors in training data
         // and average their embeddings (weighted by distance)

data/ext/clusterkit/src/svd.rs

@@ -1,6 +1,6 @@
-use magnus::{function, prelude::*, Error, Value, RArray, TryConvert};
+use magnus::{function, prelude::*, Error, Value, RArray};
 use annembed::tools::svdapprox::{SvdApprox, RangeApproxMode, RangeRank, MatRepr};
-use ndarray::Array2;
+use crate::utils::ruby_array_to_ndarray2;
 
 pub fn init(parent: &magnus::RModule) -> Result<(), Error> {
     let svd_module = parent.define_module("SVD")?;
@@ -14,20 +14,9 @@ pub fn init(parent: &magnus::RModule) -> Result<(), Error> {
 }
 
 fn randomized_svd(matrix: Value, k: usize, n_iter: usize) -> Result<RArray, Error> {
-    // Convert Ruby array to ndarray
-    let rarray: RArray = TryConvert::try_convert(matrix)?;
-    
-    // Check if it's a 2D array
-    let first_row: RArray = rarray.entry::<RArray>(0)?;
-    let n_rows = rarray.len();
-    let n_cols = first_row.len();
-    
-    if n_rows == 0 || n_cols == 0 {
-        return Err(Error::new(
-            magnus::exception::arg_error(),
-            "Matrix cannot be empty",
-        ));
-    }
+    // Convert Ruby array to ndarray using shared helper
+    let matrix_data = ruby_array_to_ndarray2(matrix)?;
+    let (n_rows, n_cols) = matrix_data.dim();
     
     if k > n_rows.min(n_cols) {
         return Err(Error::new(
@@ -36,16 +25,6 @@ fn randomized_svd(matrix: Value, k: usize, n_iter: usize) -> Result<RArray, Erro
         ));
     }
     
-    // Convert to ndarray Array2
-    let mut matrix_data = Array2::<f64>::zeros((n_rows, n_cols));
-    for i in 0..n_rows {
-        let row: RArray = rarray.entry(i as isize)?;
-        for j in 0..n_cols {
-            let val: f64 = row.entry(j as isize)?;
-            matrix_data[[i, j]] = val;
-        }
-    }
-    
     // Create MatRepr for the full matrix
     let mat_repr = MatRepr::from_array2(matrix_data.clone());
     

data/ext/clusterkit/src/utils.rs

@@ -1,4 +1,5 @@
-use magnus::{function, prelude::*, Error, Value};
+use magnus::{function, prelude::*, Error, Value, RArray, TryConvert, Float, Integer};
+use ndarray::Array2;
 
 pub fn init(parent: &magnus::RModule) -> Result<(), Error> {
     let utils_module = parent.define_module("Utils")?;
@@ -30,4 +31,150 @@ fn estimate_hubness(_data: Value) -> Result<Value, Error> {
         magnus::exception::not_imp_error(),
         "Hubness estimation not implemented yet",
     ))
+}
+
+/// Convert Ruby 2D array to ndarray Array2<f64>
+/// Handles validation and provides consistent error messages
+pub fn ruby_array_to_ndarray2(data: Value) -> Result<Array2<f64>, Error> {
+    let rarray: RArray = TryConvert::try_convert(data)?;
+    let n_samples = rarray.len();
+
+    if n_samples == 0 {
+        return Err(Error::new(
+            magnus::exception::arg_error(),
+            "Data cannot be empty",
+        ));
+    }
+
+    // Get dimensions from first row
+    let first_row: RArray = rarray.entry::<RArray>(0)?;
+    let n_features = first_row.len();
+
+    if n_features == 0 {
+        return Err(Error::new(
+            magnus::exception::arg_error(),
+            "Data rows cannot be empty",
+        ));
+    }
+
+    // Create ndarray and populate
+    let mut data_array = Array2::<f64>::zeros((n_samples, n_features));
+    for i in 0..n_samples {
+        let row: RArray = rarray.entry(i as isize)?;
+        
+        // Validate row length consistency
+        if row.len() != n_features {
+            return Err(Error::new(
+                magnus::exception::arg_error(),
+                format!("Row {} has {} elements, expected {}", i, row.len(), n_features),
+            ));
+        }
+
+        for j in 0..n_features {
+            let val: f64 = row.entry(j as isize)?;
+            data_array[[i, j]] = val;
+        }
+    }
+
+    Ok(data_array)
+}
+
+/// Convert Ruby 2D array to Vec<Vec<f64>>
+/// Handles validation and provides consistent error messages  
+pub fn ruby_array_to_vec_vec_f64(data: Value) -> Result<Vec<Vec<f64>>, Error> {
+    let rarray: RArray = TryConvert::try_convert(data)?;
+    let n_samples = rarray.len();
+
+    if n_samples == 0 {
+        return Err(Error::new(
+            magnus::exception::arg_error(),
+            "Data cannot be empty",
+        ));
+    }
+
+    let mut data_vec: Vec<Vec<f64>> = Vec::with_capacity(n_samples);
+    let mut expected_features: Option<usize> = None;
+
+    for i in 0..n_samples {
+        let row: RArray = rarray.entry(i as isize)?;
+        let n_features = row.len();
+        
+        // Check row length consistency
+        match expected_features {
+            Some(expected) => {
+                if n_features != expected {
+                    return Err(Error::new(
+                        magnus::exception::arg_error(),
+                        format!("Row {} has {} elements, expected {}", i, n_features, expected),
+                    ));
+                }
+            }
+            None => expected_features = Some(n_features),
+        }
+
+        let mut row_vec: Vec<f64> = Vec::with_capacity(n_features);
+        for j in 0..n_features {
+            let val: f64 = row.entry(j as isize)?;
+            row_vec.push(val);
+        }
+        data_vec.push(row_vec);
+    }
+
+    Ok(data_vec)
+}
+
+/// Convert Ruby 2D array to Vec<Vec<f32>>
+/// For algorithms that require f32 precision (like UMAP)
+pub fn ruby_array_to_vec_vec_f32(data: Value) -> Result<Vec<Vec<f32>>, Error> {
+    let rarray: RArray = TryConvert::try_convert(data)?;
+    let array_len = rarray.len();
+
+    if array_len == 0 {
+        return Err(Error::new(
+            magnus::exception::arg_error(),
+            "Input data cannot be empty",
+        ));
+    }
+
+    let mut rust_data: Vec<Vec<f32>> = Vec::with_capacity(array_len);
+
+    for i in 0..array_len {
+        let row = rarray.entry::<Value>(i as isize)?;
+        let row_array = RArray::try_convert(row).map_err(|_| {
+            Error::new(
+                magnus::exception::type_error(),
+                "Expected array of arrays (2D array)",
+            )
+        })?;
+
+        let mut rust_row: Vec<f32> = Vec::new();
+        let row_len = row_array.len();
+
+        for j in 0..row_len {
+            let val = row_array.entry::<Value>(j as isize)?;
+            let float_val = if let Ok(f) = Float::try_convert(val) {
+                f.to_f64() as f32
+            } else if let Ok(i) = Integer::try_convert(val) {
+                i.to_i64()? as f32
+            } else {
+                return Err(Error::new(
+                    magnus::exception::type_error(),
+                    "All values must be numeric",
+                ));
+            };
+            rust_row.push(float_val);
+        }
+
+        // Validate row length consistency
+        if !rust_data.is_empty() && rust_row.len() != rust_data[0].len() {
+            return Err(Error::new(
+                magnus::exception::arg_error(),
+                "All rows must have the same length",
+            ));
+        }
+
+        rust_data.push(rust_row);
+    }
+
+    Ok(rust_data)
 }

data/lib/clusterkit/clustering/hdbscan.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../data_validator'
+
 module ClusterKit
   module Clustering
     # HDBSCAN clustering algorithm - matching KMeans API pattern
@@ -128,23 +130,8 @@ module ClusterKit
       private
 
       def validate_data(data)
-        # Exact same validation as KMeans for consistency
-        raise ArgumentError, "Data must be an array" unless data.is_a?(Array)
-        raise ArgumentError, "Data cannot be empty" if data.empty?
-        raise ArgumentError, "Data must be 2D array" unless data.first.is_a?(Array)
-        
-        row_length = data.first.length
-        unless data.all? { |row| row.is_a?(Array) && row.length == row_length }
-          raise ArgumentError, "All rows must have the same length"
-        end
-        
-        data.each_with_index do |row, i|
-          row.each_with_index do |val, j|
-            unless val.is_a?(Numeric)
-              raise ArgumentError, "Element at position [#{i}, #{j}] is not numeric"
-            end
-          end
-        end
+        # Use same validation as KMeans for consistency
+        DataValidator.validate_clustering(data, check_finite: false)
       end
     end
 

data/lib/clusterkit/clustering.rb

@@ -2,6 +2,7 @@
 
 require_relative 'clusterkit'
 require_relative 'clustering/hdbscan'
+require_relative 'data_validator'
 
 module ClusterKit
   # Module for clustering algorithms
@@ -28,11 +29,8 @@ module ClusterKit
       def fit(data)
         validate_data(data)
         
-        # Set random seed if provided
-        srand(@random_seed) if @random_seed
-        
-        # Call Rust implementation
-        @labels, @centroids, @inertia = Clustering.kmeans_rust(data, @k, @max_iter)
+        # Call Rust implementation with optional seed
+        @labels, @centroids, @inertia = Clustering.kmeans_rust(data, @k, @max_iter, @random_seed)
         @fitted = true
         
         self
@@ -132,24 +130,7 @@ module ClusterKit
       private
 
       def validate_data(data)
-        raise ArgumentError, "Data must be an array" unless data.is_a?(Array)
-        raise ArgumentError, "Data cannot be empty" if data.empty?
-        raise ArgumentError, "Data must be 2D array" unless data.first.is_a?(Array)
-        
-        # Check all rows have same length
-        row_length = data.first.length
-        unless data.all? { |row| row.is_a?(Array) && row.length == row_length }
-          raise ArgumentError, "All rows must have the same length"
-        end
-        
-        # Check all values are numeric
-        data.each_with_index do |row, i|
-          row.each_with_index do |val, j|
-            unless val.is_a?(Numeric)
-              raise ArgumentError, "Element at position [#{i}, #{j}] is not numeric"
-            end
-          end
-        end
+        DataValidator.validate_clustering(data, check_finite: false)
       end
     end
 

data/lib/clusterkit/data_validator.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module ClusterKit
+  # Shared data validation methods for all algorithms
+  module DataValidator
+    class << self
+      # Validate basic data structure and types
+      # @param data [Array] Data to validate
+      # @raise [ArgumentError] If data structure is invalid
+      def validate_basic_structure(data)
+        raise ArgumentError, "Input must be an array" unless data.is_a?(Array)
+        raise ArgumentError, "Input cannot be empty" if data.empty?
+
+        first_row = data.first
+        raise ArgumentError, "Input must be a 2D array (array of arrays)" unless first_row.is_a?(Array)
+      end
+
+      # Validate row consistency (all rows have same length)
+      # @param data [Array] 2D array to validate
+      # @raise [ArgumentError] If rows have different lengths
+      def validate_row_consistency(data)
+        row_length = data.first.length
+
+        data.each_with_index do |row, i|
+          unless row.is_a?(Array)
+            raise ArgumentError, "Row #{i} is not an array"
+          end
+
+          if row.length != row_length
+            raise ArgumentError, "All rows must have the same length (row #{i} has #{row.length} elements, expected #{row_length})"
+          end
+        end
+      end
+
+      # Validate that all elements are numeric
+      # @param data [Array] 2D array to validate
+      # @raise [ArgumentError] If any element is not numeric
+      def validate_numeric_types(data)
+        data.each_with_index do |row, i|
+          row.each_with_index do |val, j|
+            unless val.is_a?(Numeric)
+              raise ArgumentError, "Element at position [#{i}, #{j}] is not numeric"
+            end
+          end
+        end
+      end
+
+      # Validate finite values (no NaN or Infinite)
+      # @param data [Array] 2D array to validate
+      # @raise [ArgumentError] If any float is NaN or Infinite
+      def validate_finite_values(data)
+        data.each_with_index do |row, i|
+          row.each_with_index do |val, j|
+            # Only check for NaN/Infinite on floats
+            if val.is_a?(Float) && (val.nan? || val.infinite?)
+              raise ArgumentError, "Element at position [#{i}, #{j}] is NaN or Infinite"
+            end
+          end
+        end
+      end
+
+      # Standard validation for most algorithms
+      # @param data [Array] 2D array to validate
+      # @param check_finite [Boolean] Whether to check for NaN/Infinite values
+      # @raise [ArgumentError] If data is invalid
+      def validate_standard(data, check_finite: true)
+        validate_basic_structure(data)
+        validate_row_consistency(data)
+        validate_numeric_types(data)
+        validate_finite_values(data) if check_finite
+      end
+
+      # Validation for clustering algorithms (KMeans, HDBSCAN) with specific error messages
+      # @param data [Array] 2D array to validate
+      # @param check_finite [Boolean] Whether to check for NaN/Infinite values
+      # @raise [ArgumentError] If data is invalid
+      def validate_clustering(data, check_finite: false)
+        raise ArgumentError, "Data must be an array" unless data.is_a?(Array)
+        raise ArgumentError, "Data cannot be empty" if data.empty?
+        raise ArgumentError, "Data must be 2D array" unless data.first.is_a?(Array)
+
+        validate_row_consistency(data)
+        validate_numeric_types(data)
+        validate_finite_values(data) if check_finite
+      end
+
+      # Validation for PCA with specific error messages (same as clustering but without finite checks)
+      # @param data [Array] 2D array to validate
+      # @raise [ArgumentError] If data is invalid
+      def validate_pca(data)
+        raise ArgumentError, "Data must be an array" unless data.is_a?(Array)
+        raise ArgumentError, "Data cannot be empty" if data.empty?
+        raise ArgumentError, "Data must be 2D array" unless data.first.is_a?(Array)
+
+        validate_row_consistency(data)
+        validate_numeric_types(data)
+      end
+
+      # Get data statistics for warnings/error context
+      # @param data [Array] 2D array
+      # @return [Hash] Statistics about the data
+      def data_statistics(data)
+        return { n_samples: 0, n_features: 0, data_range: 0.0 } if data.empty?
+
+        n_samples = data.size
+        n_features = data.first&.size || 0
+        
+        # Calculate data range for warnings
+        min_val = Float::INFINITY
+        max_val = -Float::INFINITY
+
+        data.each do |row|
+          row.each do |val|
+            val_f = val.to_f
+            min_val = val_f if val_f < min_val
+            max_val = val_f if val_f > max_val
+          end
+        end
+
+        data_range = max_val - min_val
+
+        {
+          n_samples: n_samples,
+          n_features: n_features,
+          data_range: data_range,
+          min_value: min_val,
+          max_value: max_val
+        }
+      end
+    end
+  end
+end

data/lib/clusterkit/dimensionality/pca.rb

@@ -2,6 +2,7 @@
 
 require_relative '../clusterkit'
 require_relative 'svd'
+require_relative '../data_validator'
 
 module ClusterKit
   module Dimensionality
@@ -30,7 +31,7 @@ module ClusterKit
       
       # Perform SVD on centered data
       # U contains the transformed data, S contains singular values, VT contains components
-      u, s, vt = ClusterKit.svd(centered_data, @n_components, n_iter: 5)
+      u, s, vt = perform_svd(centered_data)
       
       # Store the principal components (eigenvectors)
       @components = vt  # Shape: (n_components, n_features)
@@ -76,7 +77,7 @@ module ClusterKit
       centered_data = center_data(data, @mean)
       
       # Perform SVD on centered data
-      u, s, vt = SVD.randomized_svd(centered_data, @n_components, n_iter: 5)
+      u, s, vt = perform_svd(centered_data)
       
       # Store the principal components (eigenvectors)
       @components = vt
@@ -166,17 +167,10 @@ module ClusterKit
     private
 
     def validate_data(data)
-      raise ArgumentError, "Data must be an array" unless data.is_a?(Array)
-      raise ArgumentError, "Data cannot be empty" if data.empty?
-      raise ArgumentError, "Data must be 2D array" unless data.first.is_a?(Array)
-      
-      # Check all rows have same length
-      row_length = data.first.length
-      unless data.all? { |row| row.is_a?(Array) && row.length == row_length }
-        raise ArgumentError, "All rows must have the same length"
-      end
+      # Use shared validation for common checks
+      DataValidator.validate_pca(data)
       
-      # Check we have enough samples for n_components
+      # PCA-specific validations
       if data.size < @n_components
         raise ArgumentError, "n_components (#{@n_components}) cannot be larger than n_samples (#{data.size})"
       end
@@ -237,6 +231,12 @@ module ClusterKit
       
       transformed
     end
+    
+    # Shared SVD computation for both fit and fit_transform
+    # Ensures both methods use identical SVD invocation and parameters
+    def perform_svd(centered_data)
+      SVD.randomized_svd(centered_data, @n_components, n_iter: 5)
+    end
   end
 
   # Module-level convenience method

data/lib/clusterkit/dimensionality/svd.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../clusterkit'
+require_relative '../data_validator'
 
 module ClusterKit
   module Dimensionality
@@ -8,7 +9,7 @@ module ClusterKit
     # Decomposes a matrix into U, S, V^T components
     class SVD
       attr_reader :n_components, :n_iter, :random_seed
-      attr_reader :u, :s, :vt
+      attr_reader :u, :s, :vt, :n_features
       
       # Initialize a new SVD instance
       # @param n_components [Integer] Number of components to compute
@@ -27,7 +28,8 @@ module ClusterKit
       def fit_transform(data)
         validate_input(data)
         
-        # Store reference to original data for transform detection
+        # Store data characteristics for later transform operations
+        @n_features = data.first.size
         @original_data_id = data.object_id
         
         # Determine n_components if not set
@@ -77,26 +79,21 @@ module ClusterKit
       
       # Transform data using fitted SVD (project onto components)
       # @param data [Array<Array<Numeric>>] Data to transform
-      # @return [Array<Array<Float>>] Transformed data (U * S)
+      # @return [Array<Array<Float>>] Transformed data projected onto SVD components
       def transform(data)
         raise RuntimeError, "Model must be fitted first" unless fitted?
-        validate_input(data)
-        
-        # For SVD, transform typically means projecting onto the components
-        # This is equivalent to data * V (or data * V^T.T)
-        # But for dimensionality reduction, we usually want U * S
-        # which is already computed in fit_transform
+        validate_transform_input(data)
         
-        # If transforming new data, we'd need to project it
-        # For now, return U * S for the fitted data
         if data.object_id == @original_data_id
           # Same data that was fitted - return U * S
           @u.map.with_index do |row, i|
             row.map.with_index { |val, j| val * @s[j] }
           end
         else
-          # New data - would need proper projection
-          raise NotImplementedError, "Transform for new data not yet implemented"
+          # New data - project onto V components: data × V
+          # Since we have V^T, we need to transpose it back to V
+          # V = V^T^T, so we project: data × V^T^T
+          transform_new_data(data)
         end
       end
       
@@ -135,9 +132,43 @@ module ClusterKit
       private
       
       def validate_input(data)
-        raise ArgumentError, "Input must be an array" unless data.is_a?(Array)
-        raise ArgumentError, "Input cannot be empty" if data.empty?
-        raise ArgumentError, "Input must be a 2D array" unless data.first.is_a?(Array)
+        DataValidator.validate_standard(data, check_finite: false)
+      end
+      
+      def validate_transform_input(data)
+        DataValidator.validate_standard(data, check_finite: false)
+        
+        # Check feature count matches training data
+        if data.first.size != @n_features
+          raise ArgumentError, "New data has #{data.first.size} features, but model was fitted with #{@n_features} features"
+        end
+      end
+      
+      # Transform new data by projecting onto V components
+      # Mathematical operation: new_data × V, where V = V^T^T
+      def transform_new_data(data)
+        # V^T is stored as @vt (shape: n_components × n_features)
+        # We need V (shape: n_features × n_components)
+        # V = V^T^T, so we transpose @vt
+        
+        result = []
+        data.each do |sample|
+          # Project sample onto each component (column of V = row of V^T)
+          projected = Array.new(@vt.size, 0.0)
+          
+          @vt.each_with_index do |vt_row, comp_idx|
+            # Dot product: sample · vt_row (this is sample · V[:, comp_idx])
+            dot_product = 0.0
+            sample.each_with_index do |val, feat_idx|
+              dot_product += val * vt_row[feat_idx]
+            end
+            projected[comp_idx] = dot_product
+          end
+          
+          result << projected
+        end
+        
+        result
       end
     end
   end

data/lib/clusterkit/dimensionality/umap.rb

@@ -4,6 +4,7 @@ require 'fileutils'
 require 'json'
 require_relative '../configuration'
 require_relative '../silence'
+require_relative '../data_validator'
 
 module ClusterKit
   module Dimensionality
@@ -224,44 +225,10 @@ module ClusterKit
     end
 
     def validate_input(data, check_min_samples: true)
-      raise ArgumentError, "Input must be an array" unless data.is_a?(Array)
-      raise ArgumentError, "Input cannot be empty" if data.empty?
+      # Use shared validation for common checks
+      DataValidator.validate_standard(data)
 
-      first_row = data.first
-      raise ArgumentError, "Input must be a 2D array (array of arrays)" unless first_row.is_a?(Array)
-
-      row_length = first_row.length
-      min_val = Float::INFINITY
-      max_val = -Float::INFINITY
-
-      # First validate data structure and types
-      data.each_with_index do |row, i|
-        unless row.is_a?(Array)
-          raise ArgumentError, "Row #{i} is not an array"
-        end
-
-        if row.length != row_length
-          raise ArgumentError, "All rows must have the same length (row #{i} has #{row.length} elements, expected #{row_length})"
-        end
-
-        row.each_with_index do |val, j|
-          unless val.is_a?(Numeric)
-            raise ArgumentError, "Element at position [#{i}, #{j}] is not numeric"
-          end
-
-          # Only check for NaN/Infinite on floats
-          if val.is_a?(Float) && (val.nan? || val.infinite?)
-            raise ArgumentError, "Element at position [#{i}, #{j}] is NaN or Infinite"
-          end
-
-          # Track data range
-          val_f = val.to_f
-          min_val = val_f if val_f < min_val
-          max_val = val_f if val_f > max_val
-        end
-      end
-
-      # Check for sufficient data points after validating structure (only for fit operations)
+      # UMAP-specific validations
       if check_min_samples && data.size < 10
         raise ::ClusterKit::InsufficientDataError, <<~MSG
           UMAP requires at least 10 data points, but only #{data.size} provided.
@@ -274,9 +241,9 @@ module ClusterKit
       end
 
       # Check for extreme data ranges that might cause numerical issues
-      data_range = max_val - min_val
-      if data_range > 1000
-        warn "WARNING: Large data range detected (#{data_range.round(2)}). Consider normalizing your data to prevent numerical instability."
+      stats = DataValidator.data_statistics(data)
+      if stats[:data_range] > 1000
+        warn "WARNING: Large data range detected (#{stats[:data_range].round(2)}). Consider normalizing your data to prevent numerical instability."
       end
     end
 

data/lib/clusterkit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ClusterKit
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: clusterkit
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Chris Petersen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-09-05 00:00:00.000000000 Z
+date: 2025-09-06 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: csv
   requirement: !ruby/object:Gem::Requirement
@@ -52,20 +66,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
-- !ruby/object:Gem::Dependency
-  name: rb_sys
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.9'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -159,6 +159,7 @@ files:
 - lib/clusterkit/clustering/hdbscan.rb
 - lib/clusterkit/clusterkit.rb
 - lib/clusterkit/configuration.rb
+- lib/clusterkit/data_validator.rb
 - lib/clusterkit/dimensionality.rb
 - lib/clusterkit/dimensionality/pca.rb
 - lib/clusterkit/dimensionality/svd.rb