clusterkit 0.1.0 → 0.1.1

data/README.md

@@ -1,4 +1,4 @@
-# ClusterKit
+<img src="/docs/assets/clusterkit-wide.png" alt="clusterkit" height="80px">
 
 A high-performance clustering and dimensionality reduction toolkit for Ruby, powered by best-in-class Rust implementations.
 
@@ -44,7 +44,7 @@ ClusterKit organizes its functionality into clear modules:
 
 - **`ClusterKit::Dimensionality`** - All dimensionality reduction algorithms
   - `ClusterKit::Dimensionality::UMAP` - UMAP implementation
-  - `ClusterKit::Dimensionality::PCA` - PCA implementation  
+  - `ClusterKit::Dimensionality::PCA` - PCA implementation
   - `ClusterKit::Dimensionality::SVD` - SVD implementation
 - **`ClusterKit::Clustering`** - All clustering algorithms
   - `ClusterKit::Clustering::KMeans` - K-means clustering
@@ -96,7 +96,7 @@ data = []
 3.times do |cluster|
   # Each cluster has a different center, well-separated
   center = Array.new(50) { rand * 0.1 + cluster * 2.0 }
-  
+
   # Add 33 points around each center with controlled noise
   33.times do
     point = center.map { |c| c + (rand - 0.5) * 0.3 }
@@ -329,6 +329,223 @@ probabilities = hdbscan.probabilities      # Cluster membership probabilities
 outlier_scores = hdbscan.outlier_scores   # Outlier scores for each point
 ```
 
+### HNSW - Fast Nearest Neighbor Search
+
+ClusterKit includes HNSW (Hierarchical Navigable Small World) for fast approximate nearest neighbor search, useful for building recommendation systems, similarity search, and as a building block for other algorithms.
+
+Copy and paste this **entire block** into IRB to try HNSW with real embeddings:
+
+```ruby
+require 'clusterkit'
+require 'candle'
+
+# Step 1: Initialize the embedding model
+puts "Loading embedding model..."
+embedding_model = Candle::EmbeddingModel.from_pretrained(
+  'sentence-transformers/all-MiniLM-L6-v2',
+  device: Candle::Device.best
+)
+puts "  ✓ Model loaded: #{embedding_model.model_id}"
+
+# Step 2: Create sample documents for semantic search
+documents = [
+  "The cat sat on the mat",
+  "Dogs are loyal pets that love their owners",
+  "Machine learning algorithms can classify text documents",
+  "Natural language processing helps computers understand human language",
+  "Ruby is a programming language known for its simplicity",
+  "Python is popular for data science and machine learning",
+  "The weather today is sunny and warm",
+  "Climate change affects global weather patterns",
+  "Artificial intelligence is transforming many industries",
+  "Deep learning models require large amounts of training data",
+  "Cats and dogs are common household pets",
+  "Software engineering requires problem-solving skills",
+  "The ocean contains many different species of fish",
+  "Marine biology studies life in aquatic environments",
+  "Cooking requires understanding of ingredients and techniques"
+]
+
+puts "\nGenerating embeddings for #{documents.size} documents..."
+
+# Step 3: Generate embeddings for all documents
+embeddings = documents.map do |doc|
+  embedding_model.embedding(doc).first.to_a
+end
+puts "  ✓ Generated embeddings: #{embeddings.first.count} dimensions each"
+
+# Step 4: Create HNSW index
+puts "\nBuilding HNSW search index..."
+index = ClusterKit::HNSW.new(
+  dim: embeddings.first.count,  # 384 dimensions for all-MiniLM-L6-v2
+  space: :euclidean,
+  m: 16,                       # Good balance of speed vs accuracy
+  ef_construction: 200,        # Build quality
+  max_elements: documents.size,
+  random_seed: 42             # For reproducible results
+)
+
+# Step 5: Add all documents to the index
+documents.each_with_index do |doc, i|
+  index.add_item(
+    embeddings[i],
+    label: "doc_#{i}",
+    metadata: {
+      'text' => doc,
+      'length' => doc.length,
+      'word_count' => doc.split.size
+    }
+  )
+end
+puts "  ✓ Added #{documents.size} documents to index"
+
+# Step 6: Perform semantic searches
+puts "\n" + "="*50
+puts "SEMANTIC SEARCH DEMO"
+puts "="*50
+
+queries = [
+  "pets and animals",
+  "computer programming",
+  "weather and environment"
+]
+
+queries.each do |query|
+  puts "\nQuery: '#{query}'"
+  puts "-" * 30
+
+  # Generate query embedding
+  query_embedding = embedding_model.embedding(query).first.to_a
+
+  # Search for similar documents
+  results = index.search_with_metadata(query_embedding, k: 3)
+
+  results.each_with_index do |result, i|
+    similarity = (1.0 - result[:distance]).round(3)  # Convert distance to similarity
+    text = result[:metadata]['text']
+    puts "  #{i+1}. [#{similarity}] #{text}"
+  end
+end
+
+# Step 7: Demonstrate advanced features
+puts "\n" + "="*50
+puts "ADVANCED FEATURES"
+puts "="*50
+
+# Show search quality adjustment
+puts "\nAdjusting search quality (ef parameter):"
+index.set_ef(50)   # Lower ef = faster but potentially less accurate
+fast_results = index.search(embeddings[0], k: 3)
+puts "  Fast search (ef=50): #{fast_results}"
+
+index.set_ef(200)  # Higher ef = slower but more accurate
+accurate_results = index.search(embeddings[0], k: 3)
+puts "  Accurate search (ef=200): #{accurate_results}"
+
+# Show batch operations
+puts "\nBatch search example:"
+query_embeddings = [embeddings[0], embeddings[5], embeddings[10]]
+batch_results = query_embeddings.map { |emb| index.search(emb, k: 2) }
+puts "  Found #{batch_results.size} result sets"
+
+# Save and load demonstration
+puts "\nSaving and loading index:"
+index.save('demo_index')
+puts "  ✓ Index saved to 'demo_index'"
+
+loaded_index = ClusterKit::HNSW.load('demo_index')
+test_results = loaded_index.search(embeddings[0], k: 2)
+puts "  ✓ Loaded index works: #{test_results}"
+
+puts "\n✅ HNSW demo complete!"
+puts "\nTry your own queries by running:"
+puts "query_embedding = embedding_model.embedding('your search query').first.to_a"
+puts "results = index.search_with_metadata(query_embedding, k: 5)"
+```
+
+#### When to Use HNSW
+
+HNSW is ideal for:
+- **Recommendation Systems**: Find similar items/users quickly
+- **Semantic Search**: Find documents with similar embeddings
+- **Duplicate Detection**: Identify near-duplicate content
+- **Clustering Support**: As a fast neighbor graph for HDBSCAN
+- **Real-time Applications**: When you need sub-millisecond search times
+
+#### Configuration Guidelines
+
+```ruby
+# High recall (>0.95) - Best quality, slower
+ClusterKit::HNSW.new(
+  dim: dim,
+  m: 32,
+  ef_construction: 400
+).tap { |idx| idx.set_ef(100) }
+
+# Balanced (>0.90 recall) - Good quality, fast
+ClusterKit::HNSW.new(
+  dim: dim,
+  m: 16,
+  ef_construction: 200
+).tap { |idx| idx.set_ef(50) }
+
+# Speed optimized (>0.85 recall) - Fastest, acceptable quality
+ClusterKit::HNSW.new(
+  dim: dim,
+  m: 8,
+  ef_construction: 100
+).tap { |idx| idx.set_ef(20) }
+```
+
+#### Important Notes
+
+1. **Memory Usage**: HNSW keeps the entire index in memory. Estimate: `(num_items * (dim * 4 + m * 16))` bytes
+2. **Distance Metrics**: Currently only Euclidean distance is fully supported
+3. **Loading Behavior**: Due to Rust lifetime constraints, loading an index creates a small memory leak (the index metadata persists until program exit). This is typically negligible for most applications.
+4. **Build Time**: Index construction is O(N * log(N)). For large datasets (>1M items), consider building offline
+
+#### Example: Semantic Search System
+
+```ruby
+# Build a simple semantic search system
+documents = load_documents()
+embeddings = generate_embeddings(documents)  # Use red-candle or similar
+
+# Build search index
+search_index = ClusterKit::HNSW.new(
+  dim: embeddings.first.size,
+  m: 16,
+  ef_construction: 200,
+  max_elements: documents.size
+)
+
+# Add all documents
+documents.each_with_index do |doc, i|
+  search_index.add_item(
+    embeddings[i],
+    label: i,
+    metadata: { title: doc[:title], url: doc[:url] }
+  )
+end
+
+# Search function
+def search(query, index, k: 10)
+  query_embedding = generate_embedding(query)
+  results = index.search_with_metadata(query_embedding, k: k)
+
+  results.map do |result|
+    {
+      title: result[:metadata]['title'],
+      url: result[:metadata]['url'],
+      similarity: 1.0 - result[:distance]  # Convert distance to similarity
+    }
+  end
+end
+
+# Save for later use
+search_index.save('document_index')
+```
+
 ### Visualization
 
 ClusterKit includes a built-in visualization tool:
@@ -350,6 +567,9 @@ This creates an interactive HTML file with:
 - Performance metrics
 - Interactive Plotly.js charts
 
+<img src="/docs/assets/visualization.png" alt="rake clusterkit:visualize">
+
+
 ## Choosing the Right Algorithm
 
 ### Dimensionality Reduction
@@ -454,7 +674,7 @@ This error occurs when UMAP cannot find enough neighbors for some points. Soluti
    ```ruby
    # Bad: Pure random data with no structure
    data = Array.new(100) { Array.new(50) { rand } }
-   
+
    # Good: Data with clusters or patterns (see Quick Start example)
    # Create clusters with centers and add points around them
    ```
@@ -500,7 +720,7 @@ COVERAGE=true bundle exec rspec
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/cpetersen/clusterkit.
+Bug reports and pull requests are welcome on GitHub at https://github.com/scientist-labs/clusterkit.
 
 ## License
 
@@ -515,10 +735,10 @@ If you use ClusterKit in your research, please cite:
   author = {Chris Petersen},
   title = {ClusterKit: High-Performance Clustering and Dimensionality Reduction for Ruby},
   year = {2024},
-  url = {https://github.com/cpetersen/clusterkit}
+  url = {https://github.com/scientist-labs/clusterkit}
 }
 ```
 
 And please also cite the underlying libraries:
 - [annembed](https://github.com/jean-pierreBoth/annembed) for dimensionality reduction algorithms
-- [hdbscan](https://github.com/petabi/hdbscan) for HDBSCAN clustering
+- [hdbscan](https://github.com/petabi/hdbscan) for HDBSCAN clustering

data/docs/KNOWN_ISSUES.md

@@ -14,7 +14,7 @@ This gem has three main categories of limitations:
 
 **Reason**: UMAP needs sufficient data to construct a meaningful manifold approximation. With fewer than 10 points, the algorithm cannot create a reliable graph structure.
 
-**Workaround**: 
+**Workaround**:
 - Use PCA for datasets with fewer than 10 points
 - The `transform` method can handle smaller datasets once the model is fitted on adequate training data
 
@@ -30,12 +30,12 @@ This gem has three main categories of limitations:
 
 **Previous Issue**: The box_size assertion would panic and crash the Ruby process.
 
-**Current Status**: **FIXED** in `cpetersen/annembed:fix-box-size-panic` branch
+**Current Status**: **FIXED** in `scientist-labs/annembed:fix-box-size-panic` branch
 - The `"assertion failed: (*f).abs() <= box_size"` panic has been converted to a catchable error
 - Extreme value ranges are now handled gracefully through normalization
 - NaN/Infinite values are detected and reported with clear error messages
 
-**Remaining Uncatchable Errors**: 
+**Remaining Uncatchable Errors**:
 - Array bounds violations (accessing out-of-bounds indices)
 - Some `.unwrap()` calls on `None` or `Err` values
 - These are much less common in normal usage
@@ -98,7 +98,7 @@ def safe_umap_transform(data, options = {})
   # Save data to temporary file before processing
   temp_file = "temp_umap_data_#{Time.now.to_i}.json"
   File.write(temp_file, JSON.dump(data))
-  
+
   begin
     umap = ClusterKit::Dimensionality::UMAP.new(**options)
     result = umap.fit_transform(data)
@@ -127,4 +127,4 @@ def reduce_dimensions(data, n_components: 2)
     pca.fit_transform(data)
   end
 end
-```
+```

data/docs/RUST_ERROR_HANDLING.md

@@ -37,11 +37,11 @@ These use Rust's `assert!` or `panic!` macros and CANNOT be caught. They will cr
 
 | Error | Source | Location | Trigger Condition |
 |-------|--------|----------|-------------------|
-| ~~Box size assertion~~ | ~~annembed~~ | ~~`set_data_box`~~ | **FIXED in cpetersen/annembed:fix-box-size-panic** |
+| ~~Box size assertion~~ | ~~annembed~~ | ~~`set_data_box`~~ | **FIXED in scientist-labs/annembed:fix-box-size-panic** |
 | Array bounds | Various | Index operations | Accessing out-of-bounds indices |
 | Unwrap failures | Various | `.unwrap()` calls | Unwrapping `None` or `Err` |
 
-**Update (2025-08-19):** The box size assertion has been fixed in the `fix-box-size-panic` branch of cpetersen/annembed. It now returns a proper `Result<(), anyhow::Error>` that can be caught and handled gracefully:
+**Update (2025-08-19):** The box size assertion has been fixed in the `fix-box-size-panic` branch of scientist-labs/annembed. It now returns a proper `Result<(), anyhow::Error>` that can be caught and handled gracefully:
 
 ```rust
 // Previously (would panic):
@@ -96,13 +96,13 @@ when /isolated point/i
 
 **Previous Issue:** Would panic and crash the Ruby process
 
-**Current Status:** Fixed in `cpetersen/annembed:fix-box-size-panic` branch
-- Now returns a catchable `anyhow::Error` 
+**Current Status:** Fixed in `scientist-labs/annembed:fix-box-size-panic` branch
+- Now returns a catchable `anyhow::Error`
 - Detects NaN/Infinite values during normalization
 - Handles constant data (max_max = 0) gracefully
 - Extreme value ranges are normalized successfully
 
-**User-visible behavior:** 
+**User-visible behavior:**
 - Previously: Ruby process would crash with assertion failure
 - Now: Raises a catchable Ruby exception with helpful error message
 
@@ -161,4 +161,4 @@ when /isolated point/i
 
 The test suite mocks Rust errors to verify our error handling logic works correctly. However, actual panic conditions cannot be tested without crashing the test process.
 
-See `spec/clusterkit/error_handling_spec.rb` for error handling tests.
+See `spec/clusterkit/error_handling_spec.rb` for error handling tests.

data/ext/clusterkit/Cargo.toml

@@ -7,9 +7,9 @@ edition = "2021"
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = { version = "0.6", features = ["embed"] }
-annembed = { git = "https://github.com/cpetersen/annembed", tag = "clusterkit-0.1.0" }
-hnsw_rs = { git = "https://github.com/cpetersen/hnswlib-rs", tag = "clusterkit-0.1.0" }
+magnus = { version = "0.8", features = ["embed"] }
+annembed = { git = "https://github.com/scientist-labs/annembed", tag = "clusterkit-0.1.1" }
+hnsw_rs = { git = "https://github.com/scientist-labs/hnswlib-rs", tag = "clusterkit-0.1.0" }
 hdbscan = "0.11"
 ndarray = "0.16"
 num-traits = "0.2"
@@ -22,4 +22,5 @@ rand = "0.8"
 default = ["openblas-static"]
 openblas-static = ["annembed/openblas-static"]
 openblas-system = ["annembed/openblas-system"]
-intel-mkl-static = ["annembed/intel-mkl-static"]
+intel-mkl-static = ["annembed/intel-mkl-static"]
+macos-accelerate = ["annembed/macos-accelerate"]

data/ext/clusterkit/extconf.rb

@@ -1,4 +1,12 @@
 require "mkmf"
 require "rb_sys/mkmf"
 
-create_rust_makefile("clusterkit/clusterkit")
+create_rust_makefile("clusterkit/clusterkit") do |r|
+  if ENV["CLUSTERKIT_FEATURES"]
+    r.extra_cargo_args += ["--no-default-features"]
+    r.features = ENV["CLUSTERKIT_FEATURES"].split(",")
+  elsif RUBY_PLATFORM =~ /darwin/
+    r.extra_cargo_args += ["--no-default-features"]
+    r.features = ["macos-accelerate"]
+  end
+end

data/ext/clusterkit/src/clustering/hdbscan_wrapper.rs

@@ -1,5 +1,6 @@
-use magnus::{function, prelude::*, Error, Value, RArray, RHash, Integer, TryConvert};
+use magnus::{function, prelude::*, Error, Value, RHash, Ruby};
 use hdbscan::{Hdbscan, HdbscanHyperParams};
+use crate::utils::ruby_array_to_vec_vec_f64;
 
 /// Perform HDBSCAN clustering
 /// Returns a hash with labels and basic statistics
@@ -9,98 +10,62 @@ pub fn hdbscan_fit(
     min_cluster_size: usize,
     metric: String,
 ) -> Result<RHash, Error> {
-    // Convert Ruby array to ndarray
-    let rarray: RArray = TryConvert::try_convert(data)?;
-    let n_samples = rarray.len();
-    
-    if n_samples == 0 {
-        return Err(Error::new(
-            magnus::exception::arg_error(),
-            "Data cannot be empty",
-        ));
-    }
-    
-    // Get dimensions
-    let first_row: RArray = rarray.entry::<RArray>(0)?;
-    let n_features = first_row.len();
-    
-    // Convert to Vec<Vec<f64>> format expected by hdbscan crate
-    let mut data_vec: Vec<Vec<f64>> = Vec::with_capacity(n_samples);
-    for i in 0..n_samples {
-        let row: RArray = rarray.entry(i as isize)?;
-        let mut row_vec: Vec<f64> = Vec::with_capacity(n_features);
-        for j in 0..n_features {
-            let val: f64 = row.entry(j as isize)?;
-            row_vec.push(val);
-        }
-        data_vec.push(row_vec);
-    }
-    
-    // Note: hdbscan crate doesn't support custom metrics directly
-    // We'll use the default Euclidean distance for now
+    let ruby = Ruby::get().unwrap();
+
+    // Convert Ruby array to Vec<Vec<f64>> using shared helper
+    let data_vec = ruby_array_to_vec_vec_f64(data)?;
+    let n_samples = data_vec.len();
+
     if metric != "euclidean" && metric != "l2" {
         eprintln!("Warning: Current hdbscan version only supports Euclidean distance. Using Euclidean.");
     }
-    
+
     // Adjust parameters to avoid index out of bounds errors
-    // The hdbscan crate has issues when min_samples >= n_samples
     let adjusted_min_samples = min_samples.min(n_samples.saturating_sub(1)).max(1);
     let adjusted_min_cluster_size = min_cluster_size.min(n_samples).max(2);
-    
+
     // Create hyperparameters
     let hyper_params = HdbscanHyperParams::builder()
         .min_cluster_size(adjusted_min_cluster_size)
         .min_samples(adjusted_min_samples)
         .build();
-    
+
     // Create HDBSCAN instance and run clustering
     let clusterer = Hdbscan::new(&data_vec, hyper_params);
-    
-    // Run the clustering algorithm - cluster() returns Result<Vec<i32>, HdbscanError>
+
     let labels = clusterer.cluster().map_err(|e| {
         Error::new(
-            magnus::exception::runtime_error(),
+            ruby.exception_runtime_error(),
             format!("HDBSCAN clustering failed: {:?}", e)
         )
     })?;
-    
+
     // Convert results to Ruby types
-    let ruby = magnus::Ruby::get().unwrap();
-    let result = RHash::new();
-    
-    // Convert labels (i32 to Ruby Integer, -1 for noise)
-    let labels_array = RArray::new();
+    let result = ruby.hash_new();
+
+    let labels_array = ruby.ary_new();
     for &label in labels.iter() {
-        labels_array.push(Integer::from_value(
-            ruby.eval(&format!("{}", label)).unwrap()
-        ).unwrap())?;
+        labels_array.push(ruby.integer_from_i64(label as i64))?;
     }
     result.aset("labels", labels_array)?;
-    
-    // For now, we'll create dummy probabilities and outlier scores
-    // since the basic hdbscan crate doesn't provide these
-    // In the future, we could calculate these ourselves or use a more advanced implementation
-    
-    // Create probabilities array (all 1.0 for clustered points, 0.0 for noise)
-    let probs_array = RArray::new();
+
+    let probs_array = ruby.ary_new();
     for &label in labels.iter() {
         let prob = if label == -1 { 0.0 } else { 1.0 };
         probs_array.push(prob)?;
     }
     result.aset("probabilities", probs_array)?;
-    
-    // Create outlier scores array (0.0 for clustered points, 1.0 for noise)
-    let outlier_array = RArray::new();
+
+    let outlier_array = ruby.ary_new();
     for &label in labels.iter() {
         let score = if label == -1 { 1.0 } else { 0.0 };
         outlier_array.push(score)?;
     }
     result.aset("outlier_scores", outlier_array)?;
-    
-    // Create empty cluster persistence hash for now
-    let persistence_hash = RHash::new();
+
+    let persistence_hash = ruby.hash_new();
     result.aset("cluster_persistence", persistence_hash)?;
-    
+
     Ok(result)
 }
 
@@ -110,6 +75,6 @@ pub fn init(clustering_module: &magnus::RModule) -> Result<(), Error> {
         "hdbscan_rust",
         function!(hdbscan_fit, 4),
     )?;
-    
+
     Ok(())
-}
+}