clusterkit 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4df777f3b01fea2f411cde6233b2abd5c1ab01903117ee8f377a1ae67bb6510b
-  data.tar.gz: cbf92ed9e86d14a3959b81f348e69f4eb2fa3bcd92d86149c4872ba9a197f7c6
+  metadata.gz: ebad40c2aac3fa3569357eedf740336a3d463ecc5c038d2771d3fd266d414b1e
+  data.tar.gz: 0ab2851e0adab583567460e469d4073e755514854e57bca464d5a835c1534bea
 SHA512:
-  metadata.gz: bae2d9fac87d5cb27458ae8ace71edffe754216ea5b9af088c98fecfc4489dc3f572ca45e7d926ed658516dac2ab18b6bcf04fc138daf48a01d17569f5698521
-  data.tar.gz: 560c59116a016c60dae79ed3817158005551c8a5e6a1d7d7b47a67cdb87a0c566d07db7771bcad0507e21111d380b1b4795bd85a5e75d5e00ce1229a181704b9
+  metadata.gz: 16412a2db10bf55593c778c7a02813b8db9652e39479d17fa3cd10da1e87298d5d95f4afccd8f5da10bb89bc950b314faeb9a8f0149ecd2dd3c3305121c1b5b2
+  data.tar.gz: edf73f2d0ce8f73441c07975a5cd4da31faaece9b216d908a743b9c1bd3f2d8e213d1b85a7e82f2f397c60f8d2fd2c4a0109a5a337963e008ecf069b3fa40266

data/README.md

@@ -1,4 +1,4 @@
-# ClusterKit
+<img src="/docs/assets/clusterkit-wide.png" alt="clusterkit" height="80px">
 
 A high-performance clustering and dimensionality reduction toolkit for Ruby, powered by best-in-class Rust implementations.
 
@@ -44,7 +44,7 @@ ClusterKit organizes its functionality into clear modules:
 
 - **`ClusterKit::Dimensionality`** - All dimensionality reduction algorithms
   - `ClusterKit::Dimensionality::UMAP` - UMAP implementation
-  - `ClusterKit::Dimensionality::PCA` - PCA implementation  
+  - `ClusterKit::Dimensionality::PCA` - PCA implementation
   - `ClusterKit::Dimensionality::SVD` - SVD implementation
 - **`ClusterKit::Clustering`** - All clustering algorithms
   - `ClusterKit::Clustering::KMeans` - K-means clustering
@@ -96,7 +96,7 @@ data = []
 3.times do |cluster|
   # Each cluster has a different center, well-separated
   center = Array.new(50) { rand * 0.1 + cluster * 2.0 }
-  
+
   # Add 33 points around each center with controlled noise
   33.times do
     point = center.map { |c| c + (rand - 0.5) * 0.3 }
@@ -329,6 +329,223 @@ probabilities = hdbscan.probabilities      # Cluster membership probabilities
 outlier_scores = hdbscan.outlier_scores   # Outlier scores for each point
 ```
 
+### HNSW - Fast Nearest Neighbor Search
+
+ClusterKit includes HNSW (Hierarchical Navigable Small World) for fast approximate nearest neighbor search, useful for building recommendation systems, similarity search, and as a building block for other algorithms.
+
+Copy and paste this **entire block** into IRB to try HNSW with real embeddings:
+
+```ruby
+require 'clusterkit'
+require 'candle'
+
+# Step 1: Initialize the embedding model
+puts "Loading embedding model..."
+embedding_model = Candle::EmbeddingModel.from_pretrained(
+  'sentence-transformers/all-MiniLM-L6-v2',
+  device: Candle::Device.best
+)
+puts "  ✓ Model loaded: #{embedding_model.model_id}"
+
+# Step 2: Create sample documents for semantic search
+documents = [
+  "The cat sat on the mat",
+  "Dogs are loyal pets that love their owners",
+  "Machine learning algorithms can classify text documents",
+  "Natural language processing helps computers understand human language",
+  "Ruby is a programming language known for its simplicity",
+  "Python is popular for data science and machine learning",
+  "The weather today is sunny and warm",
+  "Climate change affects global weather patterns",
+  "Artificial intelligence is transforming many industries",
+  "Deep learning models require large amounts of training data",
+  "Cats and dogs are common household pets",
+  "Software engineering requires problem-solving skills",
+  "The ocean contains many different species of fish",
+  "Marine biology studies life in aquatic environments",
+  "Cooking requires understanding of ingredients and techniques"
+]
+
+puts "\nGenerating embeddings for #{documents.size} documents..."
+
+# Step 3: Generate embeddings for all documents
+embeddings = documents.map do |doc|
+  embedding_model.embedding(doc).first.to_a
+end
+puts "  ✓ Generated embeddings: #{embeddings.first.count} dimensions each"
+
+# Step 4: Create HNSW index
+puts "\nBuilding HNSW search index..."
+index = ClusterKit::HNSW.new(
+  dim: embeddings.first.count,  # 384 dimensions for all-MiniLM-L6-v2
+  space: :euclidean,
+  m: 16,                       # Good balance of speed vs accuracy
+  ef_construction: 200,        # Build quality
+  max_elements: documents.size,
+  random_seed: 42             # For reproducible results
+)
+
+# Step 5: Add all documents to the index
+documents.each_with_index do |doc, i|
+  index.add_item(
+    embeddings[i],
+    label: "doc_#{i}",
+    metadata: {
+      'text' => doc,
+      'length' => doc.length,
+      'word_count' => doc.split.size
+    }
+  )
+end
+puts "  ✓ Added #{documents.size} documents to index"
+
+# Step 6: Perform semantic searches
+puts "\n" + "="*50
+puts "SEMANTIC SEARCH DEMO"
+puts "="*50
+
+queries = [
+  "pets and animals",
+  "computer programming",
+  "weather and environment"
+]
+
+queries.each do |query|
+  puts "\nQuery: '#{query}'"
+  puts "-" * 30
+
+  # Generate query embedding
+  query_embedding = embedding_model.embedding(query).first.to_a
+
+  # Search for similar documents
+  results = index.search_with_metadata(query_embedding, k: 3)
+
+  results.each_with_index do |result, i|
+    similarity = (1.0 - result[:distance]).round(3)  # Convert distance to similarity
+    text = result[:metadata]['text']
+    puts "  #{i+1}. [#{similarity}] #{text}"
+  end
+end
+
+# Step 7: Demonstrate advanced features
+puts "\n" + "="*50
+puts "ADVANCED FEATURES"
+puts "="*50
+
+# Show search quality adjustment
+puts "\nAdjusting search quality (ef parameter):"
+index.set_ef(50)   # Lower ef = faster but potentially less accurate
+fast_results = index.search(embeddings[0], k: 3)
+puts "  Fast search (ef=50): #{fast_results}"
+
+index.set_ef(200)  # Higher ef = slower but more accurate
+accurate_results = index.search(embeddings[0], k: 3)
+puts "  Accurate search (ef=200): #{accurate_results}"
+
+# Show batch operations
+puts "\nBatch search example:"
+query_embeddings = [embeddings[0], embeddings[5], embeddings[10]]
+batch_results = query_embeddings.map { |emb| index.search(emb, k: 2) }
+puts "  Found #{batch_results.size} result sets"
+
+# Save and load demonstration
+puts "\nSaving and loading index:"
+index.save('demo_index')
+puts "  ✓ Index saved to 'demo_index'"
+
+loaded_index = ClusterKit::HNSW.load('demo_index')
+test_results = loaded_index.search(embeddings[0], k: 2)
+puts "  ✓ Loaded index works: #{test_results}"
+
+puts "\n✅ HNSW demo complete!"
+puts "\nTry your own queries by running:"
+puts "query_embedding = embedding_model.embedding('your search query').first.to_a"
+puts "results = index.search_with_metadata(query_embedding, k: 5)"
+```
+
+#### When to Use HNSW
+
+HNSW is ideal for:
+- **Recommendation Systems**: Find similar items/users quickly
+- **Semantic Search**: Find documents with similar embeddings
+- **Duplicate Detection**: Identify near-duplicate content
+- **Clustering Support**: As a fast neighbor graph for HDBSCAN
+- **Real-time Applications**: When you need sub-millisecond search times
+
+#### Configuration Guidelines
+
+```ruby
+# High recall (>0.95) - Best quality, slower
+ClusterKit::HNSW.new(
+  dim: dim,
+  m: 32,
+  ef_construction: 400
+).tap { |idx| idx.set_ef(100) }
+
+# Balanced (>0.90 recall) - Good quality, fast
+ClusterKit::HNSW.new(
+  dim: dim,
+  m: 16,
+  ef_construction: 200
+).tap { |idx| idx.set_ef(50) }
+
+# Speed optimized (>0.85 recall) - Fastest, acceptable quality
+ClusterKit::HNSW.new(
+  dim: dim,
+  m: 8,
+  ef_construction: 100
+).tap { |idx| idx.set_ef(20) }
+```
+
+#### Important Notes
+
+1. **Memory Usage**: HNSW keeps the entire index in memory. Estimate: `(num_items * (dim * 4 + m * 16))` bytes
+2. **Distance Metrics**: Currently only Euclidean distance is fully supported
+3. **Loading Behavior**: Due to Rust lifetime constraints, loading an index creates a small memory leak (the index metadata persists until program exit). This is typically negligible for most applications.
+4. **Build Time**: Index construction is O(N * log(N)). For large datasets (>1M items), consider building offline
+
+#### Example: Semantic Search System
+
+```ruby
+# Build a simple semantic search system
+documents = load_documents()
+embeddings = generate_embeddings(documents)  # Use red-candle or similar
+
+# Build search index
+search_index = ClusterKit::HNSW.new(
+  dim: embeddings.first.size,
+  m: 16,
+  ef_construction: 200,
+  max_elements: documents.size
+)
+
+# Add all documents
+documents.each_with_index do |doc, i|
+  search_index.add_item(
+    embeddings[i],
+    label: i,
+    metadata: { title: doc[:title], url: doc[:url] }
+  )
+end
+
+# Search function
+def search(query, index, k: 10)
+  query_embedding = generate_embedding(query)
+  results = index.search_with_metadata(query_embedding, k: k)
+
+  results.map do |result|
+    {
+      title: result[:metadata]['title'],
+      url: result[:metadata]['url'],
+      similarity: 1.0 - result[:distance]  # Convert distance to similarity
+    }
+  end
+end
+
+# Save for later use
+search_index.save('document_index')
+```
+
 ### Visualization
 
 ClusterKit includes a built-in visualization tool:
@@ -350,6 +567,9 @@ This creates an interactive HTML file with:
 - Performance metrics
 - Interactive Plotly.js charts
 
+<img src="/docs/assets/visualization.png" alt="rake clusterkit:visualize">
+
+
 ## Choosing the Right Algorithm
 
 ### Dimensionality Reduction
@@ -454,7 +674,7 @@ This error occurs when UMAP cannot find enough neighbors for some points. Soluti
    ```ruby
    # Bad: Pure random data with no structure
    data = Array.new(100) { Array.new(50) { rand } }
-   
+
    # Good: Data with clusters or patterns (see Quick Start example)
    # Create clusters with centers and add points around them
    ```
@@ -521,4 +741,4 @@ If you use ClusterKit in your research, please cite:
 
 And please also cite the underlying libraries:
 - [annembed](https://github.com/jean-pierreBoth/annembed) for dimensionality reduction algorithms
-- [hdbscan](https://github.com/petabi/hdbscan) for HDBSCAN clustering
+- [hdbscan](https://github.com/petabi/hdbscan) for HDBSCAN clustering

data/ext/clusterkit/src/hnsw.rs

@@ -0,0 +1,613 @@
+use magnus::{
+    class, exception, function, method, prelude::*, 
+    Error, Float, Integer, RArray, RHash, RString, Symbol, Value, value, TryConvert, r_hash::ForEach
+};
+use hnsw_rs::prelude::*;
+use hnsw_rs::hnswio::HnswIo;
+// use ndarray::Array1; // Not used currently
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use serde::{Serialize, Deserialize};
+use std::fs::File;
+
+// Store metadata alongside vectors
+#[derive(Clone, Debug, Serialize, Deserialize)]
+struct ItemMetadata {
+    label: String,
+    metadata: Option<HashMap<String, String>>,
+}
+
+// Main HNSW wrapper struct
+#[magnus::wrap(class = "ClusterKit::HNSW", free_immediately, size)]
+pub struct HnswIndex {
+    hnsw: Arc<Mutex<Hnsw<'static, f32, DistL2>>>,
+    dim: usize,
+    space: DistanceType,
+    metadata_store: Arc<Mutex<HashMap<usize, ItemMetadata>>>,
+    current_id: Arc<Mutex<usize>>,
+    label_to_id: Arc<Mutex<HashMap<String, usize>>>,
+    ef_search: Arc<Mutex<usize>>,
+}
+
+#[derive(Clone, Copy)]
+#[allow(dead_code)]  // These variants will be implemented in the future
+enum DistanceType {
+    Euclidean,
+    Cosine,
+    InnerProduct,
+}
+
+impl HnswIndex {
+    // Initialize a new HNSW index
+    pub fn new(kwargs: RHash) -> Result<Self, Error> {
+        // Parse arguments
+        let dim_opt: Option<Value> = kwargs.delete(Symbol::new("dim"))?;
+        let dim_value = dim_opt.ok_or_else(|| Error::new(exception::arg_error(), "dim is required"))?;
+        let dim: usize = TryConvert::try_convert(dim_value)
+            .map_err(|_| Error::new(exception::arg_error(), "dim must be an integer"))?;
+        
+        // Validate dimension
+        if dim == 0 {
+            return Err(Error::new(exception::arg_error(), "dim must be a positive integer (got 0)"));
+        }
+        
+        let space: String = if let Some(v) = kwargs.delete(Symbol::new("space"))? {
+            // Convert Ruby symbol to string properly
+            if let Ok(sym) = Symbol::try_convert(v) {
+                sym.name()?.to_string()
+            } else if let Ok(s) = String::try_convert(v) {
+                s
+            } else {
+                return Err(Error::new(
+                    exception::type_error(),
+                    "space must be a string or symbol"
+                ));
+            }
+        } else {
+            "euclidean".to_string()
+        };
+        
+        let max_elements: usize = if let Some(v) = kwargs.delete(Symbol::new("max_elements"))? {
+            TryConvert::try_convert(v).unwrap_or(10_000)
+        } else {
+            10_000
+        };
+        
+        let m: usize = if let Some(v) = kwargs.delete(Symbol::new("M"))? {
+            TryConvert::try_convert(v).unwrap_or(16)
+        } else {
+            16
+        };
+        
+        let ef_construction: usize = if let Some(v) = kwargs.delete(Symbol::new("ef_construction"))? {
+            TryConvert::try_convert(v).unwrap_or(200)
+        } else {
+            200
+        };
+        
+        let random_seed: Option<u64> = if let Some(v) = kwargs.delete(Symbol::new("random_seed"))? {
+            TryConvert::try_convert(v).ok()
+        } else {
+            None
+        };
+        
+        // Validate and convert space parameter
+        // For now, only support Euclidean distance
+        let distance_type = match space.as_str() {
+            "euclidean" => DistanceType::Euclidean,
+            "cosine" => {
+                return Err(Error::new(
+                    exception::runtime_error(),
+                    "Cosine distance is not yet implemented, please use :euclidean"
+                ));
+            },
+            "inner_product" => {
+                return Err(Error::new(
+                    exception::runtime_error(),
+                    "Inner product distance is not yet implemented, please use :euclidean"
+                ));
+            },
+            _ => return Err(Error::new(
+                exception::arg_error(),
+                format!("space must be :euclidean, :cosine, or :inner_product (got: {})", space)
+            )),
+        };
+        
+        // Create HNSW instance with Euclidean distance
+        let hnsw = if let Some(seed) = random_seed {
+            Hnsw::<f32, DistL2>::new_with_seed(m, max_elements, 16, ef_construction, DistL2, seed)
+        } else {
+            Hnsw::<f32, DistL2>::new(m, max_elements, 16, ef_construction, DistL2)
+        };
+        
+        Ok(Self {
+            hnsw: Arc::new(Mutex::new(hnsw)),
+            dim,
+            space: distance_type,
+            metadata_store: Arc::new(Mutex::new(HashMap::new())),
+            current_id: Arc::new(Mutex::new(0)),
+            label_to_id: Arc::new(Mutex::new(HashMap::new())),
+            ef_search: Arc::new(Mutex::new(ef_construction)),
+        })
+    }
+    
+    // Add a single item to the index
+    pub fn add_item(&self, vector: RArray, kwargs: RHash) -> Result<Value, Error> {
+        // Parse vector
+        let vec_data = parse_vector(vector, self.dim)?;
+        
+        // Get or generate label
+        let label: String = if let Some(v) = kwargs.delete(Symbol::new("label"))? {
+            TryConvert::try_convert(v).unwrap_or_else(|_| {
+                let mut id = self.current_id.lock().unwrap();
+                let label = id.to_string();
+                *id += 1;
+                label
+            })
+        } else {
+            let mut id = self.current_id.lock().unwrap();
+            let label = id.to_string();
+            *id += 1;
+            label
+        };
+        
+        // Get metadata if provided
+        let metadata: Option<HashMap<String, String>> = if let Some(v) = kwargs.delete(Symbol::new("metadata"))? {
+            Some(parse_metadata(v)?)
+        } else {
+            None
+        };
+        
+        // Get internal ID for this item
+        let internal_id = {
+            let mut label_map = self.label_to_id.lock().unwrap();
+            let mut current_id = self.current_id.lock().unwrap();
+            
+            if label_map.contains_key(&label) {
+                return Err(Error::new(
+                    exception::arg_error(),
+                    format!("Label '{}' already exists in index", label)
+                ));
+            }
+            
+            let id = *current_id;
+            label_map.insert(label.clone(), id);
+            *current_id += 1;
+            id
+        };
+        
+        // Store metadata
+        {
+            let mut metadata_store = self.metadata_store.lock().unwrap();
+            metadata_store.insert(internal_id, ItemMetadata {
+                label: label.clone(),
+                metadata,
+            });
+        }
+        
+        // Add to HNSW
+        {
+            let hnsw = self.hnsw.lock().unwrap();
+            hnsw.insert((&vec_data, internal_id));
+        }
+        
+        Ok(value::qnil().as_value())
+    }
+    
+    // Add multiple items in batch
+    pub fn add_batch(&self, vectors: RArray, kwargs: RHash) -> Result<Value, Error> {
+        let parallel: bool = if let Some(v) = kwargs.delete(Symbol::new("parallel"))? {
+            TryConvert::try_convert(v).unwrap_or(true)
+        } else {
+            true
+        };
+        
+        let labels: Option<RArray> = if let Some(v) = kwargs.delete(Symbol::new("labels"))? {
+            TryConvert::try_convert(v).ok()
+        } else {
+            None
+        };
+        
+        // Parse all vectors
+        let mut data_points: Vec<(Vec<f32>, usize)> = Vec::new();
+        let mut metadata_entries: Vec<(usize, ItemMetadata)> = Vec::new();
+        
+        for (i, vector) in vectors.each().enumerate() {
+            let vector: RArray = TryConvert::try_convert(vector?)?;
+            let vec_data = parse_vector(vector, self.dim)?;
+            
+            // Get or generate label
+            let label = if let Some(ref labels_array) = labels {
+                labels_array.entry::<String>(i as isize)?
+            } else {
+                let mut id = self.current_id.lock().unwrap();
+                let label = id.to_string();
+                *id += 1;
+                label
+            };
+            
+            // Get internal ID
+            let internal_id = {
+                let mut label_map = self.label_to_id.lock().unwrap();
+                let mut current_id = self.current_id.lock().unwrap();
+                
+                if label_map.contains_key(&label) {
+                    return Err(Error::new(
+                        exception::arg_error(),
+                        format!("Label '{}' already exists in index", label)
+                    ));
+                }
+                
+                let id = *current_id;
+                label_map.insert(label.clone(), id);
+                *current_id += 1;
+                id
+            };
+            
+            data_points.push((vec_data, internal_id));
+            metadata_entries.push((internal_id, ItemMetadata {
+                label,
+                metadata: None,
+            }));
+        }
+        
+        // Store metadata
+        {
+            let mut metadata_store = self.metadata_store.lock().unwrap();
+            for (id, metadata) in metadata_entries {
+                metadata_store.insert(id, metadata);
+            }
+        }
+        
+        // Insert into HNSW
+        {
+            let hnsw = self.hnsw.lock().unwrap();
+            if parallel {
+                let data_refs: Vec<(&Vec<f32>, usize)> = data_points.iter().map(|(v, id)| (v, *id)).collect();
+                hnsw.parallel_insert(&data_refs);
+            } else {
+                for (vec, id) in data_points {
+                    hnsw.insert((&vec, id));
+                }
+            }
+        }
+        
+        Ok(value::qnil().as_value())
+    }
+    
+    // Search for k nearest neighbors
+    pub fn search(&self, query: RArray, kwargs: RHash) -> Result<Value, Error> {
+        let k: usize = if let Some(v) = kwargs.delete(Symbol::new("k"))? {
+            TryConvert::try_convert(v).unwrap_or(10)
+        } else {
+            10
+        };
+        
+        let include_distances: bool = if let Some(v) = kwargs.delete(Symbol::new("include_distances"))? {
+            TryConvert::try_convert(v).unwrap_or(false)
+        } else {
+            false
+        };
+        
+        // Parse query vector
+        let query_vec = parse_vector(query, self.dim)?;
+        
+        // Set search ef if provided
+        if let Some(v) = kwargs.delete(Symbol::new("ef"))? {
+            if let Ok(ef) = TryConvert::try_convert(v) as Result<usize, _> {
+                let mut ef_search = self.ef_search.lock().unwrap();
+                *ef_search = ef;
+            }
+        }
+        
+        // Perform search
+        let neighbors = {
+            let hnsw = self.hnsw.lock().unwrap();
+            let ef_search = self.ef_search.lock().unwrap();
+            hnsw.search(&query_vec, k, *ef_search)
+        };
+        
+        // Convert results
+        let metadata_store = self.metadata_store.lock().unwrap();
+        
+        let indices = RArray::new();
+        let distances = RArray::new();
+        
+        for neighbor in neighbors {
+            if let Some(metadata) = metadata_store.get(&neighbor.d_id) {
+                indices.push(RString::new(&metadata.label))?;
+                distances.push(Float::from_f64(neighbor.distance as f64))?;
+            }
+        }
+        
+        if include_distances {
+            let result = RArray::new();
+            result.push(indices)?;
+            result.push(distances)?;
+            Ok(result.as_value())
+        } else {
+            Ok(indices.as_value())
+        }
+    }
+    
+    // Search with metadata included
+    pub fn search_with_metadata(&self, query: RArray, kwargs: RHash) -> Result<Value, Error> {
+        let k: usize = if let Some(v) = kwargs.delete(Symbol::new("k"))? {
+            TryConvert::try_convert(v).unwrap_or(10)
+        } else {
+            10
+        };
+        
+        // Parse query vector
+        let query_vec = parse_vector(query, self.dim)?;
+        
+        // Perform search
+        let neighbors = {
+            let hnsw = self.hnsw.lock().unwrap();
+            let ef_search = self.ef_search.lock().unwrap();
+            hnsw.search(&query_vec, k, *ef_search)
+        };
+        
+        // Build results with metadata
+        let metadata_store = self.metadata_store.lock().unwrap();
+        let results = RArray::new();
+        
+        for neighbor in neighbors {
+            if let Some(item_metadata) = metadata_store.get(&neighbor.d_id) {
+                let result = RHash::new();
+                result.aset(Symbol::new("label"), RString::new(&item_metadata.label))?;
+                result.aset(Symbol::new("distance"), Float::from_f64(neighbor.distance as f64))?;
+                
+                let meta_hash = RHash::new();
+                if let Some(ref meta) = item_metadata.metadata {
+                    for (key, value) in meta {
+                        meta_hash.aset(RString::new(key), RString::new(value))?;
+                    }
+                }
+                result.aset(Symbol::new("metadata"), meta_hash)?;
+                
+                results.push(result)?;
+            }
+        }
+        
+        Ok(results.as_value())
+    }
+    
+    // Get current size of the index
+    pub fn size(&self) -> Result<usize, Error> {
+        let metadata_store = self.metadata_store.lock().unwrap();
+        Ok(metadata_store.len())
+    }
+    
+    // Check if index is empty
+    pub fn empty(&self) -> Result<bool, Error> {
+        Ok(self.size()? == 0)
+    }
+    
+    // Set the ef parameter for search
+    pub fn set_ef(&self, ef: usize) -> Result<Value, Error> {
+        let mut ef_search = self.ef_search.lock().unwrap();
+        *ef_search = ef;
+        Ok(value::qnil().as_value())
+    }
+    
+    // Get configuration
+    pub fn config(&self) -> Result<RHash, Error> {
+        let config = RHash::new();
+        config.aset(Symbol::new("dim"), Integer::from_i64(self.dim as i64))?;
+        
+        let space_str = match self.space {
+            DistanceType::Euclidean => "euclidean",
+            DistanceType::Cosine => "cosine",
+            DistanceType::InnerProduct => "inner_product",
+        };
+        config.aset(Symbol::new("space"), RString::new(space_str))?;
+        
+        let ef_search = self.ef_search.lock().unwrap();
+        config.aset(Symbol::new("ef"), Integer::from_i64(*ef_search as i64))?;
+        config.aset(Symbol::new("size"), Integer::from_i64(self.size()? as i64))?;
+        
+        Ok(config)
+    }
+    
+    // Get statistics about the index
+    pub fn stats(&self) -> Result<RHash, Error> {
+        let stats = RHash::new();
+        
+        stats.aset(Symbol::new("size"), Integer::from_i64(self.size()? as i64))?;
+        stats.aset(Symbol::new("dim"), Integer::from_i64(self.dim as i64))?;
+        
+        let ef_search = self.ef_search.lock().unwrap();
+        stats.aset(Symbol::new("ef_search"), Integer::from_i64(*ef_search as i64))?;
+        
+        // TODO: Add more statistics from HNSW structure
+        
+        Ok(stats)
+    }
+    
+    // Load index from file (class method)
+    pub fn load(path: RString) -> Result<Self, Error> {
+        let path_str = path.to_string()?;
+        
+        // Load metadata first to get dimensions and space
+        let metadata_path = format!("{}.metadata", path_str);
+        let metadata_file = File::open(&metadata_path)
+            .map_err(|e| Error::new(exception::runtime_error(), format!("Failed to open metadata file: {}", e)))?;
+        
+        let (
+            _metadata_store,
+            _label_to_id, 
+            _current_id,
+            _dim,
+            _space_str,
+        ): (
+            HashMap<usize, ItemMetadata>,
+            HashMap<String, usize>,
+            usize,
+            usize,
+            String,  // Changed from &str to String for deserialization
+        ) = bincode::deserialize_from(metadata_file)
+            .map_err(|e| Error::new(exception::runtime_error(), format!("Failed to load metadata: {}", e)))?;
+        
+        // Load HNSW structure
+        let hnsw_dir = format!("{}_hnsw_data", path_str);
+        let hnsw_path = std::path::Path::new(&hnsw_dir);
+        
+        // Create HnswIo and leak it to get 'static lifetime
+        // This is a memory leak, but necessary due to hnsw_rs lifetime constraints
+        // The memory will never be freed until the program exits
+        let hnswio = Box::new(HnswIo::new(hnsw_path, "hnsw"));
+        let hnswio_static: &'static mut HnswIo = Box::leak(hnswio);
+        
+        // Now we can load the HNSW with 'static lifetime
+        let hnsw: Hnsw<'static, f32, DistL2> = hnswio_static.load_hnsw()
+            .map_err(|e| Error::new(exception::runtime_error(), format!("Failed to load HNSW index: {}", e)))?;
+        
+        // Use the loaded metadata
+        let metadata_store = _metadata_store;
+        let label_to_id = _label_to_id;
+        let current_id = _current_id;
+        let dim = _dim;
+        let space = match _space_str.as_str() {
+            "euclidean" => DistanceType::Euclidean,
+            "cosine" => DistanceType::Cosine,
+            "inner_product" => DistanceType::InnerProduct,
+            _ => return Err(Error::new(exception::runtime_error(), "Unknown distance type in saved file")),
+        };
+        
+        // Use default ef_construction as ef_search
+        let ef_search = 200;
+        
+        Ok(Self {
+            hnsw: Arc::new(Mutex::new(hnsw)),
+            dim,
+            space,
+            metadata_store: Arc::new(Mutex::new(metadata_store)),
+            current_id: Arc::new(Mutex::new(current_id)),
+            label_to_id: Arc::new(Mutex::new(label_to_id)),
+            ef_search: Arc::new(Mutex::new(ef_search)),
+        })
+    }
+    
+    // Save index to file
+    pub fn save(&self, path: RString) -> Result<Value, Error> {
+        let path_str = path.to_string()?;
+        
+        // Create directory for HNSW structure
+        let hnsw_dir = format!("{}_hnsw_data", path_str);
+        std::fs::create_dir_all(&hnsw_dir)
+            .map_err(|e| Error::new(exception::runtime_error(), format!("Failed to create directory: {}", e)))?;
+        
+        // Save HNSW structure
+        {
+            let hnsw = self.hnsw.lock().unwrap();
+            hnsw.file_dump(&std::path::Path::new(&hnsw_dir), "hnsw")
+                .map_err(|e| Error::new(exception::runtime_error(), format!("Failed to save HNSW: {}", e)))?;
+        }
+        
+        // Save metadata
+        let metadata_path = format!("{}.metadata", path_str);
+        {
+            let metadata_store = self.metadata_store.lock().unwrap();
+            let label_to_id = self.label_to_id.lock().unwrap();
+            let current_id = self.current_id.lock().unwrap();
+            
+            let metadata_data = (
+                &*metadata_store,
+                &*label_to_id,
+                *current_id,
+                self.dim,
+                match self.space {
+                    DistanceType::Euclidean => "euclidean",
+                    DistanceType::Cosine => "cosine",
+                    DistanceType::InnerProduct => "inner_product",
+                },
+            );
+            
+            let file = File::create(&metadata_path)
+                .map_err(|e| Error::new(exception::runtime_error(), format!("Failed to create metadata file: {}", e)))?;
+            
+            bincode::serialize_into(file, &metadata_data)
+                .map_err(|e| Error::new(exception::runtime_error(), format!("Failed to save metadata: {}", e)))?;
+        }
+        
+        Ok(value::qnil().as_value())
+    }
+}
+
+// Helper function to parse a Ruby array into a Vec<f32>
+fn parse_vector(array: RArray, expected_dim: usize) -> Result<Vec<f32>, Error> {
+    let len = array.len();
+    if len != expected_dim {
+        return Err(Error::new(
+            exception::arg_error(),
+            format!("Vector dimension mismatch: expected {}, got {}", expected_dim, len)
+        ));
+    }
+    
+    let mut vec = Vec::with_capacity(len);
+    for item in array.each() {
+        let value: f64 = TryConvert::try_convert(item?)
+            .map_err(|_| Error::new(exception::type_error(), "Vector elements must be numeric"))?;
+        vec.push(value as f32);
+    }
+    
+    Ok(vec)
+}
+
+// Helper function to parse metadata
+fn parse_metadata(value: Value) -> Result<HashMap<String, String>, Error> {
+    let hash: RHash = TryConvert::try_convert(value)
+        .map_err(|_| Error::new(exception::type_error(), "Metadata must be a hash"))?;
+    
+    let mut metadata = HashMap::new();
+    
+    hash.foreach(|key: Value, value: Value| {
+        // Handle both string and symbol keys
+        let key_str = if let Ok(s) = String::try_convert(key) {
+            s
+        } else if let Ok(sym) = Symbol::try_convert(key) {
+            sym.name()?.to_string()
+        } else {
+            return Err(Error::new(exception::type_error(), "Metadata keys must be strings or symbols"));
+        };
+        
+        // Convert value to string (handle various Ruby types)
+        let value_str = if let Ok(s) = String::try_convert(value) {
+            s
+        } else if let Ok(i) = Integer::try_convert(value) {
+            i.to_string()
+        } else if let Ok(f) = Float::try_convert(value) {
+            f.to_f64().to_string()
+        } else {
+            // Fallback: use Ruby's to_s method
+            let to_s_method = value.funcall::<_, _, RString>("to_s", ())?;
+            to_s_method.to_string()?
+        };
+        
+        metadata.insert(key_str, value_str);
+        Ok(ForEach::Continue)
+    })?;
+    
+    Ok(metadata)
+}
+
+// Initialize the HNSW module
+pub fn init(parent: &magnus::RModule) -> Result<(), Error> {
+    let class = parent.define_class("HNSW", class::object())?;
+    
+    class.define_singleton_method("new", function!(HnswIndex::new, 1))?;
+    class.define_singleton_method("load", function!(HnswIndex::load, 1))?;
+    class.define_method("add_item", method!(HnswIndex::add_item, 2))?;
+    class.define_method("add_batch", method!(HnswIndex::add_batch, 2))?;
+    class.define_method("search", method!(HnswIndex::search, 2))?;
+    class.define_method("search_with_metadata", method!(HnswIndex::search_with_metadata, 2))?;
+    class.define_method("size", method!(HnswIndex::size, 0))?;
+    class.define_method("empty?", method!(HnswIndex::empty, 0))?;
+    class.define_method("set_ef", method!(HnswIndex::set_ef, 1))?;
+    class.define_method("config", method!(HnswIndex::config, 0))?;
+    class.define_method("stats", method!(HnswIndex::stats, 0))?;
+    class.define_method("save", method!(HnswIndex::save, 1))?;
+    
+    Ok(())
+}

data/ext/clusterkit/src/lib.rs

@@ -4,6 +4,7 @@ mod embedder;
 mod svd;
 mod utils;
 mod clustering;
+mod hnsw;
 
 #[cfg(test)]
 mod tests;
@@ -17,6 +18,7 @@ fn init() -> Result<(), Error> {
     svd::init(&module)?;
     utils::init(&module)?;
     clustering::init(&module)?;
+    hnsw::init(&module)?;
     
     Ok(())
 }

data/lib/clusterkit/hnsw.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+module ClusterKit
+  # HNSW (Hierarchical Navigable Small World) index for fast approximate nearest neighbor search
+  #
+  # @example Basic usage
+  #   index = ClusterKit::HNSW.new(dim: 128, space: :euclidean)
+  #   index.add_batch(vectors, labels: labels)
+  #   neighbors = index.search(query_vector, k: 10)
+  #
+  # @example With metadata
+  #   index = ClusterKit::HNSW.new(dim: 768, space: :cosine)
+  #   index.add_item(vector, label: "doc_1", metadata: { title: "Introduction", date: "2024-01-01" })
+  #   results = index.search_with_metadata(query, k: 5)
+  #   # => [{ label: "doc_1", distance: 0.23, metadata: { title: "...", date: "..." } }, ...]
+  class HNSW
+    # Note: The actual HNSW class is defined in Rust (ext/clusterkit/src/hnsw.rs)
+    # This Ruby file adds additional convenience methods and documentation.
+    # The Rust implementation provides these core methods:
+    #   - new(kwargs) - constructor
+    #   - add_item(vector, kwargs) - add single item
+    #   - add_batch(vectors, kwargs) - add multiple items
+    #   - search(query, kwargs) - search for neighbors
+    #   - search_with_metadata(query, kwargs) - search with metadata
+    #   - size() - get number of items
+    #   - config() - get configuration
+    #   - stats() - get statistics
+    #   - set_ef(ef) - set search quality parameter
+    #   - save(path) - save to file
+    
+    # Initialize is actually handled by the Rust code
+    # This documentation is for reference
+    #
+    # @param dim [Integer] Dimension of vectors (required)
+    # @param space [Symbol] Distance metric: :euclidean, :cosine, or :inner_product (default: :euclidean)
+    # @param max_elements [Integer] Maximum number of elements (default: 10_000)
+    # @param m [Integer] Number of bi-directional links (default: 16)
+    # @param ef_construction [Integer] Size of dynamic candidate list (default: 200)
+    # @param random_seed [Integer, nil] Random seed for reproducible builds (default: nil)
+    # @param dynamic_list [Boolean] Allow index to grow dynamically (not yet implemented)
+    
+    # Fit the index with training data (alias for add_batch)
+    #
+    # @param data [Array<Array>, Numo::NArray] Training vectors
+    # @param labels [Array, nil] Optional labels for vectors
+    # @return [self]
+    def fit(data, labels: nil)
+      add_batch(data, labels: labels)
+      self
+    end
+    
+    # Fit and return transformed data (for compatibility with sklearn-like interface)
+    #
+    # @param data [Array<Array>, Numo::NArray] Training vectors
+    # @return [self]
+    def fit_transform(data)
+      fit(data)
+      self
+    end
+    
+    # Add a vector using the << operator
+    #
+    # @param vector [Array, Numo::NArray] Vector to add
+    # @return [self]
+    def <<(vector)
+      add_item(vector, {})
+      self
+    end
+    
+    # Alias for search that always includes distances
+    #
+    # @param query [Array, Numo::NArray] Query vector
+    # @param k [Integer] Number of neighbors
+    # @param ef [Integer, nil] Search parameter (higher = better quality, slower)
+    # @return [Array<Array>] Array of [indices, distances]
+    def knn_query(query, k: 10, ef: nil)
+      search(query, k: k, ef: ef, include_distances: true)
+    end
+    
+    # Batch search for multiple queries
+    #
+    # @param queries [Array<Array>, Numo::NArray] Multiple query vectors
+    # @param k [Integer] Number of neighbors per query
+    # @param parallel [Boolean] Process queries in parallel
+    # @return [Array<Array>] Results for each query
+    def batch_search(queries, k: 10, parallel: true)
+      queries = ensure_array(queries)
+      
+      if parallel && queries.size > 1
+        require 'parallel'
+        Parallel.map(queries) { |query| search(query, k: k) }
+      else
+        queries.map { |query| search(query, k: k) }
+      end
+    rescue LoadError
+      # Parallel gem not available, fall back to sequential
+      queries.map { |query| search(query, k: k) }
+    end
+    
+    # Range search - find all points within a given radius
+    #
+    # @param query [Array, Numo::NArray] Query vector
+    # @param radius [Float] Search radius
+    # @param limit [Integer, nil] Maximum number of results
+    # @return [Array<Hash>] Results within radius
+    def range_search(query, radius:, limit: nil)
+      # Get a large number of candidates
+      k = limit || size
+      k = [k, size].min
+      
+      results = search_with_metadata(query, k: k)
+      
+      # Filter by radius
+      results.select { |r| r[:distance] <= radius }
+             .take(limit || results.size)
+    end
+    
+    # Check if index is empty
+    # @return [Boolean]
+    def empty?
+      size == 0
+    end
+    
+    # Clear all elements from the index
+    #
+    # @return [self]
+    def clear!
+      # Would need to recreate the index
+      raise NotImplementedError, "Clear not yet implemented"
+    end
+    
+    # Check if a label exists in the index
+    #
+    # @param label [String, Integer] Label to check
+    # @return [Boolean]
+    def include?(label)
+      # This would need to be implemented in Rust
+      # For now, return false
+      false
+    end
+    
+    # Get recall rate for a test set
+    #
+    # @param test_queries [Array<Array>] Query vectors
+    # @param ground_truth [Array<Array>] True nearest neighbors for each query
+    # @param k [Integer] Number of neighbors to evaluate
+    # @return [Float] Recall rate (0.0 to 1.0)
+    def recall(test_queries, ground_truth, k: 10)
+      test_queries = ensure_array(test_queries)
+      
+      require 'set'
+      total_correct = 0
+      total_possible = 0
+      
+      test_queries.each_with_index do |query, i|
+        predicted = Set.new(search(query, k: k))
+        actual = Set.new(ground_truth[i].take(k))
+        
+        total_correct += (predicted & actual).size
+        total_possible += [k, actual.size].min
+      end
+      
+      total_possible > 0 ? total_correct.to_f / total_possible : 0.0
+    end
+    
+    # Load an index from file
+    # Note: This uses Box::leak internally to work around hnsw_rs lifetime constraints
+    # This causes a small memory leak - the HnswIo struct won't be freed until program exit
+    #
+    # @param path [String] File path to load from
+    # @return [HNSW] New HNSW instance loaded from file
+    # (The actual implementation is in Rust)
+    
+    # Create an index from embeddings produced by UMAP or other dimensionality reduction
+    #
+    # @param embeddings [Array<Array>, Numo::NArray] Embedding vectors
+    # @param kwargs [Hash] Additional options for HNSW initialization
+    # @return [HNSW] New HNSW instance
+    def self.from_embedding(embeddings, **kwargs)
+      embeddings = ensure_array(embeddings)
+      
+      dim = embeddings.first.size
+      index = new(dim: dim, **kwargs)
+      index.fit(embeddings)
+      index
+    end
+    
+    # Builder pattern for creating HNSW indices
+    class Builder
+      def initialize
+        @config = {}
+      end
+      
+      def space(type)
+        @config[:space] = type
+        self
+      end
+      
+      def dimensions(dim)
+        @config[:dim] = dim
+        self
+      end
+      
+      def max_elements(n)
+        @config[:max_elements] = n
+        self
+      end
+      
+      def m_parameter(m)
+        @config[:m] = m
+        self
+      end
+      
+      def ef_construction(ef)
+        @config[:ef_construction] = ef
+        self
+      end
+      
+      def seed(seed)
+        @config[:random_seed] = seed
+        self
+      end
+      
+      def build
+        HNSW.new(**@config)
+      end
+    end
+    
+    private
+    
+    # Ensure input is a proper array format
+    def ensure_array(data)
+      case data
+      when Array
+        data
+      else
+        data.respond_to?(:to_a) ? data.to_a : raise(ArgumentError, "Data must be convertible to Array")
+      end
+    end
+    
+    # Class method to make it available to class methods
+    def self.ensure_array(data)
+      case data
+      when Array
+        data
+      else
+        data.respond_to?(:to_a) ? data.to_a : raise(ArgumentError, "Data must be convertible to Array")
+      end
+    end
+  end
+end

data/lib/clusterkit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ClusterKit
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/clusterkit.rb

@@ -29,6 +29,7 @@ module ClusterKit
   # Load modules that depend on the extension
   require_relative "clusterkit/dimensionality"
   require_relative "clusterkit/clustering"
+  require_relative "clusterkit/hnsw"
   
   # Make RustUMAP private - it's an implementation detail
   # Users should use Dimensionality::UMAP instead

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clusterkit
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Chris Petersen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-08-21 00:00:00.000000000 Z
+date: 2025-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -136,6 +136,9 @@ files:
 - docs/UMAP_EXPLAINED.md
 - docs/UMAP_TROUBLESHOOTING.md
 - docs/VERBOSE_OUTPUT.md
+- docs/assets/clusterkit-wide.png
+- docs/assets/clusterkit.png
+- docs/assets/visualization.png
 - examples/hdbscan_example.rb
 - examples/optimal_kmeans_example.rb
 - examples/pca_example.rb
@@ -146,6 +149,7 @@ files:
 - ext/clusterkit/src/clustering.rs
 - ext/clusterkit/src/clustering/hdbscan_wrapper.rs
 - ext/clusterkit/src/embedder.rs
+- ext/clusterkit/src/hnsw.rs
 - ext/clusterkit/src/lib.rs
 - ext/clusterkit/src/svd.rs
 - ext/clusterkit/src/tests.rs
@@ -160,6 +164,7 @@ files:
 - lib/clusterkit/dimensionality/svd.rb
 - lib/clusterkit/dimensionality/umap.rb
 - lib/clusterkit/hdbscan_api_design.rb
+- lib/clusterkit/hnsw.rb
 - lib/clusterkit/preprocessing.rb
 - lib/clusterkit/silence.rb
 - lib/clusterkit/utils.rb