clusterkit 0.1.0 → 0.1.1

data/lib/clusterkit/dimensionality/svd.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../clusterkit'
+require_relative '../data_validator'
 
 module ClusterKit
   module Dimensionality
@@ -8,7 +9,7 @@ module ClusterKit
     # Decomposes a matrix into U, S, V^T components
     class SVD
       attr_reader :n_components, :n_iter, :random_seed
-      attr_reader :u, :s, :vt
+      attr_reader :u, :s, :vt, :n_features
       
       # Initialize a new SVD instance
       # @param n_components [Integer] Number of components to compute
@@ -27,7 +28,8 @@ module ClusterKit
       def fit_transform(data)
         validate_input(data)
         
-        # Store reference to original data for transform detection
+        # Store data characteristics for later transform operations
+        @n_features = data.first.size
         @original_data_id = data.object_id
         
         # Determine n_components if not set
@@ -77,26 +79,21 @@ module ClusterKit
       
       # Transform data using fitted SVD (project onto components)
       # @param data [Array<Array<Numeric>>] Data to transform
-      # @return [Array<Array<Float>>] Transformed data (U * S)
+      # @return [Array<Array<Float>>] Transformed data projected onto SVD components
       def transform(data)
         raise RuntimeError, "Model must be fitted first" unless fitted?
-        validate_input(data)
-        
-        # For SVD, transform typically means projecting onto the components
-        # This is equivalent to data * V (or data * V^T.T)
-        # But for dimensionality reduction, we usually want U * S
-        # which is already computed in fit_transform
+        validate_transform_input(data)
         
-        # If transforming new data, we'd need to project it
-        # For now, return U * S for the fitted data
         if data.object_id == @original_data_id
           # Same data that was fitted - return U * S
           @u.map.with_index do |row, i|
             row.map.with_index { |val, j| val * @s[j] }
           end
         else
-          # New data - would need proper projection
-          raise NotImplementedError, "Transform for new data not yet implemented"
+          # New data - project onto V components: data × V
+          # Since we have V^T, we need to transpose it back to V
+          # V = V^T^T, so we project: data × V^T^T
+          transform_new_data(data)
         end
       end
       
@@ -135,9 +132,43 @@ module ClusterKit
       private
       
       def validate_input(data)
-        raise ArgumentError, "Input must be an array" unless data.is_a?(Array)
-        raise ArgumentError, "Input cannot be empty" if data.empty?
-        raise ArgumentError, "Input must be a 2D array" unless data.first.is_a?(Array)
+        DataValidator.validate_standard(data, check_finite: false)
+      end
+      
+      def validate_transform_input(data)
+        DataValidator.validate_standard(data, check_finite: false)
+        
+        # Check feature count matches training data
+        if data.first.size != @n_features
+          raise ArgumentError, "New data has #{data.first.size} features, but model was fitted with #{@n_features} features"
+        end
+      end
+      
+      # Transform new data by projecting onto V components
+      # Mathematical operation: new_data × V, where V = V^T^T
+      def transform_new_data(data)
+        # V^T is stored as @vt (shape: n_components × n_features)
+        # We need V (shape: n_features × n_components)
+        # V = V^T^T, so we transpose @vt
+        
+        result = []
+        data.each do |sample|
+          # Project sample onto each component (column of V = row of V^T)
+          projected = Array.new(@vt.size, 0.0)
+          
+          @vt.each_with_index do |vt_row, comp_idx|
+            # Dot product: sample · vt_row (this is sample · V[:, comp_idx])
+            dot_product = 0.0
+            sample.each_with_index do |val, feat_idx|
+              dot_product += val * vt_row[feat_idx]
+            end
+            projected[comp_idx] = dot_product
+          end
+          
+          result << projected
+        end
+        
+        result
       end
     end
   end

data/lib/clusterkit/dimensionality/umap.rb

@@ -4,6 +4,7 @@ require 'fileutils'
 require 'json'
 require_relative '../configuration'
 require_relative '../silence'
+require_relative '../data_validator'
 
 module ClusterKit
   module Dimensionality
@@ -224,44 +225,10 @@ module ClusterKit
     end
 
     def validate_input(data, check_min_samples: true)
-      raise ArgumentError, "Input must be an array" unless data.is_a?(Array)
-      raise ArgumentError, "Input cannot be empty" if data.empty?
+      # Use shared validation for common checks
+      DataValidator.validate_standard(data)
 
-      first_row = data.first
-      raise ArgumentError, "Input must be a 2D array (array of arrays)" unless first_row.is_a?(Array)
-
-      row_length = first_row.length
-      min_val = Float::INFINITY
-      max_val = -Float::INFINITY
-
-      # First validate data structure and types
-      data.each_with_index do |row, i|
-        unless row.is_a?(Array)
-          raise ArgumentError, "Row #{i} is not an array"
-        end
-
-        if row.length != row_length
-          raise ArgumentError, "All rows must have the same length (row #{i} has #{row.length} elements, expected #{row_length})"
-        end
-
-        row.each_with_index do |val, j|
-          unless val.is_a?(Numeric)
-            raise ArgumentError, "Element at position [#{i}, #{j}] is not numeric"
-          end
-
-          # Only check for NaN/Infinite on floats
-          if val.is_a?(Float) && (val.nan? || val.infinite?)
-            raise ArgumentError, "Element at position [#{i}, #{j}] is NaN or Infinite"
-          end
-
-          # Track data range
-          val_f = val.to_f
-          min_val = val_f if val_f < min_val
-          max_val = val_f if val_f > max_val
-        end
-      end
-
-      # Check for sufficient data points after validating structure (only for fit operations)
+      # UMAP-specific validations
       if check_min_samples && data.size < 10
         raise ::ClusterKit::InsufficientDataError, <<~MSG
           UMAP requires at least 10 data points, but only #{data.size} provided.
@@ -274,9 +241,9 @@ module ClusterKit
       end
 
       # Check for extreme data ranges that might cause numerical issues
-      data_range = max_val - min_val
-      if data_range > 1000
-        warn "WARNING: Large data range detected (#{data_range.round(2)}). Consider normalizing your data to prevent numerical instability."
+      stats = DataValidator.data_statistics(data)
+      if stats[:data_range] > 1000
+        warn "WARNING: Large data range detected (#{stats[:data_range].round(2)}). Consider normalizing your data to prevent numerical instability."
       end
     end
 

data/lib/clusterkit/hnsw.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+module ClusterKit
+  # HNSW (Hierarchical Navigable Small World) index for fast approximate nearest neighbor search
+  #
+  # @example Basic usage
+  #   index = ClusterKit::HNSW.new(dim: 128, space: :euclidean)
+  #   index.add_batch(vectors, labels: labels)
+  #   neighbors = index.search(query_vector, k: 10)
+  #
+  # @example With metadata
+  #   index = ClusterKit::HNSW.new(dim: 768, space: :cosine)
+  #   index.add_item(vector, label: "doc_1", metadata: { title: "Introduction", date: "2024-01-01" })
+  #   results = index.search_with_metadata(query, k: 5)
+  #   # => [{ label: "doc_1", distance: 0.23, metadata: { title: "...", date: "..." } }, ...]
+  class HNSW
+    # Note: The actual HNSW class is defined in Rust (ext/clusterkit/src/hnsw.rs)
+    # This Ruby file adds additional convenience methods and documentation.
+    # The Rust implementation provides these core methods:
+    #   - new(kwargs) - constructor
+    #   - add_item(vector, kwargs) - add single item
+    #   - add_batch(vectors, kwargs) - add multiple items
+    #   - search(query, kwargs) - search for neighbors
+    #   - search_with_metadata(query, kwargs) - search with metadata
+    #   - size() - get number of items
+    #   - config() - get configuration
+    #   - stats() - get statistics
+    #   - set_ef(ef) - set search quality parameter
+    #   - save(path) - save to file
+    
+    # Initialize is actually handled by the Rust code
+    # This documentation is for reference
+    #
+    # @param dim [Integer] Dimension of vectors (required)
+    # @param space [Symbol] Distance metric: :euclidean, :cosine, or :inner_product (default: :euclidean)
+    # @param max_elements [Integer] Maximum number of elements (default: 10_000)
+    # @param m [Integer] Number of bi-directional links (default: 16)
+    # @param ef_construction [Integer] Size of dynamic candidate list (default: 200)
+    # @param random_seed [Integer, nil] Random seed for reproducible builds (default: nil)
+    # @param dynamic_list [Boolean] Allow index to grow dynamically (not yet implemented)
+    
+    # Fit the index with training data (alias for add_batch)
+    #
+    # @param data [Array<Array>, Numo::NArray] Training vectors
+    # @param labels [Array, nil] Optional labels for vectors
+    # @return [self]
+    def fit(data, labels: nil)
+      add_batch(data, labels: labels)
+      self
+    end
+    
+    # Fit and return transformed data (for compatibility with sklearn-like interface)
+    #
+    # @param data [Array<Array>, Numo::NArray] Training vectors
+    # @return [self]
+    def fit_transform(data)
+      fit(data)
+      self
+    end
+    
+    # Add a vector using the << operator
+    #
+    # @param vector [Array, Numo::NArray] Vector to add
+    # @return [self]
+    def <<(vector)
+      add_item(vector, {})
+      self
+    end
+    
+    # Alias for search that always includes distances
+    #
+    # @param query [Array, Numo::NArray] Query vector
+    # @param k [Integer] Number of neighbors
+    # @param ef [Integer, nil] Search parameter (higher = better quality, slower)
+    # @return [Array<Array>] Array of [indices, distances]
+    def knn_query(query, k: 10, ef: nil)
+      search(query, k: k, ef: ef, include_distances: true)
+    end
+    
+    # Batch search for multiple queries
+    #
+    # @param queries [Array<Array>, Numo::NArray] Multiple query vectors
+    # @param k [Integer] Number of neighbors per query
+    # @param parallel [Boolean] Process queries in parallel
+    # @return [Array<Array>] Results for each query
+    def batch_search(queries, k: 10, parallel: true)
+      queries = ensure_array(queries)
+      
+      if parallel && queries.size > 1
+        require 'parallel'
+        Parallel.map(queries) { |query| search(query, k: k) }
+      else
+        queries.map { |query| search(query, k: k) }
+      end
+    rescue LoadError
+      # Parallel gem not available, fall back to sequential
+      queries.map { |query| search(query, k: k) }
+    end
+    
+    # Range search - find all points within a given radius
+    #
+    # @param query [Array, Numo::NArray] Query vector
+    # @param radius [Float] Search radius
+    # @param limit [Integer, nil] Maximum number of results
+    # @return [Array<Hash>] Results within radius
+    def range_search(query, radius:, limit: nil)
+      # Get a large number of candidates
+      k = limit || size
+      k = [k, size].min
+      
+      results = search_with_metadata(query, k: k)
+      
+      # Filter by radius
+      results.select { |r| r[:distance] <= radius }
+             .take(limit || results.size)
+    end
+    
+    # Check if index is empty
+    # @return [Boolean]
+    def empty?
+      size == 0
+    end
+    
+    # Clear all elements from the index
+    #
+    # @return [self]
+    def clear!
+      # Would need to recreate the index
+      raise NotImplementedError, "Clear not yet implemented"
+    end
+    
+    # Check if a label exists in the index
+    #
+    # @param label [String, Integer] Label to check
+    # @return [Boolean]
+    def include?(label)
+      # This would need to be implemented in Rust
+      # For now, return false
+      false
+    end
+    
+    # Get recall rate for a test set
+    #
+    # @param test_queries [Array<Array>] Query vectors
+    # @param ground_truth [Array<Array>] True nearest neighbors for each query
+    # @param k [Integer] Number of neighbors to evaluate
+    # @return [Float] Recall rate (0.0 to 1.0)
+    def recall(test_queries, ground_truth, k: 10)
+      test_queries = ensure_array(test_queries)
+      
+      require 'set'
+      total_correct = 0
+      total_possible = 0
+      
+      test_queries.each_with_index do |query, i|
+        predicted = Set.new(search(query, k: k))
+        actual = Set.new(ground_truth[i].take(k))
+        
+        total_correct += (predicted & actual).size
+        total_possible += [k, actual.size].min
+      end
+      
+      total_possible > 0 ? total_correct.to_f / total_possible : 0.0
+    end
+    
+    # Load an index from file
+    # Note: This uses Box::leak internally to work around hnsw_rs lifetime constraints
+    # This causes a small memory leak - the HnswIo struct won't be freed until program exit
+    #
+    # @param path [String] File path to load from
+    # @return [HNSW] New HNSW instance loaded from file
+    # (The actual implementation is in Rust)
+    
+    # Create an index from embeddings produced by UMAP or other dimensionality reduction
+    #
+    # @param embeddings [Array<Array>, Numo::NArray] Embedding vectors
+    # @param kwargs [Hash] Additional options for HNSW initialization
+    # @return [HNSW] New HNSW instance
+    def self.from_embedding(embeddings, **kwargs)
+      embeddings = ensure_array(embeddings)
+      
+      dim = embeddings.first.size
+      index = new(dim: dim, **kwargs)
+      index.fit(embeddings)
+      index
+    end
+    
+    # Builder pattern for creating HNSW indices
+    class Builder
+      def initialize
+        @config = {}
+      end
+      
+      def space(type)
+        @config[:space] = type
+        self
+      end
+      
+      def dimensions(dim)
+        @config[:dim] = dim
+        self
+      end
+      
+      def max_elements(n)
+        @config[:max_elements] = n
+        self
+      end
+      
+      def m_parameter(m)
+        @config[:m] = m
+        self
+      end
+      
+      def ef_construction(ef)
+        @config[:ef_construction] = ef
+        self
+      end
+      
+      def seed(seed)
+        @config[:random_seed] = seed
+        self
+      end
+      
+      def build
+        HNSW.new(**@config)
+      end
+    end
+    
+    private
+    
+    # Ensure input is a proper array format
+    def ensure_array(data)
+      case data
+      when Array
+        data
+      else
+        data.respond_to?(:to_a) ? data.to_a : raise(ArgumentError, "Data must be convertible to Array")
+      end
+    end
+    
+    # Class method to make it available to class methods
+    def self.ensure_array(data)
+      case data
+      when Array
+        data
+      else
+        data.respond_to?(:to_a) ? data.to_a : raise(ArgumentError, "Data must be convertible to Array")
+      end
+    end
+  end
+end

data/lib/clusterkit/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ClusterKit
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/clusterkit.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "clusterkit/version"
-require_relative "clusterkit/clusterkit"
+require "clusterkit/clusterkit"
 require_relative "clusterkit/configuration"
 
 # Main module for ClusterKit gem
@@ -29,6 +29,7 @@ module ClusterKit
   # Load modules that depend on the extension
   require_relative "clusterkit/dimensionality"
   require_relative "clusterkit/clustering"
+  require_relative "clusterkit/hnsw"
   
   # Make RustUMAP private - it's an implementation detail
   # Users should use Dimensionality::UMAP instead

metadata

@@ -1,17 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: clusterkit
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Chris Petersen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-08-21 00:00:00.000000000 Z
+date: 2026-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: csv
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: benchmark
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,47 +39,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: csv
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake-compiler
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
-  name: rb_sys
+  name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '1.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +136,7 @@ files:
 - ".simplecov"
 - CHANGELOG.md
 - CLAUDE.md
+- Cargo.lock
 - Cargo.toml
 - Gemfile
 - IMPLEMENTATION_NOTES.md
@@ -129,13 +144,15 @@ files:
 - PYTHON_COMPARISON.md
 - README.md
 - Rakefile
-- clusterkit.gemspec
 - docs/KNOWN_ISSUES.md
 - docs/RUST_ERROR_HANDLING.md
 - docs/TEST_FIXTURES.md
 - docs/UMAP_EXPLAINED.md
 - docs/UMAP_TROUBLESHOOTING.md
 - docs/VERBOSE_OUTPUT.md
+- docs/assets/clusterkit-wide.png
+- docs/assets/clusterkit.png
+- docs/assets/visualization.png
 - examples/hdbscan_example.rb
 - examples/optimal_kmeans_example.rb
 - examples/pca_example.rb
@@ -146,6 +163,7 @@ files:
 - ext/clusterkit/src/clustering.rs
 - ext/clusterkit/src/clustering/hdbscan_wrapper.rs
 - ext/clusterkit/src/embedder.rs
+- ext/clusterkit/src/hnsw.rs
 - ext/clusterkit/src/lib.rs
 - ext/clusterkit/src/svd.rs
 - ext/clusterkit/src/tests.rs
@@ -155,23 +173,25 @@ files:
 - lib/clusterkit/clustering/hdbscan.rb
 - lib/clusterkit/clusterkit.rb
 - lib/clusterkit/configuration.rb
+- lib/clusterkit/data_validator.rb
 - lib/clusterkit/dimensionality.rb
 - lib/clusterkit/dimensionality/pca.rb
 - lib/clusterkit/dimensionality/svd.rb
 - lib/clusterkit/dimensionality/umap.rb
 - lib/clusterkit/hdbscan_api_design.rb
+- lib/clusterkit/hnsw.rb
 - lib/clusterkit/preprocessing.rb
 - lib/clusterkit/silence.rb
 - lib/clusterkit/utils.rb
 - lib/clusterkit/version.rb
 - lib/tasks/visualize.rake
-homepage: https://github.com/cpetersen/clusterkit
+homepage: https://github.com/scientist-labs/clusterkit
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/cpetersen/clusterkit
-  source_code_uri: https://github.com/cpetersen/clusterkit
-  changelog_uri: https://github.com/cpetersen/clusterkit/blob/main/CHANGELOG.md
+  homepage_uri: https://github.com/scientist-labs/clusterkit
+  source_code_uri: https://github.com/scientist-labs/clusterkit
+  changelog_uri: https://github.com/scientist-labs/clusterkit/blob/main/CHANGELOG.md
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -187,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: High-performance clustering and dimensionality reduction for Ruby

data/clusterkit.gemspec

@@ -1,45 +0,0 @@
-require_relative "lib/clusterkit/version"
-
-Gem::Specification.new do |spec|
-  spec.name = "clusterkit"
-  spec.version = ClusterKit::VERSION
-  spec.authors = ["Chris Petersen"]
-  spec.email = ["chris@petersen.io"]
-
-  spec.summary = "High-performance clustering and dimensionality reduction for Ruby"
-  spec.description = "A comprehensive clustering toolkit for Ruby, providing UMAP, PCA, K-means, HDBSCAN and more. Built on top of annembed and hdbscan Rust crates for blazing-fast performance."
-  spec.homepage = "https://github.com/cpetersen/clusterkit"
-  spec.license = "MIT"
-  spec.required_ruby_version = ">= 2.7.0"
-
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = spec.homepage
-  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
-
-  # Specify which files should be added to the gem when it is released.
-  spec.files = Dir.chdir(__dir__) do
-    `git ls-files -z`.split("\x0").reject do |f|
-      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
-    end + Dir["ext/**/*.rs", "ext/**/*.toml"]
-  end
-  spec.bindir = "exe"
-  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-  spec.extensions = ["ext/clusterkit/extconf.rb"]
-
-  # Runtime dependencies
-  # Numo is optional but recommended for better performance
-  # spec.add_dependency "numo-narray", "~> 0.9"
-
-  # Development dependencies
-  spec.add_development_dependency "csv"
-  spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rake-compiler", "~> 1.2"
-  spec.add_development_dependency "rb_sys", "~> 0.9"
-  spec.add_development_dependency "rspec", "~> 3.0"
-  spec.add_development_dependency "simplecov", "~> 0.22"
-  spec.add_development_dependency "yard", "~> 0.9"
-
-  # For more information and examples about making a new gem, check out our
-  # guide at: https://bundler.io/guides/creating_gem.html
-end