ai4r 1.12 → 2.0

data/lib/ai4r/clusterers/diana.rb

@@ -1,74 +1,85 @@
+# frozen_string_literal: true
+
 # Author::    Sergio Fierens (implementation)
 # License::   MPL 1.1
 # Project::   ai4r
-# Url::       http://www.ai4r.org/
+# Url::       https://github.com/SergioFierens/ai4r
 #
-# You can redistribute it and/or modify it under the terms of 
-# the Mozilla Public License version 1.1  as published by the 
+# You can redistribute it and/or modify it under the terms of
+# the Mozilla Public License version 1.1  as published by the
 # Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
 
-require File.dirname(__FILE__) + '/../data/data_set'
-require File.dirname(__FILE__) + '/../data/proximity'
-require File.dirname(__FILE__) + '/../clusterers/clusterer'
+require_relative '../data/data_set'
+require_relative '../data/proximity'
+require_relative '../clusterers/clusterer'
 
 module Ai4r
   module Clusterers
-     
-    # DIANA (Divisive ANAlysis) (Kaufman and Rousseeuw, 1990; 
+    # DIANA (Divisive ANAlysis) (Kaufman and Rousseeuw, 1990;
     # Macnaughton - Smith et al. 1964) is a Divisive Hierarchical
     # Clusterer. It begins with only one cluster with all data items,
     # and divides the clusters until the desired clusters number is reached.
     class Diana < Clusterer
-      
       attr_reader :data_set, :number_of_clusters, :clusters
-      
-      parameters_info :distance_function => 
-          "Custom implementation of distance function. " +
-          "It must be a closure receiving two data items and return the " +
-          "distance bewteen them. By default, this algorithm uses " + 
-          "ecuclidean distance of numeric attributes to the power of 2."
-      
+
+      parameters_info distance_function:
+          'Custom implementation of distance function. ' \
+          'It must be a closure receiving two data items and return the ' \
+          'distance between them. By default, this algorithm uses ' \
+          'euclidean distance of numeric attributes to the power of 2.'
+
+      # @return [Object]
       def initialize
-        @distance_function = lambda do |a,b| 
-            Ai4r::Data::Proximity.squared_euclidean_distance(
-              a.select {|att_a| att_a.is_a? Numeric} , 
-              b.select {|att_b| att_b.is_a? Numeric})
-          end
+        super()
+        @distance_function = lambda do |a, b|
+          Ai4r::Data::Proximity.squared_euclidean_distance(
+            a.select { |att_a| att_a.is_a? Numeric },
+            b.select { |att_b| att_b.is_a? Numeric }
+          )
+        end
       end
-      
+
       # Build a new clusterer, using divisive analysis (DIANA algorithm)
+      # @param data_set [Object]
+      # @param number_of_clusters [Object]
+      # @return [Object]
       def build(data_set, number_of_clusters)
         @data_set = data_set
         @number_of_clusters = number_of_clusters
-        @clusters = [@data_set[0..-1]]
-        
-        while(@clusters.length < @number_of_clusters)
+        @clusters = [@data_set]
+
+        while @clusters.length < @number_of_clusters
           cluster_index_to_split = max_diameter_cluster(@clusters)
           cluster_to_split = @clusters[cluster_index_to_split]
           splinter_cluster = init_splinter_cluster(cluster_to_split)
-          while true
+          loop do
             dist_diff, index = max_distance_difference(cluster_to_split, splinter_cluster)
-            break if dist_diff < 0
+            break if dist_diff.negative?
+
             splinter_cluster << cluster_to_split.data_items[index]
             cluster_to_split.data_items.delete_at(index)
           end
           @clusters << splinter_cluster
         end
-       
-        return self
+
+        self
       end
-      
-      # Classifies the given data item, returning the cluster index it belongs 
+
+      # Classifies the given data item, returning the cluster index it belongs
       # to (0-based).
+      # @param data_item [Object]
+      # @return [Object]
       def eval(data_item)
         get_min_index(@clusters.collect do |cluster|
           distance_sum(data_item, cluster) / cluster.data_items.length
-          end)
+        end)
       end
-      
+
       protected
-      
+
       # return the cluster with max diameter
+      # @param clusters [Object]
+      # @return [Object]
       def max_diameter_cluster(clusters)
         max_index = 0
         max_diameter = 0
@@ -79,10 +90,12 @@ module Ai4r
             max_diameter = diameter
           end
         end
-        return max_index
+        max_index
       end
-      
+
       # Max distance between 2 items in a cluster
+      # @param cluster [Object]
+      # @return [Object]
       def cluster_diameter(cluster)
         diameter = 0
         cluster.data_items.each_with_index do |item_a, item_a_pos|
@@ -91,49 +104,62 @@ module Ai4r
             diameter = d if d > diameter
           end
         end
-        return diameter
+        diameter
       end
-      
+
       # Create a cluster with the item with mx distance
       # to the rest of the cluster's items.
       # That item is removed from the initial cluster.
+      # @param cluster_to_split [Object]
+      # @return [Object]
       def init_splinter_cluster(cluster_to_split)
         max = 0.0
         max_index = 0
         cluster_to_split.data_items.each_with_index do |item, index|
           sum = distance_sum(item, cluster_to_split)
-          max, max_index = sum, index if sum > max
+          if sum > max
+            max = sum
+            max_index = index
+          end
         end
         splinter_cluster = cluster_to_split[max_index]
         cluster_to_split.data_items.delete_at(max_index)
-        return splinter_cluster
+        splinter_cluster
       end
-      
-      # Return the max average distance between any item of 
+
+      # Return the max average distance between any item of
       # cluster_to_split and the rest of items in that cluster,
       # minus the average distance with the items of splinter_cluster,
       # and the index of the item.
       # A positive value means that the items is closer to the
       # splinter group than to its current cluster.
+      # @param cluster_to_split [Object]
+      # @param splinter_cluster [Object]
+      # @return [Object]
       def max_distance_difference(cluster_to_split, splinter_cluster)
-        max_diff = -1.0/0
+        max_diff = -Float::INFINITY
         max_diff_index = 0
         cluster_to_split.data_items.each_with_index do |item, index|
-          dist_a = distance_sum(item, cluster_to_split) / (cluster_to_split.data_items.length-1)
-          dist_b = distance_sum(item, splinter_cluster) / (splinter_cluster.data_items.length)
+          dist_a = distance_sum(item, cluster_to_split) / (cluster_to_split.data_items.length - 1)
+          dist_b = distance_sum(item, splinter_cluster) / splinter_cluster.data_items.length
           dist_diff = dist_a - dist_b
-          max_diff, max_diff_index = dist_diff, index if dist_diff > max_diff
+          if dist_diff > max_diff
+            max_diff = dist_diff
+            max_diff_index = index
+          end
         end
-        return max_diff, max_diff_index
+        [max_diff, max_diff_index]
       end
-      
+
       # Sum up the distance between an item and all the items in a cluster
+      # @param item_a [Object]
+      # @param cluster [Object]
+      # @return [Object]
       def distance_sum(item_a, cluster)
         cluster.data_items.inject(0.0) do |sum, item_b|
           sum + @distance_function.call(item_a, item_b)
         end
       end
-      
     end
   end
 end

data/lib/ai4r/clusterers/k_means.rb

@@ -1,126 +1,363 @@
+# frozen_string_literal: true
+
 # Author::    Sergio Fierens (implementation)
 # License::   MPL 1.1
 # Project::   ai4r
-# Url::       http://ai4r.org/
+# Url::       https://github.com/SergioFierens/ai4r
 #
-# You can redistribute it and/or modify it under the terms of 
-# the Mozilla Public License version 1.1  as published by the 
+# You can redistribute it and/or modify it under the terms of
+# the Mozilla Public License version 1.1  as published by the
 # Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
 
-require File.dirname(__FILE__) + '/../data/data_set'
-require File.dirname(__FILE__) + '/../clusterers/clusterer'
+require_relative '../data/data_set'
+require_relative '../data/proximity'
+require_relative '../clusterers/clusterer'
 
 module Ai4r
   module Clusterers
-     
-    # The k-means algorithm is an algorithm to cluster n objects 
+    # The k-means algorithm is an algorithm to cluster n objects
     # based on attributes into k partitions, with k < n.
-    # 
+    #
     # More about K Means algorithm:
-    # http://en.wikipedia.org/wiki/K-means_algorithm   
+    # http://en.wikipedia.org/wiki/K-means_algorithm
     class KMeans < Clusterer
-      
-      attr_reader :data_set, :number_of_clusters
-      attr_reader :clusters, :centroids, :iterations
-      
-      parameters_info :max_iterations => "Maximum number of iterations to " + 
-        "build the clusterer. By default it is uncapped.",
-        :distance_function => "Custom implementation of distance function. " +
-          "It must be a closure receiving two data items and return the " +
-          "distance bewteen them. By default, this algorithm uses " + 
-          "ecuclidean distance of numeric attributes to the power of 2.",
-        :centroid_function => "Custom implementation to calculate the " +
-          "centroid of a cluster. It must be a closure receiving an array of " +
-          "data sets, and return an array of data items, representing the " + 
-          "centroids of for each data set. " +
-          "By default, this algorithm returns a data items using the mode "+
-          "or mean of each attribute on each data set."
-      
+      attr_reader :data_set, :number_of_clusters, :clusters, :centroids, :iterations, :history
+
+      parameters_info(
+        max_iterations: 'Maximum number of iterations to build the clusterer. By default it is uncapped.',
+        distance_function: 'Custom implementation of distance function. ' \
+                           'It must be a closure receiving two data items and return the ' \
+                           'distance between them. By default, this algorithm uses ' \
+                           'euclidean distance of numeric attributes to the power of 2.',
+        centroid_function: 'Custom implementation to calculate the ' \
+                           'centroid of a cluster. It must be a closure receiving an array of ' \
+                           'data sets, and return an array of data items, representing the ' \
+                           'centroids of for each data set. ' \
+                           'By default, this algorithm returns a data items using the mode ' \
+                           'or mean of each attribute on each data set.',
+        centroid_indices: 'Indices of data items (indexed from 0) to be ' \
+                          'the initial centroids.  Otherwise, the initial centroids will be ' \
+                          'assigned randomly from the data set.',
+        on_empty: 'Action to take if a cluster becomes empty, with values ' \
+                  "'eliminate' (the default action, eliminate the empty cluster), " \
+                  "'terminate' (terminate with error), 'random' (relocate the " \
+                  "empty cluster to a random point), 'outlier' (relocate the " \
+                  'empty cluster to the point furthest from its centroid).',
+        random_seed: "Seed value used to initialize Ruby's random number " \
+                     'generator when selecting random centroids.',
+        init_method: 'Strategy to initialize centroids. Available values: ' \
+                     ':random (default) and :kmeans_plus_plus.',
+        restarts: 'Number of random initializations to perform. ' \
+                  'The best run (lowest SSE) will be kept.',
+        track_history: 'Keep centroids and assignments for each iteration ' \
+                       'when building the clusterer.'
+      )
+
+      # @return [Object]
       def initialize
+        super()
         @distance_function = nil
         @max_iterations = nil
-        @old_centroids = nil
-        @centroid_function = lambda do |data_sets| 
-          data_sets.collect{ |data_set| data_set.get_mean_or_mode}
+        @centroid_function = lambda do |data_sets|
+          data_sets.collect(&:get_mean_or_mode)
         end
+        @centroid_indices = []
+        @on_empty = 'eliminate' # default if none specified
+        @random_seed = nil
+        @rng = nil
+        @init_method = :random
+        @restarts = 1
+        @track_history = false
       end
-      
-      
+
       # Build a new clusterer, using data examples found in data_set.
       # Items will be clustered in "number_of_clusters" different
       # clusters.
+      # @param data_set [Object]
+      # @param number_of_clusters [Object]
+      # @return [Object]
       def build(data_set, number_of_clusters)
         @data_set = data_set
         @number_of_clusters = number_of_clusters
-        @iterations = 0
-        
-        calc_initial_centroids
-        while(not stop_criteria_met)
-          calculate_membership_clusters
-          recompute_centroids
+        raise ArgumentError, 'Number of clusters larger than data items' if @number_of_clusters > @data_set.data_items.length
+
+        unless @centroid_indices.empty? || @centroid_indices.length == @number_of_clusters
+          raise ArgumentError,
+                'Length of centroid indices array differs from the specified number of clusters'
+        end
+        unless @on_empty == 'eliminate' || @on_empty == 'terminate' || @on_empty == 'random' || @on_empty == 'outlier'
+          raise ArgumentError,
+                'Invalid value for on_empty'
+        end
+
+        seed_base = @random_seed
+        best_sse = nil
+        best_centroids = nil
+        best_clusters = nil
+        best_iterations = nil
+
+        (@restarts || 1).times do |i|
+          @random_seed = seed_base.nil? ? nil : seed_base + i
+          @rng = @random_seed.nil? ? Random.new : Random.new(@random_seed)
+          @iterations = 0
+          @history = [] if @track_history
+          calc_initial_centroids
+          until stop_criteria_met
+            calculate_membership_clusters
+            if @track_history
+              @history << {
+                centroids: @centroids.collect(&:dup),
+                assignments: @assignments.dup
+              }
+            end
+            recompute_centroids
+          end
+          current_sse = sse
+          next unless best_sse.nil? || current_sse < best_sse
+
+          best_sse = current_sse
+          best_centroids = Marshal.load(Marshal.dump(@centroids))
+          best_clusters = Marshal.load(Marshal.dump(@clusters))
+          best_iterations = @iterations
         end
-        
-        return self
+
+        @random_seed = seed_base
+        @rng = @random_seed.nil? ? Random.new : Random.new(@random_seed)
+        @centroids = best_centroids
+        @clusters = best_clusters
+        @iterations = best_iterations
+        self
       end
-      
-      # Classifies the given data item, returning the cluster index it belongs 
+
+      # Classifies the given data item, returning the cluster index it belongs
       # to (0-based).
+      # @param data_item [Object]
+      # @return [Object]
       def eval(data_item)
-        get_min_index(@centroids.collect {|centroid| 
-            distance(data_item, centroid)})
+        get_min_index(@centroids.collect do |centroid|
+          distance(data_item, centroid)
+        end)
+      end
+
+      # Sum of squared distances of all points to their respective centroids.
+      # It can be used as a measure of cluster compactness (SSE).
+      # @return [Object]
+      def sse
+        sum = 0.0
+        @clusters.each_with_index do |cluster, i|
+          centroid = @centroids[i]
+          cluster.data_items.each do |item|
+            sum += distance(item, centroid)
+          end
+        end
+        sum
       end
-      
+
       # This function calculates the distance between 2 different
-      # instances. By default, it returns the euclidean distance to the 
+      # instances. By default, it returns the euclidean distance to the
       # power of 2.
-      # You can provide a more convinient distance implementation:
-      # 
+      # You can provide a more convenient distance implementation:
+      #
       # 1- Overwriting this method
-      # 
+      #
       # 2- Providing a closure to the :distance_function parameter
+      # @param a [Object]
+      # @param b [Object]
+      # @return [Object]
       def distance(a, b)
         return @distance_function.call(a, b) if @distance_function
-        return euclidean_distance(a, b)
+
+        Ai4r::Data::Proximity.squared_euclidean_distance(
+          a.select { |att_a| att_a.is_a? Numeric },
+          b.select { |att_b| att_b.is_a? Numeric }
+        )
       end
-      
-      protected      
-      
+
+      protected
+
+      # @return [Object]
       def calc_initial_centroids
         @centroids = []
-        tried_indexes = []
-        while @centroids.length < @number_of_clusters && 
-            tried_indexes.length < @data_set.data_items.length
-          random_index = rand(@data_set.data_items.length)
-          if !tried_indexes.include?(random_index)
-            tried_indexes << random_index
-            if !@centroids.include? @data_set.data_items[random_index] 
-              @centroids << @data_set.data_items[random_index] 
-            end
+        @old_centroids = nil
+        if @centroid_indices.empty?
+          if @init_method == :kmeans_plus_plus
+            kmeans_plus_plus_init
+          else
+            populate_centroids('random')
           end
+        else
+          populate_centroids('indices')
         end
-        @number_of_clusters = @centroids.length
       end
-      
+
+      # @return [Object]
       def stop_criteria_met
-        @old_centroids == @centroids || 
+        @old_centroids == @centroids ||
           (@max_iterations && (@max_iterations <= @iterations))
       end
-      
+
+      # @return [Object]
       def calculate_membership_clusters
-        @clusters = Array.new(@number_of_clusters) do 
-          Ai4r::Data::DataSet.new :data_labels => @data_set.data_labels
+        @clusters = Array.new(@number_of_clusters) do
+          Ai4r::Data::DataSet.new data_labels: @data_set.data_labels
         end
-        @data_set.data_items.each do |data_item|
-          @clusters[eval(data_item)] << data_item
+        @cluster_indices = Array.new(@number_of_clusters) { [] }
+        @assignments = Array.new(@data_set.data_items.length)
+
+        @data_set.data_items.each_with_index do |data_item, data_index|
+          c = eval(data_item)
+          @clusters[c] << data_item
+          @cluster_indices[c] << data_index if @on_empty == 'outlier'
+          @assignments[data_index] = c
         end
+        manage_empty_clusters if empty_cluster?
       end
-      
+
+      # @return [Object]
       def recompute_centroids
         @old_centroids = @centroids
         @iterations += 1
-        @centroids = @centroid_function.call(@clusters) 
+        @centroids = @centroid_function.call(@clusters)
+      end
+
+      # @return [Object]
+      def kmeans_plus_plus_init
+        chosen_indices = []
+        first_index = (0...@data_set.data_items.length).to_a.sample(random: @rng)
+        return if first_index.nil?
+
+        @centroids << @data_set.data_items[first_index]
+        chosen_indices << first_index
+        while @centroids.length < @number_of_clusters &&
+              chosen_indices.length < @data_set.data_items.length
+          distances = []
+          total = 0.0
+          @data_set.data_items.each_with_index do |item, index|
+            next if chosen_indices.include?(index)
+
+            min_dist = @centroids.map { |c| distance(item, c) }.min
+            distances << [index, min_dist]
+            total += min_dist
+          end
+          break if distances.empty?
+
+          r = @rng.rand * total
+          cumulative = 0.0
+          chosen = distances.find do |_idx, dist|
+            cumulative += dist
+            cumulative >= r
+          end
+          chosen_indices << chosen[0]
+          @centroids << @data_set.data_items[chosen[0]]
+        end
+        @number_of_clusters = @centroids.length
+      end
+
+      # @param populate_method [Object]
+      # @param number_of_clusters [Object]
+      # @return [Object]
+      def populate_centroids(populate_method, number_of_clusters = @number_of_clusters)
+        tried_indexes = []
+        case populate_method
+        when 'random' # for initial assignment (without the :centroid_indices option) and for reassignment of empty cluster centroids (with :on_empty option 'random')
+          while @centroids.length < number_of_clusters &&
+                tried_indexes.length < @data_set.data_items.length
+            random_index = (0...@data_set.data_items.length).to_a.sample(random: @rng)
+            next if tried_indexes.include?(random_index)
+
+            tried_indexes << random_index
+            @centroids << @data_set.data_items[random_index] unless @centroids.include? @data_set.data_items[random_index]
+          end
+        when 'indices' # for initial assignment only (with the :centroid_indices option)
+          @centroid_indices.each do |index|
+            unless (index.is_a? Integer) && index >= 0 && index < @data_set.data_items.length
+              raise ArgumentError,
+                    "Invalid centroid index #{index}"
+            end
+
+            next if tried_indexes.include?(index)
+
+            tried_indexes << index
+            @centroids << @data_set.data_items[index] unless @centroids.include? @data_set.data_items[index]
+          end
+        when 'outlier' # for reassignment of empty cluster centroids only (with :on_empty option 'outlier')
+          sorted_data_indices = sort_data_indices_by_dist_to_centroid
+          i = sorted_data_indices.length - 1 # the last item is the furthest from its centroid
+          while @centroids.length < number_of_clusters &&
+                tried_indexes.length < @data_set.data_items.length
+            outlier_index = sorted_data_indices[i]
+            unless tried_indexes.include?(outlier_index)
+              tried_indexes << outlier_index
+              @centroids << @data_set.data_items[outlier_index] unless @centroids.include? @data_set.data_items[outlier_index]
+            end
+            i.positive? ? i -= 1 : break
+          end
+        end
+        @number_of_clusters = @centroids.length
+      end
+
+      # Sort cluster points by distance to assigned centroid.  Utilizes @cluster_indices.
+      # Returns indices, sorted in order from the nearest to furthest.
+      # @return [Object]
+      def sort_data_indices_by_dist_to_centroid
+        h = {}
+        @clusters.each_with_index do |cluster, c|
+          centroid = @centroids[c]
+          cluster.data_items.each_with_index do |data_item, i|
+            dist_to_centroid = distance(data_item, centroid)
+            data_index = @cluster_indices[c][i]
+            h[data_index] = dist_to_centroid
+          end
+        end
+        # sort hash of {index => dist to centroid} by dist to centroid (ascending) and then return an array of only the indices
+        h.sort_by { |_k, v| v }.collect { |a, _b| a }
+      end
+
+      # @return [Object]
+      def empty_cluster?
+        found_empty = false
+        @number_of_clusters.times do |c|
+          found_empty = true if @clusters[c].data_items.empty?
+        end
+        found_empty
+      end
+
+      # @return [Object]
+      def manage_empty_clusters
+        # Do nothing to terminate with error. (The empty cluster will be assigned a nil centroid, and then calculating the distance from this centroid to another point will raise an exception.)
+        return if on_empty == 'terminate'
+
+        initial_number_of_clusters = @number_of_clusters
+        eliminate_empty_clusters
+        return if on_empty == 'eliminate'
+
+        populate_centroids(on_empty, initial_number_of_clusters) # Add initial_number_of_clusters - @number_of_clusters
+        calculate_membership_clusters
+      end
+
+      # @return [Object]
+      def eliminate_empty_clusters
+        old_clusters = @clusters
+        old_centroids = @centroids
+        old_cluster_indices = @cluster_indices
+        old_assignments = @assignments
+        @clusters = []
+        @centroids = []
+        @cluster_indices = []
+        remap = {}
+        new_index = 0
+        @number_of_clusters.times do |i|
+          next if old_clusters[i].data_items.empty?
+
+          remap[i] = new_index
+          @clusters << old_clusters[i]
+          @cluster_indices << old_cluster_indices[i]
+          @centroids << old_centroids[i]
+          new_index += 1
+        end
+        @number_of_clusters = @centroids.length
+        @assignments = old_assignments.map { |c| remap[c] }
       end
-  
     end
   end
 end