ai4r 1.5 → 1.6

data/lib/ai4r/clusterers/median_linkage.rb

@@ -0,0 +1,61 @@
+# Author::    Sergio Fierens (implementation)
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       http://ai4r.rubyforge.org/
+#
+# You can redistribute it and/or modify it under the terms of 
+# the Mozilla Public License version 1.1  as published by the 
+# Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
+
+require File.dirname(__FILE__) + '/../data/data_set'
+require File.dirname(__FILE__) + '/../clusterers/single_linkage'
+
+module Ai4r
+  module Clusterers
+     
+    # Implementation of an Agglomerative Hierarchical clusterer with 
+    # median linkage algorithm, aka weighted pair group method centroid 
+    # or WPGMC (Everitt et al., 2001 ; Gower, 1967 ; Jain and Dubes, 1988 ).
+    # Hierarchical clusteres create one cluster per element, and then 
+    # progressively merge clusters, until the required number of clusters
+    # is reached.
+    # Similar to centroid linkages, but using fix weight: 
+    # 
+    #   D(cx, (ci U cj)) =  (1/2)*D(cx, ci) + 
+    #                       (1/2)*D(cx, cj) - 
+    #                       (1/4)*D(ci, cj)
+    class MedianLinkage < SingleLinkage
+      
+    parameters_info :distance_function => 
+          "Custom implementation of distance function. " +
+          "It must be a closure receiving two data items and return the " +
+          "distance bewteen them. By default, this algorithm uses " + 
+          "ecuclidean distance of numeric attributes to the power of 2."
+      
+      # Build a new clusterer, using data examples found in data_set.
+      # Items will be clustered in "number_of_clusters" different
+      # clusters.
+      def build(data_set, number_of_clusters)
+        super
+      end
+      
+      # This algorithms does not allow classification of new data items 
+      # once it has been built. Rebuild the cluster including you data element.
+      def eval(data_item)
+        Raise "Eval of new data is not supported by this algorithm."
+      end
+      
+      protected
+       
+      # return distance between cluster cx and cluster (ci U cj),
+      # using median linkage
+      def linkage_distance(cx, ci, cj)
+        ( 0.5  * read_distance_matrix(cx, ci) +
+          0.5  * read_distance_matrix(cx, cj) -
+          0.25 * read_distance_matrix(ci, cj))
+      end
+      
+    end
+  end
+end
+

data/lib/ai4r/clusterers/single_linkage.rb

@@ -8,17 +8,21 @@
 # Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
 
 require File.dirname(__FILE__) + '/../data/data_set'
+require File.dirname(__FILE__) + '/../data/proximity'
 require File.dirname(__FILE__) + '/../clusterers/clusterer'
 
 module Ai4r
   module Clusterers
      
-    # Implementation of a Hierarchical clusterer with single linkage.
+    # Implementation of a Hierarchical clusterer with single linkage (Everitt et 
+    # al., 2001 ; Johnson, 1967 ; Jain and Dubes, 1988 ; Sneath, 1957 )
     # Hierarchical clusteres create one cluster per element, and then 
     # progressively merge clusters, until the required number of clusters
     # is reached.
     # With single linkage, the distance between two clusters is computed as the 
     # distance between the two closest elements in the two clusters.
+    #
+    #   D(cx, (ci U cj) = min(D(cx, ci), D(cx, cj))
     class SingleLinkage < Clusterer
       
       attr_reader :data_set, :number_of_clusters, :clusters
@@ -30,7 +34,11 @@ module Ai4r
           "ecuclidean distance of numeric attributes to the power of 2."
       
       def initialize
-        @distance_function = nil
+        @distance_function = lambda do |a,b| 
+            Ai4r::Data::Proximity.squared_euclidean_distance(
+              a.select {|att_a| att_a.is_a? Numeric} , 
+              b.select {|att_b| att_b.is_a? Numeric})
+          end
       end
       
       # Build a new clusterer, using data examples found in data_set.
@@ -40,13 +48,14 @@ module Ai4r
         @data_set = data_set
         @number_of_clusters = number_of_clusters
         
-        index_clusters = create_initial_index_clusters
+        @index_clusters = create_initial_index_clusters
         create_distance_matrix(data_set)
-        while index_clusters.length > @number_of_clusters
-          clusters_to_merge = get_closest_clusters(index_clusters)
-          index_clusters = merge_clusters(clusters_to_merge, index_clusters)
+        while @index_clusters.length > @number_of_clusters
+          ci, cj = get_closest_clusters(@index_clusters)
+          update_distance_matrix(ci, cj)
+          merge_clusters(ci, cj, @index_clusters)
         end
-        @clusters = build_clusters_from_index_clusters index_clusters
+        @clusters = build_clusters_from_index_clusters @index_clusters
         
         return self
       end
@@ -58,19 +67,6 @@ module Ai4r
             distance_between_item_and_cluster(data_item, cluster)})
       end
       
-      # This function calculates the distance between 2 different
-      # instances. By default, it returns the euclidean distance to the 
-      # power of 2.
-      # You can provide a more convinient distance implementation:
-      # 
-      # 1- Overwriting this method
-      # 
-      # 2- Providing a closure to the :distance_function parameter
-      def distance(a, b)
-        return @distance_function.call(a, b) if @distance_function
-        return euclidean_distance(a, b)
-      end
-      
       protected
       
       # returns [ [0], [1], [2], ... , [n-1] ]
@@ -95,7 +91,7 @@ module Ai4r
         data_set.data_items.each_with_index do |a, i|
           i.times do |j|
             b = data_set.data_items[j]
-            @distance_matrix[i-1][j] = distance(a, b)
+            @distance_matrix[i-1][j] = @distance_function.call(a, b)
           end
         end
       end
@@ -108,14 +104,46 @@ module Ai4r
         return @distance_matrix[index_a-1][index_b]
       end
 
-      # clusters_to_merge = [index_cluster_a, index_cluster_b].
+      # ci and cj are the indexes of the clusters that are going to
+      # be merged. We need to remove distances from/to ci and ci, 
+      # and add distances from/to new cluster (ci U cj)
+      def update_distance_matrix(ci, cj)
+        ci, cj = cj, ci if cj > ci
+        distances_to_new_cluster = Array.new
+        (@distance_matrix.length+1).times do |cx|
+          if cx!= ci && cx!=cj
+            distances_to_new_cluster << linkage_distance(cx, ci, cj) 
+          end
+        end
+        if cj==0 && ci==1
+          @distance_matrix.delete_at(1) 
+          @distance_matrix.delete_at(0)         
+        elsif cj==0
+          @distance_matrix.delete_at(ci-1) 
+          @distance_matrix.delete_at(0) 
+        else
+          @distance_matrix.delete_at(ci-1) 
+          @distance_matrix.delete_at(cj-1)
+        end
+        @distance_matrix.each do |d| 
+          d.delete_at(ci)
+          d.delete_at(cj)
+        end
+        @distance_matrix << distances_to_new_cluster
+      end
+      
+      # return distance between cluster cx and new cluster (ci U cj),
+      # using single linkage
+      def linkage_distance(cx, ci, cj)
+        [read_distance_matrix(cx, ci),
+          read_distance_matrix(cx, cj)].min
+      end
+      
       # cluster_a and cluster_b are removed from index_cluster, 
       # and a new cluster with all members of cluster_a and cluster_b
       # is added. 
-      # It returns the new clusters array.
-      def merge_clusters(clusters_to_merge, index_clusters)
-        index_a = clusters_to_merge.first
-        index_b = clusters_to_merge.last
+      # It modifies index clusters array.
+      def merge_clusters(index_a, index_b, index_clusters)
         index_a, index_b = index_b, index_a if index_b > index_a
         new_index_cluster = index_clusters[index_a] +
           index_clusters[index_b]
@@ -140,10 +168,9 @@ module Ai4r
       def get_closest_clusters(index_clusters)
         min_distance = 1.0/0
         closest_clusters = [1, 0]
-        index_clusters.each_with_index do |cluster_a, index_a|
+        index_clusters.each_index do |index_a|
           index_a.times do |index_b|
-            cluster_b = index_clusters[index_b]
-            cluster_distance = calc_index_clusters_distance(cluster_a, cluster_b)
+            cluster_distance = read_distance_matrix(index_a, index_b)
             if cluster_distance < min_distance
               closest_clusters = [index_a, index_b]
               min_distance = cluster_distance
@@ -153,22 +180,10 @@ module Ai4r
         return closest_clusters
       end
       
-      # Calculate cluster distance using the single linkage method
-      def calc_index_clusters_distance(cluster_a, cluster_b)
-        min_dist = 1.0/0
-        cluster_a.each do |index_a|
-          cluster_b.each do |index_b|
-            dist = read_distance_matrix(index_a, index_b)
-            min_dist = dist if dist < min_dist
-          end
-        end
-        return min_dist
-      end
-      
       def distance_between_item_and_cluster(data_item, cluster)
         min_dist = 1.0/0
         cluster.data_items.each do |another_item|
-          dist = distance(data_item, another_item)
+          dist = @distance_function.call(data_item, another_item)
           min_dist = dist if dist < min_dist
         end
         return min_dist

data/lib/ai4r/clusterers/ward_linkage.rb

@@ -0,0 +1,64 @@
+# Author::    Sergio Fierens (implementation)
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       http://ai4r.rubyforge.org/
+#
+# You can redistribute it and/or modify it under the terms of 
+# the Mozilla Public License version 1.1  as published by the 
+# Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
+
+require File.dirname(__FILE__) + '/../data/data_set'
+require File.dirname(__FILE__) + '/../clusterers/single_linkage'
+
+module Ai4r
+  module Clusterers
+     
+    # Implementation of an Agglomerative Hierarchical clusterer with 
+    # Ward's method linkage algorithm, aka the minimum variance method (Everitt
+    # et al., 2001 ; Jain and Dubes, 1988 ; Ward, 1963 ).
+    # Hierarchical clusteres create one cluster per element, and then 
+    # progressively merge clusters, until the required number of clusters
+    # is reached.
+    # The objective of this method is to minime the variance. 
+    # 
+    #   D(cx, (ci U cj)) =  (ni/(ni+nj+nx))*D(cx, ci) + 
+    #                       (nj/(ni+nj+nx))*D(cx, cj) - 
+    #                       (nx/(ni+nj)^2)*D(ci, cj)
+    class WardLinkage < SingleLinkage
+      
+    parameters_info :distance_function => 
+          "Custom implementation of distance function. " +
+          "It must be a closure receiving two data items and return the " +
+          "distance bewteen them. By default, this algorithm uses " + 
+          "ecuclidean distance of numeric attributes to the power of 2."
+      
+      # Build a new clusterer, using data examples found in data_set.
+      # Items will be clustered in "number_of_clusters" different
+      # clusters.
+      def build(data_set, number_of_clusters)
+        super
+      end
+      
+      # This algorithms does not allow classification of new data items 
+      # once it has been built. Rebuild the cluster including you data element.
+      def eval(data_item)
+        Raise "Eval of new data is not supported by this algorithm."
+      end
+      
+      protected
+       
+      # return distance between cluster cx and cluster (ci U cj),
+      # using ward's method linkage
+      def linkage_distance(cx, ci, cj)
+        ni = @index_clusters[ci].length
+        nj = @index_clusters[cj].length
+        nx = @index_clusters[cx].length
+        ( ( ( 1.0* (ni+nx) * read_distance_matrix(cx, ci) ) +
+            ( 1.0* (nj+nx) * read_distance_matrix(cx, cj) ) ) / (ni + nj + nx)  -
+            ( 1.0 * nx * read_distance_matrix(ci, cj) / (ni+nj)**2 ) ) 
+      end
+      
+    end
+  end
+end
+

data/lib/ai4r/clusterers/weighted_average_linkage.rb

@@ -0,0 +1,61 @@
+# Author::    Sergio Fierens (implementation)
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       http://ai4r.rubyforge.org/
+#
+# You can redistribute it and/or modify it under the terms of 
+# the Mozilla Public License version 1.1  as published by the 
+# Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
+
+require File.dirname(__FILE__) + '/../data/data_set'
+require File.dirname(__FILE__) + '/../clusterers/single_linkage'
+
+module Ai4r
+  module Clusterers
+     
+    # Implementation of an Agglomerative Hierarchical clusterer with 
+    # weighted average linkage algorithm, aka weighted pair group method 
+    # average or WPGMA (Jain and Dubes, 1988 ; McQuitty, 1966 )
+    # Hierarchical clusteres create one cluster per element, and then 
+    # progressively merge clusters, until the required number of clusters
+    # is reached.
+    # Similar to AverageLinkage, but the distances between clusters are 
+    # weighted based on the number of data items in each of them.
+    #   
+    #   D(cx, (ci U cj)) =  ( ni * D(cx, ci) + nj * D(cx, cj)) / (ni + nj)
+    class WeightedAverageLinkage < SingleLinkage
+      
+    parameters_info :distance_function => 
+          "Custom implementation of distance function. " +
+          "It must be a closure receiving two data items and return the " +
+          "distance bewteen them. By default, this algorithm uses " + 
+          "ecuclidean distance of numeric attributes to the power of 2."
+      
+      # Build a new clusterer, using data examples found in data_set.
+      # Items will be clustered in "number_of_clusters" different
+      # clusters.
+      def build(data_set, number_of_clusters)
+        super
+      end
+      
+      # This algorithms does not allow classification of new data items 
+      # once it has been built. Rebuild the cluster including you data element.
+      def eval(data_item)
+        Raise "Eval of new data is not supported by this algorithm."
+      end
+      
+      protected
+      
+      # return distance between cluster cx and cluster (ci U cj),
+      # using weighted average linkage
+      def linkage_distance(cx, ci, cj)
+        ni = @index_clusters[ci].length
+        nj = @index_clusters[cj].length
+        (1.0 * ni * read_distance_matrix(cx, ci)+
+          nj * read_distance_matrix(cx, cj))/(ni+nj)
+      end
+      
+    end
+  end
+end
+

data/lib/ai4r/data/constants.rb

@@ -0,0 +1,18 @@
+# Author::    Sergio Fierens
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       http://ai4r.rubyforge.org/
+#
+# You can redistribute it and/or modify it under the terms of 
+# the Mozilla Public License version 1.1  as published by the 
+# Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
+
+module Ai4r
+  module Data
+    
+    POSITIVE_INFINTY = 1.0/0
+    
+    NEGATIVE_INFINTY = -1.0/0
+    
+  end
+end

data/lib/ai4r/data/data_set.rb

@@ -41,7 +41,9 @@ module Ai4r
       # Retrieve a new DataSet, with the item(s) selected by the provided 
       # index. You can specify an index range, too.
       def [](index)
-        return DataSet.new(:data_items=>@data_items[index], 
+        selected_items = (index.is_a?(Fixnum)) ? 
+          [@data_items[index]] : @data_items[index]
+        return DataSet.new(:data_items => selected_items, 
           :data_labels =>@data_labels)
       end
       
@@ -173,7 +175,7 @@ module Ai4r
 
       # Add a data item to the data set
       def << data_item
-        if data_item.nil? || !data_item.is_a?(Array) || data_item.empty?
+        if data_item.nil? || !data_item.is_a?(Enumerable) || data_item.empty?
           raise ArgumentError,"Data must not be an non empty array."
         elsif @data_items.empty?
           set_data_items([data_item])
@@ -205,7 +207,7 @@ module Ai4r
       def check_data_items(data_items)
         if !data_items || data_items.empty?
           raise ArgumentError,"Examples data set must not be empty." 
-        elsif !data_items.first.is_a?(Array)
+        elsif !data_items.first.is_a?(Enumerable)
           raise ArgumentError,"Unkown format for example data." 
         end
         attributes_num = data_items.first.length

data/lib/ai4r/data/proximity.rb

@@ -74,6 +74,24 @@ module Ai4r
         return count
       end
       
+      # The "Simple matching" distance between two attribute sets is given 
+      # by the number of values present on both vectors.
+      # If sets a and b have lengths da and db then:
+      # 
+      #  S = 2/(da + db) * Number of values present on both sets
+      #  D = 1.0/S - 1
+      # 
+      # Some considerations: 
+      # * a and b must not include repeated items
+      # * all attributes are treated equally
+      # * all attributes are treated equally
+      def self.simple_matching_distance(a,b)
+        similarity = 0.0
+        a.each {|item| similarity += 2 if b.include?(item)}
+        similarity /= (a.length + b.length)
+        return 1.0/similarity - 1
+      end      
+      
     end
     
   end

data/test/clusterers/average_linkage_test.rb

@@ -1,9 +1,18 @@
+# Author::    Sergio Fierens (implementation)
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       http://ai4r.rubyforge.org/
+#
+# You can redistribute it and/or modify it under the terms of 
+# the Mozilla Public License version 1.1  as published by the 
+# Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
+
 require 'test/unit'
 require File.dirname(__FILE__) + '/../../lib/ai4r/clusterers/average_linkage'
 
 class Ai4r::Clusterers::AverageLinkage < Ai4r::Clusterers::SingleLinkage
   attr_accessor :data_set, :number_of_clusters, :clusters, :distance_matrix
-  public :calc_index_clusters_distance
+  public :linkage_distance
   public :distance_between_item_and_cluster
 end
 
@@ -28,18 +37,12 @@ class AverageLinkageTest < Test::Unit::TestCase
         [49.0, 49.0, 26.0, 5.0, 25.0, 49.0, 4.0, 29.0, 37.0, 5.0], 
         [2.0, 72.0, 65.0, 50.0, 52.0, 2.0, 65.0, 10.0, 74.0, 50.0, 37.0]]
   
-  def test_calc_index_clusters_distance
-    clusterer = AverageLinkage.new
+   def test_linkage_distance
+    clusterer = Ai4r::Clusterers::AverageLinkage.new
     clusterer.distance_matrix = @@expected_distance_matrix
-    assert_equal 98.0, clusterer.calc_index_clusters_distance([0], [1])
-    assert_equal 43.0, clusterer.calc_index_clusters_distance([0, 1], [3, 4])
+    assert_equal 93.5, clusterer.linkage_distance(0,1,2)
+    assert_equal 37.5, clusterer.linkage_distance(4,2,5)
   end
 
-  def test_distance_between_item_and_cluster
-    clusterer = AverageLinkage.new
-    assert_equal 20.0, clusterer.distance_between_item_and_cluster([1,2], 
-      DataSet.new(:data_items => [[3,4],[5,6]]))
-  end
-  
 end