ai4r 1.5 → 1.6

data/examples/clusterers/simple_website_clustering.rb

@@ -0,0 +1,47 @@
+require File.dirname(__FILE__) + '/google_search'
+require File.dirname(__FILE__) + '/build_keywords'
+require File.dirname(__FILE__) + '/../../lib/ai4r/clusterers/average_linkage'
+require 'rubygems'
+require 'hpricot'
+require 'net/http'
+require 'benchmark'
+
+SITES_TO_CLASSIFY = [
+  "www.foxnews.com", "www.usatoday.com", "scm.jadeferret.com",
+  "www.accurev.com", "www.lastminute.com", "subversion.tigris.org",
+  "news.yahoo.com", "news.bbc.co.uk", "www.orbitz.com"
+]
+  
+# Return array of keywords for the site
+def get_keywords(site)
+  response = Net::HTTP.get_response(site, "/")
+  Hpricot(response.body).
+    search("meta[@name='keywords']")[0]. #Select meta keywords element
+    attributes["content"].               #Select its content
+    split(",").                          #Keywords are coma separated
+    collect{ |k| k.strip.downcase }      #Remove start and end white spaces
+end
+
+# Get keywords data for each website
+Site = Struct.new("Site", :name, :keywords)
+sites = SITES_TO_CLASSIFY.collect do |site_name|
+  Site.new(site_name, get_keywords(site_name))
+end
+data_set = Ai4r::Data::DataSet.new(:data_items => sites, 
+  :data_labels => Site.members)
+
+# The distance between sites depends on the keywords collected from internet 
+keywords_distance_function = lambda do |x,y| 
+  return Ai4r::Data::Proximity.simple_matching(x.keyword, y.keywords)
+end
+
+# Create the clusters
+clusterer = Ai4r::Clusterers::AverageLinkage.new
+clusterer.distance_function = keywords_distance_function
+clusterer.build(data_set, 3)
+
+# Print results
+clusterer.clusters.each do |cluster|
+  puts cluster.data_items.collect {|item| item.name}.join(", ")
+  puts "============"
+end

data/lib/ai4r.rb

@@ -1,6 +1,7 @@
 # Data
 require "ai4r/data/data_set"
 require "ai4r/data/statistics"
+require "ai4r/data/proximity"
 require "ai4r/data/parameterizable"
 # Clusterers
 require "ai4r/clusterers/clusterer"
@@ -9,12 +10,18 @@ require "ai4r/clusterers/bisecting_k_means"
 require "ai4r/clusterers/single_linkage"
 require "ai4r/clusterers/complete_linkage"
 require "ai4r/clusterers/average_linkage"
+require "ai4r/clusterers/weighted_average_linkage"
+require "ai4r/clusterers/centroid_linkage"
+require "ai4r/clusterers/median_linkage"
+require "ai4r/clusterers/ward_linkage"
+require "ai4r/clusterers/diana"
 # Classifiers
 require "ai4r/classifiers/classifier"
 require "ai4r/classifiers/id3"
 require "ai4r/classifiers/prism"
 require "ai4r/classifiers/one_r"
 require "ai4r/classifiers/zero_r"
+require "ai4r/classifiers/hyperpipes"
 # Neural networks
 require "ai4r/neural_network/backpropagation"
 # Genetic Algorithms

data/lib/ai4r/classifiers/hyperpipes.rb

@@ -0,0 +1,118 @@
+# Author::    Sergio Fierens (Implementation only)
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       http://ai4r.rubyforge.org/
+#
+# You can redistribute it and/or modify it under the terms of 
+# the Mozilla Public License version 1.1  as published by the 
+# Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
+
+require 'set'
+require File.dirname(__FILE__) + '/../data/constants'
+require File.dirname(__FILE__) + '/../data/data_set'
+require File.dirname(__FILE__) + '/../classifiers/classifier'
+
+module Ai4r
+  module Classifiers
+
+    include Ai4r::Data
+    
+    # = Introduction
+    # 
+    # A fast classifier algorithm, created by Lucio de Souza Coelho 
+    # and Len Trigg.
+    class Hyperpipes < Classifier
+      
+      attr_reader :data_set, :pipes
+
+      # Build a new Hyperpipes classifier. You must provide a DataSet instance
+      # as parameter. The last attribute of each item is considered as 
+      # the item class.
+      def build(data_set)
+        @data_set = data_set
+        @domains = data_set.build_domains
+        
+        @pipes = {}
+        @domains.last.each {|cat| @pipes[cat] = build_pipe(@domains)}
+        @data_set.data_item.each {|item| update_pipe(@pipes[item.last], item) }
+        
+        return self
+      end
+      
+      # You can evaluate new data, predicting its class.
+      # e.g.
+      #   classifier.eval(['New York',  '<30', 'F'])  # => 'Y'      
+      def eval(data)
+        votes = Hash.new {0}
+        @pipes.each do |category, pipe|
+          pipe.each_with_index do |bounds, i|
+            if data[i].is_a? Numeric
+              votes[category]+=1 if data[i]>bounds[:min] && data[i]<bounds[:max]
+            else
+              votes[category]+=1 if bounds[data[i]]
+            end
+          end
+        end
+        return votes.to_a.max {|x, y| x.last <=> y.last}.first
+      end
+      
+      # This method returns the generated rules in ruby code.
+      # e.g.
+      #   
+      #   classifier.get_rules
+      #     # =>  if age_range == '<30' then marketing_target = 'Y'
+      #           elsif age_range == '[30-50)' then marketing_target = 'N'
+      #           elsif age_range == '[50-80]' then marketing_target = 'N'
+      #           end
+      #
+      # It is a nice way to inspect induction results, and also to execute them:  
+      #     marketing_target = nil
+      #     eval classifier.get_rules   
+      #     puts marketing_target
+      #       # =>  'Y'
+      def get_rules
+        rules = []
+        rules << "votes = Hash.new {0}"
+        data = @data_set.data_items.first
+        labels = @data_set.data_labels.collect {|l| l.to_s}
+        @pipes.each do |category, pipe|
+          pipe.each_with_index do |bounds, i|
+            rule = "votes['#{category}'] += 1 "
+            if data[i].is_a? Numeric
+              rule += "if #{labels[i]} > #{bounds[:min]} && #{labels[i]} < #{bounds[:max]}"
+            else
+              rule += "if #{bounds.inspect}['#{labels[i]}']"
+            end
+            rules << rule
+          end
+        end
+        rules << "votes.to_a.max {|x, y| x.last <=> y.last}.first"
+        return rules.join('\n')
+      end
+      
+      protected
+
+      def build_pipe(data_set)
+        data_set.data_items.first[0...-1].collect do |att|
+          if att.is_a? Numeric
+            {:min=>POSITIVE_INFINITY, :max=>NEGATIVE_INFINITY}
+          else
+            Hash.new(false)
+          end
+        end
+      end
+      
+      def update_pipe(pipe, data_item)
+        data_item[0...-1].each_with_index do |att, i|
+          if att.first.is_a? Numeric
+            pipe[i][:min] = att if att < pipe[i][:min]
+            pipe[i][:max] = att if att > pipe[i][:max]
+          else
+            pipe[i][att] = true
+          end  
+        end
+      end
+      
+    end
+  end
+end

data/lib/ai4r/clusterers/average_linkage.rb

@@ -13,14 +13,25 @@ require File.dirname(__FILE__) + '/../clusterers/single_linkage'
 module Ai4r
   module Clusterers
      
-    # Implementation of a Hierarchical clusterer with complete linkage.
+    # Implementation of a Hierarchical clusterer with group average
+    # linkage, AKA unweighted pair group method average or UPGMA (Everitt 
+    # et al., 2001 ; Jain and Dubes, 1988 ; Sokal and Michener, 1958).
     # Hierarchical clusteres create one cluster per element, and then 
     # progressively merge clusters, until the required number of clusters
     # is reached.
-    # With average linkage, the distance between two clusters is computed as 
-    # the average distance between elements of each cluster.
+    # With average linkage, the distance between a clusters cx and 
+    # cluster (ci U cj) the the average distance between cx and ci, and
+    # cx and cj.
+    #
+    #   D(cx, (ci U cj) = (D(cx, ci) + D(cx, cj)) / 2
     class AverageLinkage < SingleLinkage
       
+      parameters_info :distance_function => 
+          "Custom implementation of distance function. " +
+          "It must be a closure receiving two data items and return the " +
+          "distance bewteen them. By default, this algorithm uses " + 
+          "ecuclidean distance of numeric attributes to the power of 2."
+      
       # Build a new clusterer, using data examples found in data_set.
       # Items will be clustered in "number_of_clusters" different
       # clusters.
@@ -28,31 +39,19 @@ module Ai4r
         super
       end
       
-      # Classifies the given data item, returning the cluster index it belongs 
-      # to (0-based).
+      # This algorithms does not allow classification of new data items 
+      # once it has been built. Rebuild the cluster including you data element.
       def eval(data_item)
-        super
+        Raise "Eval of new data is not supported by this algorithm."
       end
       
       protected
       
-      # Calculate cluster distance using the average linkage method
-      def calc_index_clusters_distance(cluster_a, cluster_b)
-        dist_sum = 0.0
-        cluster_a.each do |index_a|
-          cluster_b.each do |index_b|
-            dist_sum += read_distance_matrix(index_a, index_b)
-            end
-        end
-        return dist_sum/(cluster_a.length*cluster_b.length)
-      end
-      
-      def distance_between_item_and_cluster(data_item, cluster)
-        dist_sum = 0.0
-        cluster.data_items.each do |another_item|
-          dist_sum += distance(data_item, another_item)
-        end
-        return dist_sum/cluster.data_items.length
+      # return distance between cluster cx and cluster (ci U cj),
+      # using average linkage
+      def linkage_distance(cx, ci, cj)
+        (read_distance_matrix(cx, ci)+
+          read_distance_matrix(cx, cj))/2
       end
       
     end

data/lib/ai4r/clusterers/centroid_linkage.rb

@@ -0,0 +1,66 @@
+# Author::    Sergio Fierens (implementation)
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       http://ai4r.rubyforge.org/
+#
+# You can redistribute it and/or modify it under the terms of 
+# the Mozilla Public License version 1.1  as published by the 
+# Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
+
+require File.dirname(__FILE__) + '/../data/data_set'
+require File.dirname(__FILE__) + '/../clusterers/single_linkage'
+
+module Ai4r
+  module Clusterers
+     
+    # Implementation of an Agglomerative Hierarchical clusterer with 
+    # centroid linkage algorithm, aka unweighted pair group method 
+    # centroid (UPGMC) (Everitt et al., 2001 ; Jain and Dubes, 1988 ;
+    # Sokal and Michener, 1958 )
+    # Hierarchical clusteres create one cluster per element, and then 
+    # progressively merge clusters, until the required number of clusters
+    # is reached.
+    # The distance between clusters is the squared euclidean distance 
+    # between their centroids. 
+    # 
+    #   D(cx, (ci U cj)) = | mx - mij |^2
+    #   D(cx, (ci U cj)) =  (ni/(ni+nj))*D(cx, ci) + 
+    #                       (nj/(ni+nj))*D(cx, cj) - 
+    #                       (ni*nj/(ni+nj)^2)*D(ci, cj)
+    class CentroidLinkage < SingleLinkage
+      
+    parameters_info :distance_function => 
+          "Custom implementation of distance function. " +
+          "It must be a closure receiving two data items and return the " +
+          "distance bewteen them. By default, this algorithm uses " + 
+          "ecuclidean distance of numeric attributes to the power of 2."
+      
+      # Build a new clusterer, using data examples found in data_set.
+      # Items will be clustered in "number_of_clusters" different
+      # clusters.
+      def build(data_set, number_of_clusters)
+        super
+      end
+      
+      # This algorithms does not allow classification of new data items 
+      # once it has been built. Rebuild the cluster including you data element.
+      def eval(data_item)
+        Raise "Eval of new data is not supported by this algorithm."
+      end
+      
+      protected
+       
+      # return distance between cluster cx and cluster (ci U cj),
+      # using centroid linkage
+      def linkage_distance(cx, ci, cj)
+        ni = @index_clusters[ci].length
+        nj = @index_clusters[cj].length
+        ( ni * read_distance_matrix(cx, ci) +
+          nj * read_distance_matrix(cx, cj) -
+         1.0 * ni * nj * read_distance_matrix(ci, cj) / (ni+nj)) / (ni+nj)
+      end
+      
+    end
+  end
+end
+

data/lib/ai4r/clusterers/complete_linkage.rb

@@ -13,14 +13,24 @@ require File.dirname(__FILE__) + '/../clusterers/single_linkage'
 module Ai4r
   module Clusterers
      
-    # Implementation of a Hierarchical clusterer with complete linkage.
+    # Implementation of a Hierarchical clusterer with complete linkage (Everitt 
+    # et al., 2001 ; Jain and Dubes, 1988 ; Sorensen, 1948 ).
     # Hierarchical clusteres create one cluster per element, and then 
     # progressively merge clusters, until the required number of clusters
     # is reached.
     # With complete linkage, the distance between two clusters is computed as 
     # the maximum distance between elements of each cluster.
+    #
+    #   D(cx, (ci U cj) = max(D(cx, ci), D(cx, cj))
     class CompleteLinkage < SingleLinkage
       
+      parameters_info :distance_function => 
+          "Custom implementation of distance function. " +
+          "It must be a closure receiving two data items and return the " +
+          "distance bewteen them. By default, this algorithm uses " + 
+          "ecuclidean distance of numeric attributes to the power of 2."
+      
+      
       # Build a new clusterer, using data examples found in data_set.
       # Items will be clustered in "number_of_clusters" different
       # clusters.
@@ -36,22 +46,17 @@ module Ai4r
       
       protected
       
-      # Calculate cluster distance using the complete linkage method
-      def calc_index_clusters_distance(cluster_a, cluster_b)
-        max_dist = 0
-        cluster_a.each do |index_a|
-          cluster_b.each do |index_b|
-            dist = read_distance_matrix(index_a, index_b)
-            max_dist = dist if dist > max_dist
-          end
-        end
-        return max_dist
+      # return distance between cluster cx and new cluster (ci U cj),
+      # using complete linkage
+      def linkage_distance(cx, ci, cj)
+        [read_distance_matrix(cx, ci),
+          read_distance_matrix(cx, cj)].max
       end
       
       def distance_between_item_and_cluster(data_item, cluster)
         max_dist = 0
         cluster.data_items.each do |another_item|
-          dist = distance(data_item, another_item)
+          dist = @distance_function.call(data_item, another_item)
           max_dist = dist if dist > max_dist
         end
         return max_dist

data/lib/ai4r/clusterers/diana.rb

@@ -0,0 +1,139 @@
+# Author::    Sergio Fierens (implementation)
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       http://ai4r.rubyforge.org/
+#
+# You can redistribute it and/or modify it under the terms of 
+# the Mozilla Public License version 1.1  as published by the 
+# Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
+
+require File.dirname(__FILE__) + '/../data/data_set'
+require File.dirname(__FILE__) + '/../data/proximity'
+require File.dirname(__FILE__) + '/../clusterers/clusterer'
+
+module Ai4r
+  module Clusterers
+     
+    # DIANA (Divisive ANAlysis) (Kaufman and Rousseeuw, 1990; 
+    # Macnaughton - Smith et al. 1964) is a Divisive Hierarchical
+    # Clusterer. It begins with only one cluster with all data items,
+    # and divides the clusters until the desired clusters number is reached.
+    class Diana < Clusterer
+      
+      attr_reader :data_set, :number_of_clusters, :clusters
+      
+      parameters_info :distance_function => 
+          "Custom implementation of distance function. " +
+          "It must be a closure receiving two data items and return the " +
+          "distance bewteen them. By default, this algorithm uses " + 
+          "ecuclidean distance of numeric attributes to the power of 2."
+      
+      def initialize
+        @distance_function = lambda do |a,b| 
+            Ai4r::Data::Proximity.squared_euclidean_distance(
+              a.select {|att_a| att_a.is_a? Numeric} , 
+              b.select {|att_b| att_b.is_a? Numeric})
+          end
+      end
+      
+      # Build a new clusterer, using divisive analysis (DIANA algorithm)
+      def build(data_set, number_of_clusters)
+        @data_set = data_set
+        @number_of_clusters = number_of_clusters
+        @clusters = [@data_set[0..-1]]
+        
+        while(@clusters.length < @number_of_clusters)
+          cluster_index_to_split = max_diameter_cluster(@clusters)
+          cluster_to_split = @clusters[cluster_index_to_split]
+          splinter_cluster = init_splinter_cluster(cluster_to_split)
+          while true
+            dist_diff, index = max_distance_difference(cluster_to_split, splinter_cluster)
+            break if dist_diff < 0
+            splinter_cluster << cluster_to_split.data_items[index]
+            cluster_to_split.data_items.delete_at(index)
+          end
+          @clusters << splinter_cluster
+        end
+       
+        return self
+      end
+      
+      # Classifies the given data item, returning the cluster index it belongs 
+      # to (0-based).
+      def eval(data_item)
+        get_min_index(@clusters.collect do |cluster|
+          distance_sum(data_item, cluster) / cluster.data_items.length
+          end)
+      end
+      
+      protected
+      
+      # return the cluster with max diameter
+      def max_diameter_cluster(clusters)
+        max_index = 0
+        max_diameter = 0
+        clusters.each_with_index do |cluster, index|
+          diameter = cluster_diameter(cluster)
+          if diameter > max_diameter
+            max_index = index
+            max_diameter = diameter
+          end
+        end
+        return max_index
+      end
+      
+      # Max distance between 2 items in a cluster
+      def cluster_diameter(cluster)
+        diameter = 0
+        cluster.data_items.each_with_index do |item_a, item_a_pos|
+          item_a_pos.times do |item_b_pos|
+            d = @distance_function.call(item_a, cluster.data_items[item_b_pos])
+            diameter = d if d > diameter
+          end
+        end
+        return diameter
+      end
+      
+      # Create a cluster with the item with mx distance
+      # to the rest of the cluster's items.
+      # That item is removed from the initial cluster.
+      def init_splinter_cluster(cluster_to_split)
+        max = 0.0
+        max_index = 0
+        cluster_to_split.data_items.each_with_index do |item, index|
+          sum = distance_sum(item, cluster_to_split)
+          max, max_index = sum, index if sum > max
+        end
+        splinter_cluster = cluster_to_split[max_index]
+        cluster_to_split.data_items.delete_at(max_index)
+        return splinter_cluster
+      end
+      
+      # Return the max average distance between any item of 
+      # cluster_to_split and the rest of items in that cluster,
+      # minus the average distance with the items of splinter_cluster,
+      # and the index of the item.
+      # A positive value means that the items is closer to the
+      # splinter group than to its current cluster.
+      def max_distance_difference(cluster_to_split, splinter_cluster)
+        max_diff = -1.0/0
+        max_diff_index = 0
+        cluster_to_split.data_items.each_with_index do |item, index|
+          dist_a = distance_sum(item, cluster_to_split) / (cluster_to_split.data_items.length-1)
+          dist_b = distance_sum(item, splinter_cluster) / (splinter_cluster.data_items.length)
+          dist_diff = dist_a - dist_b
+          max_diff, max_diff_index = dist_diff, index if dist_diff > max_diff
+        end
+        return max_diff, max_diff_index
+      end
+      
+      # Sum up the distance between an item and all the items in a cluster
+      def distance_sum(item_a, cluster)
+        cluster.data_items.inject(0.0) do |sum, item_b|
+          sum + @distance_function.call(item_a, item_b)
+        end
+      end
+      
+    end
+  end
+end