ai4ruby 1.11

data/lib/ai4r/classifiers/hyperpipes.rb

@@ -0,0 +1,118 @@
+# Author::    Sergio Fierens (Implementation only)
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       http://ai4r.rubyforge.org/
+#
+# You can redistribute it and/or modify it under the terms of 
+# the Mozilla Public License version 1.1  as published by the 
+# Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
+
+require 'set'
+require File.dirname(__FILE__) + '/../data/data_set'
+require File.dirname(__FILE__) + '/../classifiers/classifier'
+
+module Ai4r
+  module Classifiers
+
+    include Ai4r::Data
+    
+    # = Introduction
+    # 
+    # A fast classifier algorithm, created by Lucio de Souza Coelho 
+    # and Len Trigg.
+    class Hyperpipes < Classifier
+      
+      attr_reader :data_set, :pipes
+
+      # Build a new Hyperpipes classifier. You must provide a DataSet instance
+      # as parameter. The last attribute of each item is considered as 
+      # the item class.
+      def build(data_set)
+        data_set.check_not_empty
+        @data_set = data_set
+        @domains = data_set.build_domains
+        
+        @pipes = {}
+        @domains.last.each {|cat| @pipes[cat] = build_pipe(@data_set)}
+        @data_set.data_items.each {|item| update_pipe(@pipes[item.last], item) }
+        
+        return self
+      end
+      
+      # You can evaluate new data, predicting its class.
+      # e.g.
+      #   classifier.eval(['New York',  '<30', 'F'])  # => 'Y'      
+      def eval(data)
+        votes = Hash.new {0}
+        @pipes.each do |category, pipe|
+          pipe.each_with_index do |bounds, i|
+            if data[i].is_a? Numeric
+              votes[category]+=1 if data[i]>=bounds[:min] && data[i]<=bounds[:max]
+            else
+              votes[category]+=1 if bounds[data[i]]
+            end
+          end
+        end
+        return votes.to_a.max {|x, y| x.last <=> y.last}.first
+      end
+      
+      # This method returns the generated rules in ruby code.
+      # e.g.
+      #   
+      #   classifier.get_rules
+      #     # =>  if age_range == '<30' then marketing_target = 'Y'
+      #           elsif age_range == '[30-50)' then marketing_target = 'N'
+      #           elsif age_range == '[50-80]' then marketing_target = 'N'
+      #           end
+      #
+      # It is a nice way to inspect induction results, and also to execute them:  
+      #     marketing_target = nil
+      #     eval classifier.get_rules   
+      #     puts marketing_target
+      #       # =>  'Y'
+      def get_rules
+        rules = []
+        rules << "votes = Hash.new {0}"
+        data = @data_set.data_items.first
+        labels = @data_set.data_labels.collect {|l| l.to_s}
+        @pipes.each do |category, pipe|
+          pipe.each_with_index do |bounds, i|
+            rule = "votes['#{category}'] += 1 "
+            if data[i].is_a? Numeric
+              rule += "if #{labels[i]} >= #{bounds[:min]} && #{labels[i]} <= #{bounds[:max]}"
+            else
+              rule += "if #{bounds.inspect}[#{labels[i]}]"
+            end
+            rules << rule
+          end
+        end
+        rules << "#{labels.last} = votes.to_a.max {|x, y| x.last <=> y.last}.first"
+        return rules.join("\n")
+      end
+      
+      protected
+
+      def build_pipe(data_set)
+        data_set.data_items.first[0...-1].collect do |att|
+          if att.is_a? Numeric
+            {:min=>1.0/0, :max=>-1.0/0}
+          else
+            Hash.new(false)
+          end
+        end
+      end
+      
+      def update_pipe(pipe, data_item)
+        data_item[0...-1].each_with_index do |att, i|
+          if att.is_a? Numeric
+            pipe[i][:min] = att if att < pipe[i][:min]
+            pipe[i][:max] = att if att > pipe[i][:max]
+          else
+            pipe[i][att] = true
+          end  
+        end
+      end
+      
+    end
+  end
+end

data/lib/ai4r/classifiers/ib1.rb

@@ -0,0 +1,121 @@
+# Author::    Sergio Fierens (Implementation only)
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       http://ai4r.rubyforge.org/
+#
+# You can redistribute it and/or modify it under the terms of 
+# the Mozilla Public License version 1.1  as published by the 
+# Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
+
+require 'set'
+require File.dirname(__FILE__) + '/../data/data_set'
+require File.dirname(__FILE__) + '/../classifiers/classifier'
+
+module Ai4r
+  module Classifiers
+
+    # = Introduction
+    # 
+    # IB1 algorithm implementation.
+    # IB1 is the simplest instance-based learning (IBL) algorithm.
+    #
+    # D. Aha, D. Kibler (1991). Instance-based learning algorithms.
+    # Machine Learning. 6:37-66.
+    #
+    # IBI is identical to the nearest neighbor algorithm except that
+    # it normalizes its attributes' ranges, processes instances
+    # incrementally, and has a simple policy for tolerating missing values
+    class IB1 < Classifier
+      
+      attr_reader :data_set
+
+      # Build a new IB1 classifier. You must provide a DataSet instance
+      # as parameter. The last attribute of each item is considered as 
+      # the item class.
+      def build(data_set)
+        data_set.check_not_empty
+        @data_set = data_set
+        @min_values = Array.new(data_set.data_labels.length)
+        @max_values = Array.new(data_set.data_labels.length)
+        data_set.data_items.each { |data_item| update_min_max(data_item[0...-1]) }
+        return self
+      end
+      
+      # You can evaluate new data, predicting its class.
+      # e.g.
+      #   classifier.eval(['New York',  '<30', 'F'])  # => 'Y'      
+      def eval(data)
+        update_min_max(data)
+        min_distance = 1.0/0
+        klass = nil
+        @data_set.data_items.each do |train_item|
+          d = distance(data, train_item)
+          if d < min_distance
+            min_distance = d
+            klass = train_item.last
+          end
+        end
+        return klass
+      end
+      
+      protected
+
+      # We keep in the state the min and max value of each attribute,
+      # to provide normalized distances between to values of a numeric attribute
+      def update_min_max(atts)
+        atts.each_with_index do |att, i|
+          if att && att.is_a?(Numeric)
+            @min_values[i] = att if @min_values[i].nil? || @min_values[i] > att
+            @max_values[i] = att if @max_values[i].nil? || @max_values[i] < att
+          end
+        end
+      end
+
+      # Normalized distance between 2 instances
+      #
+      #
+      # Returns sum of
+      #  * squared difference between normalized numeric att values
+      #  * 1 for nominal atts which differs or one is missing
+      #  * 1 if both atts are missing
+      #  * normalized numeric att value if other att value is missing and > 0.5
+      #  * 1.0-normalized numeric att value if other att value is missing and < 0.5
+      def distance(a, b)
+        d = 0
+        a.each_with_index do |att_a, i|
+          att_b = b[i]
+          if att_a.nil?
+            if att_b.is_a? Numeric
+              diff = norm(att_b, i)
+              diff = 1.0 - diff if diff < 0.5
+            else
+              diff = 1
+            end
+          elsif att_a.is_a? Numeric
+            if att_b.is_a? Numeric
+              diff = norm(att_a, i) - norm(att_b, i);
+            else
+              diff = norm(att_a, i)
+              diff = 1.0 - diff if diff < 0.5
+            end
+          elsif att_a != att_b
+            diff = 1
+          else
+            diff = 0
+          end
+          d += diff * diff
+        end
+        return d
+      end
+
+      # Returns normalized value att
+      #
+      # index is the index of the attribute in the instance.
+      def norm(att, index)
+        return 0 if @min_values[index].nil?
+        return 1.0*(att - @min_values[index]) / (@max_values[index] -@min_values[index]);
+      end
+      
+    end
+  end
+end

data/lib/ai4r/classifiers/id3.rb

@@ -0,0 +1,326 @@
+# Author::    Sergio Fierens (Implementation, Quinlan is 
+# the creator of the algorithm)
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       http://ai4r.rubyforge.org/
+#
+# You can redistribute it and/or modify it under the terms of 
+# the Mozilla Public License version 1.1  as published by the 
+# Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
+
+require File.dirname(__FILE__) + '/../data/data_set'
+require File.dirname(__FILE__) + '/../classifiers/classifier'
+
+module Ai4r
+  
+  module Classifiers
+
+    # = Introduction
+    # This is an implementation of the ID3 algorithm (Quinlan) 
+    # Given a set of preclassified examples, it builds a top-down 
+    # induction of decision tree, biased by the information gain and 
+    # entropy measure.
+    #
+    # * http://en.wikipedia.org/wiki/Decision_tree
+    # * http://en.wikipedia.org/wiki/ID3_algorithm
+    #
+    # = How to use it
+    #   
+    #   DATA_LABELS = [ 'city', 'age_range', 'gender', 'marketing_target'  ]
+    #
+    #   DATA_ITEMS = [  
+    #          ['New York',  '<30',      'M', 'Y'],
+    #          ['Chicago',     '<30',      'M', 'Y'],
+    #          ['Chicago',     '<30',      'F', 'Y'],
+    #          ['New York',  '<30',      'M', 'Y'],
+    #          ['New York',  '<30',      'M', 'Y'],
+    #          ['Chicago',     '[30-50)',  'M', 'Y'],
+    #          ['New York',  '[30-50)',  'F', 'N'],
+    #          ['Chicago',     '[30-50)',  'F', 'Y'],
+    #          ['New York',  '[30-50)',  'F', 'N'],
+    #          ['Chicago',     '[50-80]', 'M', 'N'],
+    #          ['New York',  '[50-80]', 'F', 'N'],
+    #          ['New York',  '[50-80]', 'M', 'N'],
+    #          ['Chicago',     '[50-80]', 'M', 'N'],
+    #          ['New York',  '[50-80]', 'F', 'N'],
+    #          ['Chicago',     '>80',      'F', 'Y']
+    #        ]
+    #   
+    #   data_set = DataSet.new(:data_items=>DATA_SET, :data_labels=>DATA_LABELS)
+    #   id3 = Ai4r::Classifiers::ID3.new.build(data_set)
+    #   
+    #   id3.get_rules
+    #     # =>  if age_range=='<30' then marketing_target='Y'
+    #           elsif age_range=='[30-50)' and city=='Chicago' then marketing_target='Y'
+    #           elsif age_range=='[30-50)' and city=='New York' then marketing_target='N'
+    #           elsif age_range=='[50-80]' then marketing_target='N'
+    #           elsif age_range=='>80' then marketing_target='Y'
+    #           else raise 'There was not enough information during training to do a proper induction for this data element' end
+    #   
+    #   id3.eval(['New York', '<30', 'M'])
+    #     # =>  'Y'
+    #   
+    # = A better way to load the data  
+    # 
+    # In the real life you will use lot more data training examples, with more
+    # attributes. Consider moving your data to an external CSV (comma separate 
+    # values) file.
+    #                 
+    #   data_file = "#{File.dirname(__FILE__)}/data_set.csv"
+    #   data_set = DataSet.load_csv_with_labels data_file
+    #   id3 = Ai4r::Classifiers::ID3.new.build(data_set)      
+    #   
+    # = A nice tip for data evaluation
+    # 
+    #   id3 = Ai4r::Classifiers::ID3.new.build(data_set)
+    #
+    #   age_range = '<30'
+    #   marketing_target = nil
+    #   eval id3.get_rules   
+    #   puts marketing_target
+    #     # =>  'Y'  
+    #
+    # = More about ID3 and decision trees
+    # 
+    # * http://en.wikipedia.org/wiki/Decision_tree
+    # * http://en.wikipedia.org/wiki/ID3_algorithm
+    #   
+    # = About the project
+    # Author::    Sergio Fierens
+    # License::   MPL 1.1
+    # Url::       http://ai4r.rubyforge.org/
+    class ID3 < Classifier
+      
+      attr_reader :data_set
+       
+      # Create a new ID3 classifier. You must provide a DataSet instance
+      # as parameter. The last attribute of each item is considered as the
+      # item class.
+      def build(data_set)
+        data_set.check_not_empty
+        @data_set = data_set
+        preprocess_data(@data_set.data_items)
+        return self
+      end
+
+      # You can evaluate new data, predicting its category.
+      # e.g.
+      #   id3.eval(['New York',  '<30', 'F'])  # => 'Y'
+      def eval(data)
+        @tree.value(data) if @tree
+      end
+
+      # This method returns the generated rules in ruby code.
+      # e.g.
+      #   
+      #   id3.get_rules
+      #     # =>  if age_range=='<30' then marketing_target='Y'
+      #           elsif age_range=='[30-50)' and city=='Chicago' then marketing_target='Y'
+      #           elsif age_range=='[30-50)' and city=='New York' then marketing_target='N'
+      #           elsif age_range=='[50-80]' then marketing_target='N'
+      #           elsif age_range=='>80' then marketing_target='Y'
+      #           else raise 'There was not enough information during training to do a proper induction for this data element' end
+      #
+      # It is a nice way to inspect induction results, and also to execute them:  
+      #     age_range = '<30'
+      #     marketing_target = nil
+      #     eval id3.get_rules   
+      #     puts marketing_target
+      #       # =>  'Y'
+      def get_rules
+        #return "Empty ID3 tree" if !@tree
+        rules = @tree.get_rules
+        rules = rules.collect do |rule|
+            "#{rule[0..-2].join(' and ')} then #{rule.last}"
+        end
+        return "if #{rules.join("\nelsif ")}\nelse raise 'There was not enough information during training to do a proper induction for this data element' end"
+      end
+
+      private
+      def preprocess_data(data_examples)
+        @tree = build_node(data_examples)
+      end
+
+      private
+      def build_node(data_examples, flag_att = [])
+        return ErrorNode.new if data_examples.length == 0
+        domain = domain(data_examples)   
+        return CategoryNode.new(@data_set.data_labels.last, domain.last[0]) if domain.last.length == 1
+        min_entropy_index = min_entropy_index(data_examples, domain, flag_att)
+        flag_att << min_entropy_index
+        split_data_examples = split_data_examples(data_examples, domain, min_entropy_index)
+        return CategoryNode.new(@data_set.data_labels.last, most_freq(data_examples, domain)) if split_data_examples.length == 1
+        nodes = split_data_examples.collect do |partial_data_examples|  
+          build_node(partial_data_examples, flag_att)
+        end
+        return EvaluationNode.new(@data_set.data_labels, min_entropy_index, domain[min_entropy_index], nodes)
+      end
+
+      private 
+      def self.sum(values)
+        values.inject( 0 ) { |sum,x| sum+x }
+      end
+
+      private
+      def self.log2(z)
+        return 0.0 if z == 0
+        Math.log(z)/LOG2
+      end
+
+      private       
+      def most_freq(examples, domain)
+        freqs = []
+        domain.last.length.times { freqs << 0}
+        examples.each do |example|
+          cat_index = domain.last.index(example.last)
+          freq = freqs[cat_index] + 1
+          freqs[cat_index] = freq
+        end
+        max_freq = freqs.max
+        max_freq_index = freqs.index(max_freq)
+        domain.last[max_freq_index]
+      end
+
+      private
+      def split_data_examples(data_examples, domain, att_index)
+        data_examples_array = []
+        att_value_examples = {}
+        data_examples.each do |example|
+          example_set = att_value_examples[example[att_index]]
+          example_set = [] if !example_set
+          example_set << example
+          att_value_examples.store(example[att_index], example_set)
+        end
+        att_value_examples.each_pair do |att_value, example_set|
+           att_value_index = domain[att_index].index(att_value)
+           data_examples_array[att_value_index] = example_set
+        end
+        return data_examples_array
+      end
+
+      private 
+      def min_entropy_index(data_examples, domain, flag_att=[])
+        min_entropy = nil
+        min_index = 0
+        domain[0..-2].each_index do |index|
+          freq_grid = freq_grid(index, data_examples, domain)
+          entropy = entropy(freq_grid, data_examples.length)
+          if (!min_entropy || entropy < min_entropy) && !flag_att.include?(index)
+            min_entropy = entropy 
+            min_index = index 
+          end
+        end
+        return min_index
+      end
+
+      private
+      def domain(data_examples)
+        #return build_domains(data_examples)
+        domain = []
+        @data_set.data_labels.length.times { domain << [] }
+        data_examples.each do |data|
+          data.each_index do |i|
+            domain[i] << data[i] if i<domain.length && !domain[i].include?(data[i])
+          end
+        end
+        return domain
+      end
+       
+      private 
+      def freq_grid(att_index, data_examples, domain)
+        #Initialize empty grid
+        grid_element = []
+        domain.last.length.times { grid_element << 0} 
+        grid = [] 
+        domain[att_index].length.times { grid << grid_element.clone }
+        #Fill frecuency with grid
+        data_examples.each do |example|
+          att_val = example[att_index]
+          att_val_index = domain[att_index].index(att_val)
+          category = example.last
+          category_index = domain.last.index(category)
+          freq = grid[att_val_index][category_index] + 1
+          grid[att_val_index][category_index] = freq
+        end
+        return grid
+      end
+
+      private 
+      def entropy(freq_grid, total_examples)
+        #Calc entropy of each element
+        entropy = 0
+        freq_grid.each do |att_freq|
+          att_total_freq = ID3.sum(att_freq)
+          partial_entropy = 0
+          if att_total_freq != 0
+            att_freq.each do |freq|
+              prop = freq.to_f/att_total_freq
+              partial_entropy += (-1*prop*ID3.log2(prop))
+            end
+          end
+          entropy += (att_total_freq.to_f/total_examples) * partial_entropy
+        end
+        return entropy
+      end
+
+      private
+      LOG2 = Math.log(2)
+    end
+
+    class EvaluationNode #:nodoc: all
+      
+      attr_reader :index, :values, :nodes
+      
+      def initialize(data_labels, index, values, nodes)
+        @index = index
+        @values = values
+        @nodes = nodes
+        @data_labels = data_labels
+      end
+      
+      def value(data)
+        value = data[@index]
+        return rule_not_found if !@values.include?(value)
+        return nodes[@values.index(value)].value(data)
+      end
+      
+      def get_rules
+        rule_set = []
+        @nodes.each_index do |child_node_index|
+          my_rule = "#{@data_labels[@index]}=='#{@values[child_node_index]}'"
+          child_node = @nodes[child_node_index]
+          child_node_rules = child_node.get_rules
+          child_node_rules.each do |child_rule|
+            child_rule.unshift(my_rule)
+          end
+          rule_set += child_node_rules
+        end
+        return rule_set
+      end
+      
+    end
+
+    class CategoryNode #:nodoc: all
+      def initialize(label, value)
+        @label = label
+        @value = value
+      end
+      def value(data)
+        return @value
+      end
+      def get_rules
+        return [["#{@label}='#{@value}'"]]
+      end
+    end
+
+    class ErrorNode #:nodoc: all
+      def value(data)
+        raise "There was not enough information during training to do a proper induction for this data element."
+      end
+      def get_rules
+        return []
+      end
+    end
+
+  end
+end