ai4r 1.13 → 2.0

data/lib/ai4r/classifiers/simple_linear_regression.rb

@@ -1,19 +1,19 @@
+# frozen_string_literal: true
+
 # Author::    Malav Bhavsar
 # License::   MPL 1.1
 # Project::   ai4r
-# Url::       http://ai4r.org/
+# Url::       https://github.com/SergioFierens/ai4r
 #
 # You can redistribute it and/or modify it under the terms of
 # the Mozilla Public License version 1.1  as published by the
 # Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
 
-require File.dirname(__FILE__) + '/../data/data_set'
-require File.dirname(__FILE__) + '/classifier'
+require_relative '../data/data_set'
+require_relative 'classifier'
 
 module Ai4r
   module Classifiers
-
-
     # = Introduction
     #
     # This is an implementation of a Simple Linear Regression Classifier.
@@ -30,88 +30,113 @@ module Ai4r
     #     build data
     #   c.eval([1,158,105.8,192.7,71.4,55.7,2844,136,3.19,3.4,8.5,110,5500,19,25])
     #
-    
-    class SimpleLinearRegression < Classifier
 
+    # SimpleLinearRegression performs linear regression on one attribute.
+    class SimpleLinearRegression < Classifier
       attr_reader :attribute, :attribute_index, :slope, :intercept
 
+      parameters_info selected_attribute: 'Index of attribute to use for regression.'
+
+      # @return [Object]
       def initialize
+        super()
         @attribute = nil
         @attribute_index = 0
         @slope = 0
         @intercept = 0
+        @selected_attribute = nil
       end
 
       # You can evaluate new data, predicting its category.
       # e.g.
       #   c.eval([1,158,105.8,192.7,71.4,55.7,2844,136,3.19,3.4,8.5,110,5500,19,25])
       #     => 11876.96774193548
+      # @param data [Object]
+      # @return [Object]
       def eval(data)
-        @intercept + @slope * data[@attribute_index]
+        @intercept + (@slope * data[@attribute_index])
       end
 
       # Gets the best attribute and does Linear Regression using it to find out the
       # slope and intercept.
       # Parameter data has to be an instance of DataSet
+      # @param data [Object]
+      # @return [Object]
       def build(data)
-        raise "Error instance must be passed" unless data.is_a?(DataSet)
-        raise "Data should not be empty" if data.data_items.length == 0
+        validate_data(data)
+
         y_mean = data.get_mean_or_mode[data.num_attributes - 1]
+        result = if @selected_attribute
+                   evaluate_attribute(data, @selected_attribute, y_mean)
+                 else
+                   evaluate_all_attributes(data, y_mean)
+                 end
+        assign_result(data, result)
+      end
 
-        # Choose best attribute
-        min_msq = Float::MAX
-        attribute = nil
-        chosen = -1
-        chosen_slope = 0.0 / 0.0 # Float::NAN 
-        chosen_intercept = 0.0 / 0.0 # Float::NAN 
+      def validate_data(data)
+        raise 'Error instance must be passed' unless data.is_a?(Ai4r::Data::DataSet)
+        raise 'Data should not be empty' if data.data_items.empty?
+      end
 
+      def evaluate_attribute(data, attr_index, y_mean)
+        x_mean = data.get_mean_or_mode[attr_index]
+        slope, x_diff_sq, y_diff_sq = attribute_sums(data, attr_index, x_mean, y_mean)
+        if x_diff_sq.zero?
+          { chosen: attr_index, slope: 0, intercept: y_mean, msq: Float::MAX }
+        else
+          chosen_slope = slope / x_diff_sq
+          intercept = y_mean - (chosen_slope * x_mean)
+          { chosen: attr_index, slope: chosen_slope, intercept: intercept, msq: y_diff_sq - (chosen_slope * slope) }
+        end
+      end
+
+      def evaluate_all_attributes(data, y_mean)
+        result = { chosen: -1, msq: Float::MAX }
         data.data_labels.each do |attr_name|
           attr_index = data.get_index attr_name
-          if attr_index != data.num_attributes-1
-            # Compute slope and intercept
-            x_mean = data.get_mean_or_mode[attr_index]
-            sum_x_diff_squared = 0
-            sum_y_diff_squared = 0
-            slope = 0
-            data.data_items.map do |instance|
-              x_diff = instance[attr_index] - x_mean
-              y_diff = instance[attr_index] - y_mean
-              slope += x_diff * y_diff
-              sum_x_diff_squared += x_diff * x_diff
-              sum_y_diff_squared += y_diff * y_diff
-            end
-
-            if sum_x_diff_squared == 0
-              next
-            end
-
-            numerator = slope
-            slope /= sum_x_diff_squared
-            intercept = y_mean - slope * x_mean
-            msq = sum_y_diff_squared - slope * numerator
-
-            if msq < min_msq
-              min_msq = msq
-              chosen = attr_index
-              chosen_slope = slope
-              chosen_intercept = intercept
-            end
-          end
+          next if attr_index == data.num_attributes - 1
+
+          candidate = evaluate_attribute(data, attr_index, y_mean)
+          next unless candidate[:msq] < result[:msq]
+
+          result = candidate
         end
+        result
+      end
 
-        if chosen == -1
-          raise "no useful attribute found"
-          @attribute = nil
-          @attribute_index = 0
-          @slope = 0
-          @intercept = y_mean
-        else
-          @attribute = data.data_labels[chosen]
-          @attribute_index = chosen
-          @slope = chosen_slope
-          @intercept = chosen_intercept
+      def assign_result(data, result)
+        raise 'no useful attribute found' if result[:chosen] == -1
+
+        @attribute = data.data_labels[result[:chosen]]
+        @attribute_index = result[:chosen]
+        @slope = result[:slope]
+        @intercept = result[:intercept]
+        self
+      end
+
+      # Simple Linear Regression classifiers cannot generate human readable
+      # rules. This method returns a descriptive string indicating that rule
+      # extraction is not supported.
+      def get_rules
+        'SimpleLinearRegression does not support rule extraction.'
+      end
+
+      private
+
+      # Calculate regression sums for the given attribute.
+      def attribute_sums(data, attr_index, x_mean, y_mean)
+        slope = 0
+        sum_x_diff_squared = 0
+        sum_y_diff_squared = 0
+        data.data_items.each do |instance|
+          x_diff = instance[attr_index] - x_mean
+          y_diff = instance[data.num_attributes - 1] - y_mean
+          slope += x_diff * y_diff
+          sum_x_diff_squared += x_diff * x_diff
+          sum_y_diff_squared += y_diff * y_diff
         end
-        return self
+        [slope, sum_x_diff_squared, sum_y_diff_squared]
       end
     end
   end

data/lib/ai4r/classifiers/support_vector_machine.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+# Author::    OpenAI Assistant
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       https://github.com/SergioFierens/ai4r
+#
+# A minimal linear Support Vector Machine implementation using
+# stochastic gradient descent. This implementation is intentionally
+# simple and only supports binary classification with numeric
+# attributes.
+
+require_relative '../data/data_set'
+require_relative 'classifier'
+
+module Ai4r
+  module Classifiers
+    # A lightweight linear SVM classifier trained via gradient descent.
+    # Only two classes are supported. Predictions return the same class
+    # labels used in the training data.
+    class SupportVectorMachine < Classifier
+      attr_reader :weights, :bias, :classes
+
+      parameters_info learning_rate: 'Learning rate for gradient descent.',
+                      iterations: 'Training iterations.',
+                      c: 'Regularization strength.'
+
+      def initialize
+        super()
+        @learning_rate = 0.01
+        @iterations = 1000
+        @c = 1.0
+        @weights = []
+        @bias = 0.0
+        @classes = []
+      end
+
+      # Train the SVM using the provided DataSet. Only numeric attributes and
+      # exactly two classes are supported.
+      def build(data_set)
+        data_set.check_not_empty
+        @classes = data_set.build_domains.last.to_a
+        raise ArgumentError, 'SVM only supports two classes' unless @classes.size == 2
+
+        num_features = data_set.data_labels.length - 1
+        @weights = Array.new(num_features, 0.0)
+        @bias = 0.0
+
+        samples = data_set.data_items.map do |row|
+          [row[0...-1].map(&:to_f), row.last]
+        end
+
+        @iterations.times do
+          samples.each do |features, label|
+            y = label == @classes[0] ? 1.0 : -1.0
+            prediction = dot(@weights, features) + @bias
+            if y * prediction < 1
+              @weights.map!.with_index do |w, i|
+                w + (@learning_rate * ((@c * y * features[i]) - (2 * w)))
+              end
+              @bias += @learning_rate * @c * y
+            else
+              @weights.map!.with_index { |w, _i| w - (@learning_rate * 2 * w) }
+            end
+          end
+        end
+        self
+      end
+
+      # Predict the class for the given numeric feature vector.
+      def eval(data)
+        score = dot(@weights, data.map(&:to_f)) + @bias
+        score >= 0 ? @classes[0] : @classes[1]
+      end
+
+      # Support Vector Machine classifiers cannot generate human readable rules.
+      # This method returns a string indicating rule extraction is unsupported.
+      def get_rules
+        'SupportVectorMachine does not support rule extraction.'
+      end
+
+      private
+
+      def dot(a, b)
+        sum = 0.0
+        a.each_index { |i| sum += a[i] * b[i] }
+        sum
+      end
+    end
+  end
+end

data/lib/ai4r/classifiers/votes.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# Author::    Will Warner
+# License::   MPL 1.1
+# Project::   ai4r
+# Url::       https://github.com/SergioFierens/ai4r
+#
+# You can redistribute it and/or modify it under the terms of
+# the Mozilla Public License version 1.1  as published by the
+# Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
+
+module Ai4r
+  module Classifiers
+    # Simple vote counter used by ensemble methods.
+    class Votes
+      # @return [Object]
+      def initialize
+        self.tally_sheet = Hash.new(0)
+      end
+
+      # @param category [Object]
+      # @return [Object]
+      def increment_category(category)
+        tally_sheet[category] += 1
+      end
+
+      # @param category [Object]
+      # @return [Object]
+      def tally_for(category)
+        tally_sheet[category]
+      end
+
+      # @param tie_break [Object]
+      # @return [Object]
+      def get_winner(tie_break = :last, rng: Random.new)
+        n = 0 # used to create a stable sort of the tallys
+        sorted_sheet = tally_sheet.sort_by do |_, score|
+          n += 1
+          [score, n]
+        end
+        return nil if sorted_sheet.empty?
+
+        if tie_break == :random
+          max_score = sorted_sheet.last[1]
+          tied = sorted_sheet.select { |_, score| score == max_score }.map(&:first)
+          tied.sample(random: rng)
+        else
+          sorted_sheet.last.first
+        end
+      end
+
+      private
+
+      attr_accessor :tally_sheet
+    end
+  end
+end

data/lib/ai4r/classifiers/zero_r.rb

@@ -1,73 +1,114 @@
+# frozen_string_literal: true
+
 # Author::    Sergio Fierens (Implementation only)
 # License::   MPL 1.1
 # Project::   ai4r
-# Url::       http://ai4r.org/
+# Url::       https://github.com/SergioFierens/ai4r
 #
-# You can redistribute it and/or modify it under the terms of 
-# the Mozilla Public License version 1.1  as published by the 
+# You can redistribute it and/or modify it under the terms of
+# the Mozilla Public License version 1.1  as published by the
 # Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
 
-require File.dirname(__FILE__) + '/../data/data_set.rb'
-require File.dirname(__FILE__) + '/../classifiers/classifier'
+require_relative '../data/data_set'
+require_relative '../classifiers/classifier'
 
 module Ai4r
   module Classifiers
-    
     # = Introduction
-    # 
-    # The idea behind the ZeroR classifier is to identify the 
-    # the most common class value in the training set. 
-    # It always returns that value when evaluating an instance. 
-    # It is frequently used as a baseline for evaluating other machine learning 
+    #
+    # The idea behind the ZeroR classifier is to identify the
+    # the most common class value in the training set.
+    # It always returns that value when evaluating an instance.
+    # It is frequently used as a baseline for evaluating other machine learning
     # algorithms.
     class ZeroR < Classifier
-
       attr_reader :data_set, :class_value
-    
+
+      parameters_info default_class: 'Return this value when the provided ' \
+                                     'dataset is empty.',
+                      tie_break: 'Strategy used when more than one class has the ' \
+                                 'same maximal frequency. Valid values are :first (default) ' \
+                                 'and :random.',
+                      random_seed: 'Seed for tie resolution when using :random strategy.'
+
+      # @return [Object]
+      def initialize
+        super()
+        @default_class = nil
+        @tie_break = :first
+        @random_seed = nil
+        @rng = nil
+      end
+
       # Build a new ZeroR classifier. You must provide a DataSet instance
-      # as parameter. The last attribute of each item is considered as 
+      # as parameter. The last attribute of each item is considered as
       # the item class.
+      # @param data_set [Object]
+      # @return [Object]
       def build(data_set)
-        data_set.check_not_empty
         @data_set = data_set
-        frequencies = {}
+
+        if @data_set.data_items.empty?
+          @class_value = @default_class
+          return self
+        end
+
+        frequencies = Hash.new(0)
         max_freq = 0
-        @class_value = nil
+        tied_classes = []
+
         @data_set.data_items.each do |example|
           class_value = example.last
-          frequencies[class_value] = frequencies[class_value].nil? ? 1 : frequencies[class_value] + 1
+          frequencies[class_value] += 1
           class_frequency = frequencies[class_value]
-          if max_freq < class_frequency
+          if class_frequency > max_freq
             max_freq = class_frequency
-            @class_value = class_value
+            tied_classes = [class_value]
+          elsif class_frequency == max_freq && !tied_classes.include?(class_value)
+            tied_classes << class_value
           end
         end
-        return self
+
+        rng = @rng || (@random_seed.nil? ? Random.new : Random.new(@random_seed))
+
+        @class_value = if tied_classes.length == 1
+                         tied_classes.first
+                       else
+                         case @tie_break
+                         when :random
+                           tied_classes.sample(random: rng)
+                         else
+                           tied_classes.first
+                         end
+                       end
+
+        self
       end
-      
+
       # You can evaluate new data, predicting its class.
       # e.g.
       #   classifier.eval(['New York',  '<30', 'F'])  # => 'Y'
-      def eval(data)
+      # @param data [Object]
+      # @return [Object]
+      def eval(_data)
         @class_value
       end
-      
+
       # This method returns the generated rules in ruby code.
       # e.g.
-      #   
+      #
       #   classifier.get_rules
       #     # =>  marketing_target='Y'
       #
-      # It is a nice way to inspect induction results, and also to execute them:  
+      # It is a nice way to inspect induction results, and also to execute them:
       #     marketing_target = nil
-      #     eval classifier.get_rules   
+      #     eval classifier.get_rules
       #     puts marketing_target
       #       # =>  'Y'
+      # @return [Object]
       def get_rules
-        return "#{@data_set.data_labels.last} = '#{@class_value}'"
+        "#{@data_set.category_label} = '#{@class_value}'"
       end
-
     end
-    
   end
 end

data/lib/ai4r/clusterers/average_linkage.rb

@@ -1,59 +1,78 @@
+# frozen_string_literal: true
+
 # Author::    Sergio Fierens (implementation)
 # License::   MPL 1.1
 # Project::   ai4r
-# Url::       http://ai4r.org/
+# Url::       https://github.com/SergioFierens/ai4r
 #
-# You can redistribute it and/or modify it under the terms of 
-# the Mozilla Public License version 1.1  as published by the 
+# You can redistribute it and/or modify it under the terms of
+# the Mozilla Public License version 1.1  as published by the
 # Mozilla Foundation at http://www.mozilla.org/MPL/MPL-1.1.txt
 
-require File.dirname(__FILE__) + '/../data/data_set'
-require File.dirname(__FILE__) + '/../clusterers/single_linkage'
+require_relative '../data/data_set'
+require_relative '../clusterers/single_linkage'
+require_relative '../clusterers/cluster_tree'
 
 module Ai4r
   module Clusterers
-     
     # Implementation of a Hierarchical clusterer with group average
-    # linkage, AKA unweighted pair group method average or UPGMA (Everitt 
+    # linkage, AKA unweighted pair group method average or UPGMA (Everitt
     # et al., 2001 ; Jain and Dubes, 1988 ; Sokal and Michener, 1958).
-    # Hierarchical clusterer create one cluster per element, and then 
+    # Hierarchical clusterer create one cluster per element, and then
     # progressively merge clusters, until the required number of clusters
     # is reached.
-    # With average linkage, the distance between a clusters cx and 
+    # With average linkage, the distance between a clusters cx and
     # cluster (ci U cj) the the average distance between cx and ci, and
     # cx and cj.
     #
     #   D(cx, (ci U cj) = (D(cx, ci) + D(cx, cj)) / 2
     class AverageLinkage < SingleLinkage
-      
-      parameters_info :distance_function => 
-          "Custom implementation of distance function. " +
-          "It must be a closure receiving two data items and return the " +
-          "distance between them. By default, this algorithm uses " + 
-          "euclidean distance of numeric attributes to the power of 2."
-      
+      include ClusterTree
+
+      parameters_info distance_function:
+          'Custom implementation of distance function. ' \
+          'It must be a closure receiving two data items and return the ' \
+          'distance between them. By default, this algorithm uses ' \
+          'euclidean distance of numeric attributes to the power of 2.'
+
       # Build a new clusterer, using data examples found in data_set.
       # Items will be clustered in "number_of_clusters" different
       # clusters.
-      def build(data_set, number_of_clusters)
+      # @param data_set [Object]
+      # @param number_of_clusters [Object]
+      # @param *options [Object]
+      # @return [Object]
+      def build(data_set, number_of_clusters = 1, **options)
         super
       end
-      
-      # This algorithms does not allow classification of new data items 
+
+      # This algorithms does not allow classification of new data items
       # once it has been built. Rebuild the cluster including you data element.
-      def eval(data_item)
-        Raise "Eval of new data is not supported by this algorithm."
+      # @param _data_item [Object]
+      # @return [Object]
+      def eval(_data_item)
+        raise NotImplementedError, 'Eval of new data is not supported by this algorithm.'
       end
-      
+
+      # Average linkage builds a dendrogram and cannot classify new data
+      # once built.
+      # @return [Object]
+      def supports_eval?
+        false
+      end
+
       protected
-      
+
       # return distance between cluster cx and cluster (ci U cj),
       # using average linkage
-      def linkage_distance(cx, ci, cj)
-        (read_distance_matrix(cx, ci)+
-          read_distance_matrix(cx, cj))/2
+      # @param cx [Object]
+      # @param ci [Object]
+      # @param cj [Object]
+      # @return [Object]
+      def linkage_distance(cluster_x, cluster_i, cluster_j)
+        (read_distance_matrix(cluster_x, cluster_i) +
+          read_distance_matrix(cluster_x, cluster_j)) / 2
       end
-      
     end
   end
 end