hybridforest 0.9.0

data/lib/hybridforest/trees/cart_tree.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "tree"
+require_relative "tree_growers/cart_grower"
+
+module HybridForest
+  module Trees
+    class CARTTree < Tree
+      def initialize(tree_grower: HybridForest::Trees::TreeGrowers::CARTGrower.new)
+        super(tree_grower: tree_grower)
+      end
+
+      def name
+        "CART Tree"
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/feature_selectors/all_features.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module HybridForest
+  module Trees
+    class AllFeatures
+      def select_features(all_features)
+        all_features
+      end
+
+      def update(_feature)
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/feature_selectors/max_one_split_per_feature.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module HybridForest
+  module Trees
+    class MaxOneSplitPerFeature
+      attr_reader :exhausted_features
+
+      def initialize
+        @exhausted_features = []
+      end
+
+      def select_features(all_features)
+        all_features - @exhausted_features
+      end
+
+      def update(feature)
+        @exhausted_features << feature
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/feature_selectors/random_feature_subspace.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "set"
+
+module HybridForest
+  module Trees
+    class RandomFeatureSubspace
+      def select_features(all_features)
+        n = default_subspace_size(all_features.count)
+        indices = Set.new
+        until indices.size == n
+          indices << rand(0...all_features.count)
+        end
+        all_features.values_at(*indices)
+      end
+
+      def update(_feature)
+      end
+
+      private
+
+      def default_subspace_size(num_of_features)
+        (num_of_features.to_f / 2).round
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/id3_tree.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "tree"
+require_relative "tree_growers/id3_grower"
+
+module HybridForest
+  module Trees
+    class ID3Tree < Tree
+      def initialize(tree_grower: HybridForest::Trees::TreeGrowers::ID3Grower.new)
+        super(tree_grower: tree_grower)
+      end
+
+      def name
+        "ID3 Tree"
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/impurity_metrics/entropy.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative "impurity"
+
+module HybridForest
+  module Trees
+    class Entropy
+      include HybridForest::Trees::Impurity
+
+      def compute(instances)
+        instance_count = instances.count
+        label_counts = instances.count_labels
+        label_counts.values.sum do |label_count|
+          label_probability = label_count.to_f / instance_count
+          label_surprise = Math.log2(1 / label_probability)
+          label_probability * label_surprise
+        end
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/impurity_metrics/gini_impurity.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "impurity"
+
+module HybridForest
+  module Trees
+    class GiniImpurity
+      include HybridForest::Trees::Impurity
+
+      def compute(instances)
+        total_count = instances.count.to_f
+        label_counts = instances.count_labels.each_value
+        1 - label_counts.sum { |count| (count / total_count)**2 }
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/impurity_metrics/impurity.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "active_support"
+
+module HybridForest
+  module Trees
+    module Impurity
+      def information_gain(children, parent_impurity)
+        return 0.0 if children.blank?
+
+        parent_count = children.sum(&:count)
+        children_impurity = children.sum do |child|
+          weighted_impurity(child, parent_count)
+        end
+        parent_impurity - children_impurity
+      end
+
+      def weighted_impurity(instances_in_child, parent_count)
+        weight = instances_in_child.count.to_f / parent_count
+        compute(instances_in_child) * weight
+      end
+
+      def compute(instances)
+        raise NotImplementedError, "Must be implemented by including classes"
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/nodes/binary_node.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module HybridForest
+  module Trees
+    class BinaryNode
+      attr_reader :test, :true_branch, :false_branch
+
+      def initialize(test, true_branch, false_branch)
+        @test = test
+        @true_branch = true_branch
+        @false_branch = false_branch
+      end
+
+      def branch_for(instance)
+        if @test.passed_by? instance
+          @true_branch
+        else
+          @false_branch
+        end
+      end
+
+      def classify(instance)
+        branch_for(instance).classify(instance)
+      end
+
+      def print_string(spacing = "")
+        print spacing
+        puts "#{@test} True"
+        @true_branch.print_string(spacing + "  ")
+
+        print spacing
+        puts "#{@test} False"
+        @false_branch.print_string(spacing + "  ")
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/nodes/leaf_node.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module HybridForest
+  module Trees
+    class LeafNode
+      def initialize(instances)
+        @prediction = majority_vote(instances)
+      end
+
+      def classify(_instance)
+        @prediction
+      end
+
+      def print_string(spacing = "")
+        print spacing
+        puts to_s
+      end
+
+      def to_s
+        "Predict #{@prediction}"
+      end
+
+      private
+
+      def majority_vote(instances)
+        labels = instances[instances.label].to_enum
+        labels.max_by(1) { |label| labels.count(label) }.first
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/nodes/multiway_node.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module HybridForest
+  module Trees
+    class MultiwayNode
+      attr_reader :paths, :majority_class
+
+      def initialize(paths, instances)
+        @paths = paths
+        @majority_class = majority_vote(instances)
+      end
+
+      def tests
+        @paths.keys
+      end
+
+      def branches
+        @paths.values
+      end
+
+      def branch_for(instance)
+        match = tests.find { |test| test.passed_by? instance }
+        paths[match]
+      end
+
+      def classify(instance)
+        branch = branch_for instance
+        branch&.classify(instance) || @majority_class
+      end
+
+      def print_string(spacing = "")
+        paths.each do |test, branch|
+          print spacing
+          puts "#{test} True"
+          branch.print_string(spacing + "  ")
+        end
+      end
+
+      def majority_vote(instances)
+        labels = instances[instances.label].to_enum
+        labels.max_by(1) { |label| labels.count(label) }.first
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/split.rb

@@ -0,0 +1,30 @@
+module HybridForest
+  module Trees
+    class Split
+      include Comparable
+
+      attr_reader :feature, :value, :subsets, :info_gain
+      alias_method :better_than?, :>
+      alias_method :worse_than?, :<
+
+      def initialize(feature, info_gain: 0, subsets: [Rover::DataFrame.new, Rover::DataFrame.new], value: nil)
+        @feature = feature
+        @value = value
+        @subsets = subsets
+        @info_gain = info_gain
+      end
+
+      def <=>(other)
+        info_gain <=> other.info_gain
+      end
+
+      def binary?
+        value ? true : false
+      end
+
+      def multiway?
+        !binary?
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/tests/equal.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "test"
+
+module HybridForest
+  module Trees
+    module Tests
+      class Equal < Test
+        def initialize(feature, value)
+          super(feature, value)
+        end
+
+        def passed_by?(instance)
+          instance[feature] == value
+        end
+
+        def ==(other)
+          return false unless other.is_a? Equal
+          feature == other.feature && value == other.value
+        end
+
+        def eql?(other)
+          return false unless other.is_a? Equal
+          feature.eql?(other.feature) && value.eql?(other.value)
+        end
+
+        def to_s
+          "#{feature} == #{value}?"
+        end
+
+        def description
+          "#{feature} equal to #{value}?"
+        end
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/tests/equal_or_greater.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "test"
+
+module HybridForest
+  module Trees
+    module Tests
+      class EqualOrGreater < Test
+        def initialize(feature, value)
+          super(feature, value)
+        end
+
+        def passed_by?(instance)
+          instance[feature] >= value
+        end
+
+        def ==(other)
+          return false unless other.is_a? EqualOrGreater
+          feature == other.feature && value == other.value
+        end
+
+        def eql?(other)
+          return false unless other.is_a? EqualOrGreater
+          feature.eql?(other.feature) && value.eql?(other.value)
+        end
+
+        def to_s
+          "#{feature} >= #{value}?"
+        end
+
+        def description
+          "#{feature} equal to or greater than #{value}?"
+        end
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/tests/less.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "test"
+
+module HybridForest
+  module Trees
+    module Tests
+      class Less < Test
+        def initialize(feature, value)
+          super(feature, value)
+        end
+
+        def passed_by?(instance)
+          instance[feature] < value
+        end
+
+        def ==(other)
+          return false unless other.is_a? Less
+          feature == other.feature && value == other.value
+        end
+
+        def eql?(other)
+          return false unless other.is_a? Less
+          feature.eql?(other.feature) && value.eql?(other.value)
+        end
+
+        def to_s
+          "#{feature} < #{value}?"
+        end
+
+        def description
+          "#{feature} less than #{value}?"
+        end
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/tests/not_equal.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "test"
+
+module HybridForest
+  module Trees
+    module Tests
+      class NotEqual < Test
+        def initialize(feature, value)
+          super(feature, value)
+        end
+
+        def passed_by?(instance)
+          instance[feature] != value
+        end
+
+        def ==(other)
+          return false unless other.is_a? NotEqual
+          feature == other.feature && value == other.value
+        end
+
+        def eql?(other)
+          return false unless other.is_a? NotEqual
+          feature.eql?(other.feature) && value.eql?(other.value)
+        end
+
+        def to_s
+          "#{feature} != #{value}?"
+        end
+
+        def description
+          "#{feature} not equal to #{value}?"
+        end
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/tests/test.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module HybridForest
+  module Trees
+    module Tests
+      class Test
+        attr_reader :feature, :value
+
+        def initialize(feature, value)
+          @feature = feature
+          @value = value
+        end
+
+        def passed_by?(_instance)
+          raise NotImplementedError
+        end
+
+        def description
+          raise NotImplementedError
+        end
+
+        def hash
+          feature.hash ^ value.hash
+        end
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/tree.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "../utilities/utils"
+
+module HybridForest
+  module Trees
+    class Tree
+      # Creates a new Tree using the specified tree growing algorithm.
+      def initialize(tree_grower:)
+        @tree_grower = tree_grower
+      end
+
+      ##
+      # Fits a model to the given dataset +instances+ and returns +self+.
+      #
+      def fit(instances)
+        instances = HybridForest::Utils.to_dataframe(instances)
+        @root = @tree_grower.grow_tree(instances)
+        self
+      end
+
+      ##
+      # Predicts a label for each instance in the dataset +instances+ and returns an array of labels.
+      #
+      def predict(instances)
+        if @root.nil?
+          raise Errors::InvalidStateError,
+            "You must call #fit before you call #predict"
+        end
+
+        HybridForest::Utils.to_dataframe(instances).each_row.reduce([]) do |predictions, instance|
+          predictions << @root.classify(instance)
+        end
+      end
+
+      # Prints a string representation of this Tree.
+      def inspect
+        if @root.nil?
+          "Empty tree: #{super}"
+        else
+          @root.print_string
+        end
+      end
+    end
+  end
+end

data/lib/hybridforest/trees/tree_growers/cart_grower.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative "../../utilities/utils"
+require_relative "../split"
+require_relative "../feature_selectors/random_feature_subspace"
+require_relative "../feature_selectors/all_features"
+require_relative "../feature_selectors/max_one_split_per_feature"
+require_relative "../impurity_metrics/gini_impurity"
+require_relative "../impurity_metrics/entropy"
+require_relative "../nodes/binary_node"
+require_relative "../nodes/leaf_node"
+require_relative "../tests/equal_or_greater"
+
+module HybridForest
+  module Trees
+    module TreeGrowers
+      class CARTGrower
+        def initialize(feature_selector: RandomFeatureSubspace.new, impurity_metric: GiniImpurity.new)
+          @impurity_metric = impurity_metric
+          @feature_selector = feature_selector
+        end
+
+        def grow_tree(instances)
+          split = find_best_split(instances)
+          if split.info_gain == 0
+            LeafNode.new(instances)
+          else
+            branch(split.subsets, split.feature, split.value)
+          end
+        end
+
+        private
+
+        def branch(subsets, feature, value)
+          true_instances, false_instances = subsets
+          true_branch = grow_tree(true_instances)
+          false_branch = grow_tree(false_instances)
+          test = Tests::EqualOrGreater.new(feature, value)
+          BinaryNode.new(test, true_branch, false_branch)
+        end
+
+        def find_best_split(instances)
+          considered_features = @feature_selector.select_features(instances.features)
+          current_impurity = @impurity_metric.compute(instances)
+          best_split = default_split(instances, considered_features)
+
+          considered_features.each do |feature|
+            instances[feature].uniq.each do |value|
+              subsets = if instances[feature].numeric?
+                instances.equal_or_greater_split(feature, value)
+              else
+                instances.equal_split(feature, value)
+              end
+
+              next if subsets.any? { |set| set.count == 0 }
+
+              info_gain = @impurity_metric.information_gain(subsets, current_impurity)
+              split = Split.new(feature, info_gain: info_gain, subsets: subsets, value: value)
+              if split.better_than? best_split
+                best_split = split
+              end
+            end
+          end
+
+          best_split
+        end
+
+        def default_split(instances, features)
+          first_feature = features.first
+          value = instances[first_feature].first
+          Split.new(first_feature, value: value)
+        end
+      end
+    end
+  end
+end