hybridforest 0.9.0

data/lib/hybridforest/trees/tree_growers/id3_grower.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "observer"
+require_relative "../../utilities/utils"
+require_relative "../split"
+require_relative "../feature_selectors/random_feature_subspace"
+require_relative "../feature_selectors/all_features"
+require_relative "../feature_selectors/max_one_split_per_feature"
+require_relative "../impurity_metrics/gini_impurity"
+require_relative "../impurity_metrics/entropy"
+require_relative "../nodes/binary_node"
+require_relative "../nodes/multiway_node"
+require_relative "../nodes/leaf_node"
+require_relative "../tests/less"
+require_relative "../tests/equal"
+require_relative "../tests/equal_or_greater"
+
+module HybridForest
+  module Trees
+    module TreeGrowers
+      class ID3Grower
+        include Observable
+
+        def initialize(feature_selector: MaxOneSplitPerFeature.new, impurity_metric: Entropy.new)
+          @impurity_metric = impurity_metric
+          @feature_selector = feature_selector
+          add_observer(@feature_selector)
+        end
+
+        def grow_tree(instances, parent_instances = nil)
+          features = remaining_features(instances)
+
+          if instances.count == 0
+            LeafNode.new(parent_instances)
+          elsif instances.pure? || features.count == 0
+            LeafNode.new(instances)
+          else
+            try_to_split_dataset(instances, features)
+          end
+        end
+
+        private
+
+        def try_to_split_dataset(instances, features)
+          split = find_best_split(instances, features)
+
+          if split.info_gain == 0
+            LeafNode.new(instances)
+          elsif split.binary?
+            branch_binary(instances, split.subsets, split.feature, split.value)
+          else
+            branch_multiway(instances, split.subsets, split.feature)
+          end
+        end
+
+        def find_best_split(instances, features)
+          current_impurity = @impurity_metric.compute(instances)
+          best_split = default_split(instances, features)
+
+          features.each do |feature|
+            best_split = if instances[feature].numeric?
+              best_binary_split(instances, feature, current_impurity, best_split)
+            else
+              best_multiway_split(instances, feature, current_impurity, best_split)
+            end
+          end
+
+          mark_as_used(best_split.feature)
+
+          best_split
+        end
+
+        def branch_binary(instances, subsets, feature, value)
+          true_instances, false_instances = subsets
+          true_branch = grow_tree(true_instances, instances)
+          false_branch = grow_tree(false_instances, instances)
+          test = Tests::EqualOrGreater.new(feature, value)
+          BinaryNode.new(test, true_branch, false_branch)
+        end
+
+        def branch_multiway(instances, subsets, feature)
+          branches = subsets.collect do |subset|
+            grow_tree(subset, instances)
+          end
+          paths = map_tests_to_branches(branches, subsets, feature)
+          MultiwayNode.new(paths, instances)
+        end
+
+        def map_tests_to_branches(branches, subsets_of_instances, feature)
+          paths = {}
+          branches.zip(subsets_of_instances) do |branch, subset|
+            value = subset[feature][0]
+            test = Tests::Equal.new(feature, value)
+            paths[test] = branch
+          end
+          paths
+        end
+
+        def best_binary_split(instances, feature, initial_impurity, best_split)
+          candidate_values = potential_split_values(feature, instances)
+
+          candidate_values.each do |value|
+            subsets = instances.equal_or_greater_split(feature, value)
+            info_gain = @impurity_metric.information_gain(subsets, initial_impurity)
+
+            split = Split.new(feature, info_gain: info_gain, subsets: subsets, value: value)
+            if split.better_than? best_split
+              best_split = split
+            end
+          end
+
+          best_split
+        end
+
+        def best_multiway_split(instances, feature, initial_impurity, best_split)
+          subsets = instances.multiway_equal_split(feature)
+          info_gain = @impurity_metric.information_gain(subsets, initial_impurity)
+
+          split = Split.new(feature, info_gain: info_gain, subsets: subsets)
+          if split.better_than? best_split
+            best_split = split
+          end
+
+          best_split
+        end
+
+        def potential_split_values(feature, instances)
+          label = instances.label
+          sorted = instances.sort_by { |instance| instance[feature] }
+          candidates = Set.new
+
+          sorted.each_row.each_cons(2) do |first, second|
+            next if first[label] == second[label]
+            candidates << second[feature]
+          end
+          candidates
+        end
+
+        def default_split(instances, features)
+          first_feature = features.first
+
+          if instances[first_feature].numeric?
+            value = instances[first_feature].first
+            Split.new(first_feature, value: value)
+          else
+            Split.new(first_feature)
+          end
+        end
+
+        def remaining_features(instances)
+          @feature_selector.select_features(instances.features)
+        end
+
+        def mark_as_used(feature)
+          changed
+          notify_observers(feature)
+        end
+      end
+    end
+  end
+end

data/lib/hybridforest/utilities/utils.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require "rover"
+require "rover-df"
+require "rumale"
+
+module HybridForest
+  module Utils
+    extend self
+    ##
+    # Partitions +dataset+ into training and testing datasets, and splits the testing dataset into a dataframe
+    # of independent features and an array of labels. Returns [+training_set+, +testing_set+, +testing_set_labels+]
+    #
+    def self.train_test_split(dataset, test_set_size = 0.20)
+      # TODO: Shuffle and stratify samples
+      dataset = to_dataframe(dataset)
+
+      test_set_count = (dataset.count * test_set_size).floor
+      test_set_indices = 0..test_set_count
+      test_set = dataset[test_set_indices]
+      test_set_labels = test_set.class_labels
+      test_set.except!(test_set.label)
+
+      train_set_indices = test_set_count + 1...dataset.count
+      train_set = dataset[train_set_indices]
+
+      [train_set, test_set, test_set_labels]
+    end
+
+    ##
+    # Partitions +dataset+ into training and testing datasets, drawing with replacement to the training set,
+    # and using the not drawn instances as the testing dataset. Then, splits the testing dataset into a dataframe of
+    # independent features and an array of labels.
+    # Returns [+training_set+, +testing_set+, +testing_set_labels+]
+    #
+    def self.train_test_bootstrap_split(dataset)
+      dataset = to_dataframe(dataset)
+      all_rows = (0...dataset.count).to_a
+
+      train_set = Rover::DataFrame.new
+      train_set_rows = []
+      dataset.count.times do
+        row = all_rows.sample
+        train_set_rows << row
+        train_set.concat(dataset[row])
+      end
+
+      return train_test_split(dataset) if train_set_rows.sort == all_rows
+
+      test_set_rows = all_rows - train_set_rows
+      test_set = dataset[test_set_rows]
+      test_set_labels = test_set.class_labels
+      test_set.except!(test_set.label)
+
+      [train_set, test_set, test_set_labels]
+    end
+
+    #
+    # Turns +instances+ into a dataframe prepared for fitting and applying models.
+    # Instances can be an array of hashes:
+    #
+    # [{a: 1, b: "one"},
+    # {a: 2, b: "two"},
+    # {a: 3, b: "three"}]
+    #
+    # or a hash of arrays:
+    #
+    # {a: [1, 2, 3],
+    # b: ["one", "two", "three"]}
+    #
+    # or the path to a CSV file:
+    #
+    # "dataset.csv"
+    #
+    # Accepts an optional hash for specifying feature data types:
+    # to_dataframe("dataset.csv", types: {"a" => :int, "b" => :float})
+    #
+    # Raises ArgumentError if given an invalid dataset.
+    def self.to_dataframe(instances, types: nil)
+      return instances if instances.is_a? Rover::DataFrame
+      return instances if success? { instances = Rover::DataFrame.new(instances, types: types) }
+      return instances if success? { instances = Rover.read_csv(instances, types: types) }
+      raise ArgumentError, @error
+    end
+
+    # Draws a random sample of +size+ from +data+.
+    #
+    def self.random_sample(data:, size:, with_replacement: true)
+      raise ArgumentError, "Invalid sample size" if size < 1 || size > data.count
+
+      if with_replacement
+        rows = size.times.collect { rand(0...data.count) }
+        data[rows]
+      else
+        rows = Set.new
+        until rows.size == size
+          rows << rand(0...data.count)
+        end
+        data[rows.to_a]
+      end
+    end
+
+    # Outputs a report of common prediction metrics.
+    # +actual+ and +predicted+ are expected to be equal sized arrays of class labels.
+    def self.prediction_report(actual, predicted)
+      Rumale::EvaluationMeasure.classification_report(
+        actual,
+        predicted
+      )
+    end
+
+    # Given an array of predicted labels and an array of actual labels, returns the accuracy of the predictions.
+    #
+    def self.accuracy(predicted, actual)
+      accurate = predicted.zip(actual).count { |p, a| equal_labels?(p, a) }
+      accurate.to_f / predicted.count.to_f
+    end
+
+    # Extensions to simplify common dataframe operations.
+    #
+    module DataFrameExtensions
+      class Rover::DataFrame
+        def equal_split(feature, value)
+          equal = self[self[feature] == value]
+          not_equal = self[!self[feature].in?([value])]
+          [equal, not_equal]
+        end
+
+        def equal_or_greater_split(feature, value)
+          equal_or_greater = self[self[feature] >= value]
+          less = self[self[feature] < value]
+          [equal_or_greater, less]
+        end
+
+        def multiway_equal_split(feature)
+          subsets = []
+          self[feature].uniq.each do |value|
+            subsets << self[self[feature] == value]
+          end
+          subsets
+        end
+
+        def column_by_index(index)
+          where = @vectors.keys[index]
+          @vectors[where]
+        end
+
+        def feature_count(without_label: true)
+          without_label ? names.count - 1 : names.count
+        end
+
+        def pure?
+          column_by_index(-1).uniq.size == 1
+        end
+
+        def features
+          names[0...-1]
+        end
+
+        def count_labels
+          column_by_index(-1).tally
+        end
+
+        def label
+          names[-1]
+        end
+
+        def class_labels
+          self[label].to_a
+        end
+      end
+    end
+
+    private
+
+    # Yields to the given block and rescues any errors.
+    # Returns +true+ if no exceptions were raised, +false+ otherwise.
+    def success?
+      yield
+      true
+    rescue => e
+      @error ||= e.to_s
+      false
+    end
+
+    def equal_labels?(a, b)
+      a == b || both_true?(a, b) || both_false?(a, b)
+    end
+
+    def both_true?(a, b)
+      true_label?(a) && true_label?(b)
+    end
+
+    def both_false?(a, b)
+      false_label?(a) && false_label?(b)
+    end
+
+    def true_label?(label)
+      [true, 1].include? label
+    end
+
+    def false_label?(label)
+      [false, 0].include? label
+    end
+  end
+end

data/lib/hybridforest/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module HybridForest
+  VERSION = "0.9.0"
+end

data/lib/hybridforest.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "active_support"
+require "require_all"
+require "rover"
+require "rspec"
+require "rumale"
+require "set"
+require "terminal-table"
+
+require_relative "hybridforest/version"
+require_relative "hybridforest/errors/invalid_state_error"
+require_relative "hybridforest/forests/forest_growers/hybrid_grower"
+require_relative "hybridforest/forests/forest_growers/cart_grower"
+require_relative "hybridforest/forests/forest_growers/id3_grower"
+require_relative "hybridforest/forests/grower_factory"
+require_relative "hybridforest/forests/random_forest"
+require_relative "hybridforest/trees/tree"
+require_relative "hybridforest/trees/cart_tree"
+require_relative "hybridforest/trees/id3_tree"
+require_relative "hybridforest/trees/split"
+require_relative "hybridforest/trees/feature_selectors/all_features"
+require_relative "hybridforest/trees/feature_selectors/max_one_split_per_feature"
+require_relative "hybridforest/trees/feature_selectors/random_feature_subspace"
+require_relative "hybridforest/trees/impurity_metrics/impurity"
+require_relative "hybridforest/trees/impurity_metrics/entropy"
+require_relative "hybridforest/trees/impurity_metrics/gini_impurity"
+require_relative "hybridforest/trees/nodes/binary_node"
+require_relative "hybridforest/trees/nodes/leaf_node"
+require_relative "hybridforest/trees/nodes/multiway_node"
+require_relative "hybridforest/trees/tests/test"
+require_relative "hybridforest/trees/tests/less"
+require_relative "hybridforest/trees/tests/equal"
+require_relative "hybridforest/trees/tests/equal_or_greater"
+require_relative "hybridforest/trees/tests/not_equal"
+require_relative "hybridforest/trees/tree_growers/cart_grower"
+require_relative "hybridforest/trees/tree_growers/id3_grower"
+require_relative "hybridforest/utilities/utils"
+
+module HybridForest
+  FOREST_TYPES = [
+    :hybrid,
+    :cart,
+    :id3
+  ].freeze
+end

metadata

@@ -0,0 +1,285 @@
+--- !ruby/object:Gem::Specification
+name: hybridforest
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+platform: ruby
+authors:
+- hi-tech-jazz
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2021-12-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rumale
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+- !ruby/object:Gem::Dependency
+  name: rover-df
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.6
+- !ruby/object:Gem::Dependency
+  name: require_all
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: terminal-table
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rumale
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+- !ruby/object:Gem::Dependency
+  name: rover-df
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.6
+description: HybridForest provides random forests built upon combinations of different
+  decision tree algorithms to enable diverse tree ensembles. Until version 1.0.0,
+  please expect breaking changes.
+email:
+- jazztechhi@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- hybridforest.gemspec
+- lib/hybridforest.rb
+- lib/hybridforest/errors/invalid_state_error.rb
+- lib/hybridforest/forests/forest_growers/cart_grower.rb
+- lib/hybridforest/forests/forest_growers/hybrid_grower.rb
+- lib/hybridforest/forests/forest_growers/id3_grower.rb
+- lib/hybridforest/forests/grower_factory.rb
+- lib/hybridforest/forests/random_forest.rb
+- lib/hybridforest/trees/cart_tree.rb
+- lib/hybridforest/trees/feature_selectors/all_features.rb
+- lib/hybridforest/trees/feature_selectors/max_one_split_per_feature.rb
+- lib/hybridforest/trees/feature_selectors/random_feature_subspace.rb
+- lib/hybridforest/trees/id3_tree.rb
+- lib/hybridforest/trees/impurity_metrics/entropy.rb
+- lib/hybridforest/trees/impurity_metrics/gini_impurity.rb
+- lib/hybridforest/trees/impurity_metrics/impurity.rb
+- lib/hybridforest/trees/nodes/binary_node.rb
+- lib/hybridforest/trees/nodes/leaf_node.rb
+- lib/hybridforest/trees/nodes/multiway_node.rb
+- lib/hybridforest/trees/split.rb
+- lib/hybridforest/trees/tests/equal.rb
+- lib/hybridforest/trees/tests/equal_or_greater.rb
+- lib/hybridforest/trees/tests/less.rb
+- lib/hybridforest/trees/tests/not_equal.rb
+- lib/hybridforest/trees/tests/test.rb
+- lib/hybridforest/trees/tree.rb
+- lib/hybridforest/trees/tree_growers/cart_grower.rb
+- lib/hybridforest/trees/tree_growers/id3_grower.rb
+- lib/hybridforest/utilities/utils.rb
+- lib/hybridforest/version.rb
+homepage: https://github.com/hi-tech-jazz/hybridforest
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/hi-tech-jazz/hybridforest
+  source_code_uri: https://github.com/hi-tech-jazz/hybridforest
+  changelog_uri: https://github.com/hi-tech-jazz/hybridforest/blob/master/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.3
+signing_key:
+specification_version: 4
+summary: Hybrid random forest for binary classification tasks.
+test_files: []