hybridforest 0.9.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8d0af8ba6e658989bf55d0b8026822ab0be77020c189ffd056ffac3e7a393700
+  data.tar.gz: 968c29d52088fe4ba28d93fcd3aedf7288d323f3a35ba000efde5b1673c10e2b
+SHA512:
+  metadata.gz: 2e7076a8cdea280c900f61c98a6f53715a335d7765a6861f028a6956a8c6599cbe233b5d3e867b0d55f9a8889e41067e314ca151b00e4455f55071ac407059f5
+  data.tar.gz: dbf3dc47bfba70ff33e9452cc9232d5079651410a3eb319b29172992141c1602f8c06bba50e7ee9a56c1c8530e9099763834b1a964a86cdc4ec4c89d944237f4

data/.gitignore

@@ -0,0 +1,89 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+.DS_Store
+.idea/
+.rubocop.yml
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,13 @@
+AllCops:
+  TargetRubyVersion: 3.1
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Layout/LineLength:
+  Max: 120

data/CHANGELOG.md

@@ -0,0 +1,33 @@
+## [Unreleased]
+
+## [0.3.1] - 2021-12-27
+
+- Fix bugs related to bootstrapping and random feature selection.
+- Add more tests.
+
+## [0.4.0] - 2021-12-27
+
+- Minor refactoring
+
+## [0.5.0] - 2021-12-27
+
+- Make all utility methods module methods
+- Implement random sampling without replacement
+
+## [0.6.0] - 2021-12-28
+
+- Add nicer to_s for HybridForest::RandomForest
+
+## [0.7.0] - 2021-12-28
+
+- Add specific title to HybridForest::RandomForest#to_s
+
+## [0.8.0] - 2021-12-28
+
+- Raise custom error, not string
+- Add supported tree types list
+- Minor refactoring of internal logic
+
+## [0.9.0] - 2021-12-28
+
+- Update dependencies

data/Gemfile

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in hybridforest.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "rspec", "~> 3.0"
+
+gem "rubocop", "~> 1.7"
+
+gem "rumale"
+
+gem "activesupport", "~> 6.1"
+
+gem "rover-df", "~> 0.2.6"
+
+gem "require_all"
+
+gem "terminal-table", "~> 3.0"

data/Gemfile.lock

@@ -0,0 +1,92 @@
+PATH
+  remote: .
+  specs:
+    hybridforest (0.9.0)
+      activesupport (~> 6.1)
+      rake (~> 13.0)
+      require_all
+      rover-df (~> 0.2.6)
+      rspec (~> 3.0)
+      rubocop (~> 1.7)
+      rumale
+      terminal-table (~> 3.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    ast (2.4.2)
+    concurrent-ruby (1.1.9)
+    diff-lcs (1.4.4)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    lbfgsb (0.4.1)
+      numo-narray (>= 0.9.1)
+    minitest (5.14.4)
+    numo-narray (0.9.2.0)
+    parallel (1.21.0)
+    parser (3.0.3.1)
+      ast (~> 2.4.1)
+    rainbow (3.0.0)
+    rake (13.0.6)
+    regexp_parser (2.2.0)
+    require_all (3.0.0)
+    rexml (3.2.5)
+    rover-df (0.2.6)
+      numo-narray (>= 0.9.1.9)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.3)
+    rubocop (1.23.0)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.14.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
+    rumale (0.23.1)
+      lbfgsb (>= 0.3.0)
+      numo-narray (>= 0.9.1)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.1.0)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  activesupport (~> 6.1)
+  hybridforest!
+  rake (~> 13.0)
+  require_all
+  rover-df (~> 0.2.6)
+  rspec (~> 3.0)
+  rubocop (~> 1.7)
+  rumale
+  terminal-table (~> 3.0)
+
+BUNDLED WITH
+   2.3.3

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 hi-tech-jazz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,96 @@
+# HybridForest
+
+<code>HybridForest</code> offers the possibility to build hybrid random forests, i.e., ensembles where the base learners are built from not one but several different decision tree algorithms. As of now, two types of trees are supported:
+* `CARTTree`
+    * Performs binary  splits at each internal node.
+    * Supports categorical and continuous features.
+    * Supports binary classification problems.
+    * Uses gini impurity to find the most discriminatory feature.
+    * Considers a random subset of features in each split.
+    * Loosely based on the original CART algorithm (Breiman et al., 1984).
+* `ID3Tree`
+  * Performs multiway (>=2) splits at each internal node.
+  * Supports categorical and continuous features. 
+  * Supports binary classification problems.
+  * Uses entropy to find the most discriminatory feature.
+  * Considers every feature in max one split.
+  * Loosely based on the ID3 algorithm (Quinlan, 1986).
+
+The random forest itself is represented by the `RandomForest` class.
+A random forest classifier can be created with one of three base learner configurations.
+
+1. Hybrid mode
+```ruby
+# Equivalent, hybrid is default.
+HybridForest::RandomForest.new(number_of_trees: 100, ensemble_type: :hybrid)
+HybridForest::RandomForest.new(number_of_trees: 100) 
+```
+
+
+2. CART mode
+```ruby
+HybridForest::RandomForest.new(number_of_trees: 100, ensemble_type: :cart) 
+```
+3. ID3 mode
+
+```ruby
+HybridForest::RandomForest.new(number_of_trees: 100, ensemble_type: :id3) 
+```
+
+The implementation is quite naive and there are a bunch of features that might be nice to have but are not supported, including:
+* Pruning
+* Parallelization
+* More decision trees, e.g., C4.5 and CHAID
+* Additional hyperparameters
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'hybridforest'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install hybridforest
+
+## Usage
+
+```ruby
+require "hybridforest"
+
+# Prepare data. 
+# A dataset can be passed as a CSV file path, an array of hashes, or a hash of arrays.
+training_set, test_set, actual_test_labels = HybridForest::Utils.train_test_split("data.csv")
+
+# Create classifier.
+hybrid_forest = HybridForest::RandomForest.new(number_of_trees: 100)
+
+# Fit model.
+hybrid_forest.fit(training_set)
+
+# Predict.
+predicted_labels = hybrid_forest.predict(test_set)
+
+# Report metrics.
+puts HybridForest::Utils.prediction_report(actual_test_labels, predicted_labels)
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/hi-tech-jazz/hybridforest.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "hybridforest"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/hybridforest.gemspec

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative "lib/hybridforest/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "hybridforest"
+  spec.version = HybridForest::VERSION
+  spec.authors = ["hi-tech-jazz"]
+  spec.email = ["jazztechhi@gmail.com"]
+
+  spec.summary = "Hybrid random forest for binary classification tasks."
+  spec.description = "HybridForest provides random forests built upon combinations of different decision tree algorithms to enable diverse tree ensembles. Until version 1.0.0, please expect breaking changes."
+  spec.homepage = "https://github.com/hi-tech-jazz/hybridforest"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.0.2"
+
+  spec.metadata["homepage_uri"] = "https://github.com/hi-tech-jazz/hybridforest"
+  spec.metadata["source_code_uri"] = "https://github.com/hi-tech-jazz/hybridforest"
+  spec.metadata["changelog_uri"] = "https://github.com/hi-tech-jazz/hybridforest/blob/master/CHANGELOG.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rake", "~> 13.0"
+  spec.add_dependency "rspec", "~> 3.0"
+  spec.add_dependency "rubocop", "~> 1.7"
+  spec.add_dependency "rumale"
+  spec.add_dependency "activesupport", "~> 6.1"
+  spec.add_dependency "rover-df", "~> 0.2.6"
+  spec.add_dependency "require_all"
+  spec.add_dependency "terminal-table", "~> 3.0"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rubocop", "~> 1.7"
+  spec.add_development_dependency "rumale"
+  spec.add_development_dependency "activesupport", "~> 6.1"
+  spec.add_development_dependency "rover-df", "~> 0.2.6"
+end

data/lib/hybridforest/errors/invalid_state_error.rb

@@ -0,0 +1,9 @@
+module HybridForest
+  module Errors
+    class InvalidStateError < StandardError
+      def initialize(message = nil)
+        super
+      end
+    end
+  end
+end

data/lib/hybridforest/forests/forest_growers/cart_grower.rb

@@ -0,0 +1,19 @@
+require_relative "../../utilities/utils"
+require_relative "../../trees/cart_tree"
+
+module HybridForest
+  module Forests
+    module ForestGrowers
+      class CARTGrower
+        def grow_forest(instances, number_of_trees)
+          forest = []
+          number_of_trees.times do
+            sample, _, _ = HybridForest::Utils.train_test_bootstrap_split(instances)
+            forest << HybridForest::Trees::CARTTree.new.fit(sample)
+          end
+          forest
+        end
+      end
+    end
+  end
+end

data/lib/hybridforest/forests/forest_growers/hybrid_grower.rb

@@ -0,0 +1,46 @@
+require_relative "../../utilities/utils"
+require_relative "../../trees/id3_tree"
+require_relative "../../trees/cart_tree"
+
+module HybridForest
+  module Forests
+    module ForestGrowers
+      class HybridGrower
+        TREE_TYPES = [HybridForest::Trees::CARTTree, HybridForest::Trees::ID3Tree].freeze
+
+        def grow_forest(instances, number_of_trees)
+          forest = []
+          number_of_trees.times do
+            in_of_bag, out_of_bag, out_of_bag_labels = HybridForest::Utils.train_test_bootstrap_split(instances)
+            tree_results = grow_trees(TREE_TYPES, in_of_bag, out_of_bag, out_of_bag_labels)
+            best_tree = select_best_tree(tree_results)
+            forest << best_tree
+          end
+          forest
+        end
+
+        private
+
+        def fit_and_predict(tree_class, in_of_bag, out_of_bag, out_of_bag_labels)
+          tree = tree_class.new.fit(in_of_bag)
+          tree_predictions = tree.predict(out_of_bag)
+          tree_accuracy = HybridForest::Utils.accuracy(tree_predictions, out_of_bag_labels)
+          {tree: tree, oob_accuracy: tree_accuracy}
+        end
+
+        def select_best_tree(tree_results)
+          best_result = tree_results.max_by(1) { |result| result[:oob_accuracy] }.first
+          best_result[:tree]
+        end
+
+        def grow_trees(tree_types, in_of_bag, out_of_bag, out_of_bag_labels)
+          tree_results = []
+          tree_types.each do |tree_type|
+            tree_results << fit_and_predict(tree_type, in_of_bag, out_of_bag, out_of_bag_labels)
+          end
+          tree_results
+        end
+      end
+    end
+  end
+end

data/lib/hybridforest/forests/forest_growers/id3_grower.rb

@@ -0,0 +1,19 @@
+require_relative "../../utilities/utils"
+require_relative "../../trees/id3_tree"
+
+module HybridForest
+  module Forests
+    module ForestGrowers
+      class ID3Grower
+        def grow_forest(instances, number_of_trees)
+          forest = []
+          number_of_trees.times do
+            sample, _, _ = HybridForest::Utils.train_test_bootstrap_split(instances)
+            forest << HybridForest::Trees::ID3Tree.new.fit(sample)
+          end
+          forest
+        end
+      end
+    end
+  end
+end

data/lib/hybridforest/forests/grower_factory.rb

@@ -0,0 +1,29 @@
+require_relative "../forests/forest_growers/hybrid_grower"
+require_relative "../forests/forest_growers/cart_grower"
+require_relative "../forests/forest_growers/id3_grower"
+
+module HybridForest
+  module Forests
+    class GrowerFactory
+      TYPES = {
+        cart: HybridForest::Forests::ForestGrowers::CARTGrower,
+        id3: HybridForest::Forests::ForestGrowers::ID3Grower,
+        hybrid: HybridForest::Forests::ForestGrowers::HybridGrower
+      }.freeze
+
+      class << self
+        def for(type)
+          (TYPES[type] || default).new
+        end
+
+        def default
+          HybridForest::Forests::ForestGrowers::HybridGrower
+        end
+
+        def types
+          TYPES.keys
+        end
+      end
+    end
+  end
+end

data/lib/hybridforest/forests/random_forest.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require_relative "../utilities/utils"
+
+module HybridForest
+  class RandomForest
+    ##
+    # Creates a new random forest.
+    #
+    # +number_of_trees+ dictates the size of the tree ensemble.
+    #
+    # +ensemble_type+ dictates the composition of the tree ensemble.
+    # Valid options are +:hybrid+, +:cart+, +:id3+.
+    #
+    def initialize(number_of_trees:, ensemble_type: :hybrid)
+      raise ArgumentError, "Invalid ensemble type" unless Forests::GrowerFactory.types.include? ensemble_type
+
+      @number_of_trees = number_of_trees
+      @ensemble_type = ensemble_type
+    end
+
+    ##
+    # Fits a model to the given dataset +instances+ and returns +self+.
+    #
+    def fit(instances)
+      instances = HybridForest::Utils.to_dataframe(instances)
+      forest_grower = Forests::GrowerFactory.for(@ensemble_type)
+      @forest = forest_grower.grow_forest(instances, @number_of_trees)
+      self
+    end
+
+    ##
+    # Predicts a label for each instance in the dataset +instances+ and returns an array of labels.
+    #
+    def predict(instances)
+      raise Errors::InvalidStateError, "You must call #fit before you call #predict" if @forest.nil?
+
+      instances = HybridForest::Utils.to_dataframe(instances)
+      predictions = tree_predictions(instances)
+      predictions.collect { |votes| majority_vote(votes) }
+    end
+
+    def to_s
+      return "Empty random forest: \n#{super()}" if @forest.nil?
+
+      title = case @ensemble_type
+              when :hybrid then "Hybrid random forest"
+              else "Uniform random forest"
+      end
+
+      table = Terminal::Table.new do |t|
+        t.title = title
+        t.headings = %w[Tree Count]
+        tally_ensemble.each do |tree_type, count|
+          t << [tree_type, count]
+          t << :separator
+        end
+        t << ["Total", @number_of_trees]
+      end
+      table.to_s
+    end
+
+    private
+
+    def majority_vote(votes)
+      votes.max_by(1) { |label| votes.count(label) }.first
+    end
+
+    def tree_predictions(instances)
+      predictions = @forest.collect { |tree| tree.predict(instances) }
+      predictions.transpose
+    end
+
+    def tally_ensemble
+      tree_type_counts = Hash.new(0)
+      @forest.each do |tree|
+        key = tree.name
+        tree_type_counts[key] += 1
+      end
+      tree_type_counts.default = nil
+      tree_type_counts
+    end
+  end
+end