rumale-ensemble 0.24.0

data/lib/rumale/ensemble/random_forest_classifier.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+require 'rumale/validation'
+require 'rumale/base/estimator'
+require 'rumale/base/classifier'
+require 'rumale/tree/decision_tree_classifier'
+require 'rumale/ensemble/value'
+
+module Rumale
+  # This module consists of the classes that implement ensemble-based methods.
+  module Ensemble
+    # RandomForestClassifier is a class that implements random forest for classification.
+    #
+    # @example
+    #   require 'rumale/ensemble/random_forest_classifier'
+    #
+    #   estimator =
+    #     Rumale::Ensemble::RandomForestClassifier.new(
+    #       n_estimators: 10, criterion: 'gini', max_depth: 3, max_leaf_nodes: 10, min_samples_leaf: 5, random_seed: 1)
+    #   estimator.fit(training_samples, traininig_labels)
+    #   results = estimator.predict(testing_samples)
+    #
+    class RandomForestClassifier < ::Rumale::Base::Estimator
+      include ::Rumale::Base::Classifier
+
+      # Return the set of estimators.
+      # @return [Array<DecisionTreeClassifier>]
+      attr_reader :estimators
+
+      # Return the class labels.
+      # @return [Numo::Int32] (size: n_classes)
+      attr_reader :classes
+
+      # Return the importance for each feature.
+      # @return [Numo::DFloat] (size: n_features)
+      attr_reader :feature_importances
+
+      # Return the random generator for random selection of feature index.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new classifier with random forest.
+      #
+      # @param n_estimators [Integer] The numeber of decision trees for contructing random forest.
+      # @param criterion [String] The function to evalue spliting point. Supported criteria are 'gini' and 'entropy'.
+      # @param max_depth [Integer] The maximum depth of the tree.
+      #   If nil is given, decision tree grows without concern for depth.
+      # @param max_leaf_nodes [Integer] The maximum number of leaves on decision tree.
+      #   If nil is given, number of leaves is not limited.
+      # @param min_samples_leaf [Integer] The minimum number of samples at a leaf node.
+      # @param max_features [Integer] The number of features to consider when searching optimal split point.
+      #   If nil is given, split process considers 'Math.sqrt(n_features)' features.
+      # @param n_jobs [Integer] The number of jobs for running the fit method in parallel.
+      #   If nil is given, the method does not execute in parallel.
+      #   If zero or less is given, it becomes equal to the number of processors.
+      #   This parameter is ignored if the Parallel gem is not loaded.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      #   It is used to randomly determine the order of features when deciding spliting point.
+      def initialize(n_estimators: 10,
+                     criterion: 'gini', max_depth: nil, max_leaf_nodes: nil, min_samples_leaf: 1,
+                     max_features: nil, n_jobs: nil, random_seed: nil)
+        super()
+        @params = {
+          n_estimators: n_estimators,
+          criterion: criterion,
+          max_depth: max_depth,
+          max_leaf_nodes: max_leaf_nodes,
+          min_samples_leaf: min_samples_leaf,
+          max_features: max_features,
+          n_jobs: n_jobs,
+          random_seed: random_seed || srand
+        }
+        @rng = Random.new(@params[:random_seed])
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [RandomForestClassifier] The learned classifier itself.
+      def fit(x, y)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+        y = ::Rumale::Validation.check_convert_label_array(y)
+        ::Rumale::Validation.check_sample_size(x, y)
+
+        # Initialize some variables.
+        n_samples, n_features = x.shape
+        @params[:max_features] = Math.sqrt(n_features).to_i if @params[:max_features].nil?
+        @params[:max_features] = [[1, @params[:max_features]].max, n_features].min
+        @classes = Numo::Int32.asarray(y.to_a.uniq.sort)
+        sub_rng = @rng.dup
+        rngs = Array.new(@params[:n_estimators]) { Random.new(sub_rng.rand(::Rumale::Ensemble::Value::SEED_BASE)) }
+        # Construct forest.
+        @estimators =
+          if enable_parallel?
+            parallel_map(@params[:n_estimators]) do |n|
+              bootstrap_ids = Array.new(n_samples) { rngs[n].rand(0...n_samples) }
+              plant_tree(rngs[n].seed).fit(x[bootstrap_ids, true], y[bootstrap_ids])
+            end
+          else
+            Array.new(@params[:n_estimators]) do |n|
+              bootstrap_ids = Array.new(n_samples) { rngs[n].rand(0...n_samples) }
+              plant_tree(rngs[n].seed).fit(x[bootstrap_ids, true], y[bootstrap_ids])
+            end
+          end
+        @feature_importances =
+          if enable_parallel?
+            parallel_map(@params[:n_estimators]) { |n| @estimators[n].feature_importances }.sum
+          else
+            @estimators.sum(&:feature_importances)
+          end
+        @feature_importances /= @feature_importances.sum
+        self
+      end
+
+      # Predict class labels for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
+      def predict(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        n_samples = x.shape[0]
+        n_estimators = @estimators.size
+        predicted = if enable_parallel?
+                      predict_set = parallel_map(n_estimators) { |n| @estimators[n].predict(x).to_a }.transpose
+                      parallel_map(n_samples) { |n| predict_set[n].group_by { |v| v }.max_by { |_k, v| v.size }.first }
+                    else
+                      predict_set = @estimators.map { |tree| tree.predict(x).to_a }.transpose
+                      Array.new(n_samples) { |n| predict_set[n].group_by { |v| v }.max_by { |_k, v| v.size }.first }
+                    end
+        Numo::Int32.asarray(predicted)
+      end
+
+      # Predict probability for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
+      def predict_proba(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        n_estimators = @estimators.size
+        if enable_parallel?
+          parallel_map(n_estimators) { |n| predict_proba_tree(@estimators[n], x) }.sum / n_estimators
+        else
+          @estimators.sum { |tree| predict_proba_tree(tree, x) } / n_estimators
+        end
+      end
+
+      # Return the index of the leaf that each sample reached.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples, n_estimators]) Leaf index for sample.
+      def apply(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        Numo::Int32[*Array.new(@params[:n_estimators]) { |n| @estimators[n].apply(x) }].transpose.dup
+      end
+
+      private
+
+      def plant_tree(rnd_seed)
+        ::Rumale::Tree::DecisionTreeClassifier.new(
+          criterion: @params[:criterion], max_depth: @params[:max_depth],
+          max_leaf_nodes: @params[:max_leaf_nodes], min_samples_leaf: @params[:min_samples_leaf],
+          max_features: @params[:max_features], random_seed: rnd_seed
+        )
+      end
+
+      def predict_proba_tree(tree, x)
+        # initialize some variables.
+        n_samples = x.shape[0]
+        base_classes = @classes.to_a
+        n_classes = base_classes.size
+        class_ids = tree.classes.map { |c| base_classes.index(c) }
+        # predict probabilities.
+        probs = Numo::DFloat.zeros(n_samples, n_classes)
+        tree_probs = tree.predict_proba(x)
+        class_ids.each_with_index { |i, j| probs[true, i] = tree_probs[true, j] }
+        probs
+      end
+    end
+  end
+end

data/lib/rumale/ensemble/random_forest_regressor.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require 'rumale/validation'
+require 'rumale/base/estimator'
+require 'rumale/base/regressor'
+require 'rumale/tree/decision_tree_regressor'
+require 'rumale/ensemble/value'
+
+module Rumale
+  module Ensemble
+    # RandomForestRegressor is a class that implements random forest for regression
+    #
+    # @example
+    #   require 'rumale/ensemble/random_forest_regressor'
+    #
+    #   estimator =
+    #     Rumale::Ensemble::RandomForestRegressor.new(
+    #       n_estimators: 10, criterion: 'mse', max_depth: 3, max_leaf_nodes: 10, min_samples_leaf: 5, random_seed: 1)
+    #   estimator.fit(training_samples, traininig_values)
+    #   results = estimator.predict(testing_samples)
+    #
+    class RandomForestRegressor < ::Rumale::Base::Estimator
+      include ::Rumale::Base::Regressor
+
+      # Return the set of estimators.
+      # @return [Array<DecisionTreeRegressor>]
+      attr_reader :estimators
+
+      # Return the importance for each feature.
+      # @return [Numo::DFloat] (size: n_features)
+      attr_reader :feature_importances
+
+      # Return the random generator for random selection of feature index.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new regressor with random forest.
+      #
+      # @param n_estimators [Integer] The numeber of decision trees for contructing random forest.
+      # @param criterion [String] The function to evalue spliting point. Supported criteria are 'gini' and 'entropy'.
+      # @param max_depth [Integer] The maximum depth of the tree.
+      #   If nil is given, decision tree grows without concern for depth.
+      # @param max_leaf_nodes [Integer] The maximum number of leaves on decision tree.
+      #   If nil is given, number of leaves is not limited.
+      # @param min_samples_leaf [Integer] The minimum number of samples at a leaf node.
+      # @param max_features [Integer] The number of features to consider when searching optimal split point.
+      #   If nil is given, split process considers 'Math.sqrt(n_features)' features.
+      # @param n_jobs [Integer] The number of jobs for running the fit and predict methods in parallel.
+      #   If nil is given, the methods do not execute in parallel.
+      #   If zero or less is given, it becomes equal to the number of processors.
+      #   This parameter is ignored if the Parallel gem is not loaded.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      #   It is used to randomly determine the order of features when deciding spliting point.
+      def initialize(n_estimators: 10,
+                     criterion: 'mse', max_depth: nil, max_leaf_nodes: nil, min_samples_leaf: 1,
+                     max_features: nil, n_jobs: nil, random_seed: nil)
+        super()
+        @params = {
+          n_estimators: n_estimators,
+          criterion: criterion,
+          max_depth: max_depth,
+          max_leaf_nodes: max_leaf_nodes,
+          min_samples_leaf: min_samples_leaf,
+          max_features: max_features,
+          n_jobs: n_jobs,
+          random_seed: random_seed || srand
+        }
+        @rng = Random.new(@params[:random_seed])
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values to be used for fitting the model.
+      # @return [RandomForestRegressor] The learned regressor itself.
+      def fit(x, y)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+        y = ::Rumale::Validation.check_convert_target_value_array(y)
+        ::Rumale::Validation.check_sample_size(x, y)
+
+        # Initialize some variables.
+        n_samples, n_features = x.shape
+        @params[:max_features] = Math.sqrt(n_features).to_i if @params[:max_features].nil?
+        @params[:max_features] = [[1, @params[:max_features]].max, n_features].min
+        single_target = y.shape[1].nil?
+        sub_rng = @rng.dup
+        rngs = Array.new(@params[:n_estimators]) { Random.new(sub_rng.rand(::Rumale::Ensemble::Value::SEED_BASE)) }
+        # Construct forest.
+        @estimators =
+          if enable_parallel?
+            parallel_map(@params[:n_estimators]) do |n|
+              bootstrap_ids = Array.new(n_samples) { rngs[n].rand(0...n_samples) }
+              plant_tree(rngs[n].seed).fit(x[bootstrap_ids, true], single_target ? y[bootstrap_ids] : y[bootstrap_ids, true])
+            end
+          else
+            Array.new(@params[:n_estimators]) do |n|
+              bootstrap_ids = Array.new(n_samples) { rngs[n].rand(0...n_samples) }
+              plant_tree(rngs[n].seed).fit(x[bootstrap_ids, true], single_target ? y[bootstrap_ids] : y[bootstrap_ids, true])
+            end
+          end
+        @feature_importances =
+          if enable_parallel?
+            parallel_map(@params[:n_estimators]) { |n| @estimators[n].feature_importances }.sum
+          else
+            @estimators.sum(&:feature_importances)
+          end
+        @feature_importances /= @feature_importances.sum
+        self
+      end
+
+      # Predict values for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
+      # @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted value per sample.
+      def predict(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        if enable_parallel?
+          parallel_map(@params[:n_estimators]) { |n| @estimators[n].predict(x) }.sum / @params[:n_estimators]
+        else
+          @estimators.sum { |tree| tree.predict(x) } / @params[:n_estimators]
+        end
+      end
+
+      # Return the index of the leaf that each sample reached.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to assign each leaf.
+      # @return [Numo::Int32] (shape: [n_samples, n_estimators]) Leaf index for sample.
+      def apply(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        Numo::Int32[*Array.new(@params[:n_estimators]) { |n| @estimators[n].apply(x) }].transpose.dup
+      end
+
+      private
+
+      def plant_tree(rnd_seed)
+        ::Rumale::Tree::DecisionTreeRegressor.new(
+          criterion: @params[:criterion], max_depth: @params[:max_depth],
+          max_leaf_nodes: @params[:max_leaf_nodes], min_samples_leaf: @params[:min_samples_leaf],
+          max_features: @params[:max_features], random_seed: rnd_seed
+        )
+      end
+    end
+  end
+end

data/lib/rumale/ensemble/stacking_classifier.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require 'rumale/validation'
+require 'rumale/base/estimator'
+require 'rumale/base/classifier'
+require 'rumale/linear_model/logistic_regression'
+require 'rumale/model_selection/stratified_k_fold'
+require 'rumale/preprocessing/label_encoder'
+
+module Rumale
+  module Ensemble
+    # StackingClassifier is a class that implements classifier with stacking method.
+    #
+    # @example
+    #   require 'rumale/ensemble/stacking_classifier'
+    #
+    #   estimators = {
+    #     lgr: Rumale::LinearModel::LogisticRegression.new(reg_param: 1e-2, random_seed: 1),
+    #     mlp: Rumale::NeuralNetwork::MLPClassifier.new(hidden_units: [256], random_seed: 1),
+    #     rnd: Rumale::Ensemble::RandomForestClassifier.new(random_seed: 1)
+    #   }
+    #   meta_estimator = Rumale::LinearModel::LogisticRegression.new(random_seed: 1)
+    #   classifier = Rumale::Ensemble::StackedClassifier.new(
+    #     estimators: estimators, meta_estimator: meta_estimator, random_seed: 1
+    #   )
+    #   classifier.fit(training_samples, training_labels)
+    #   results = classifier.predict(testing_samples)
+    #
+    # *Reference*
+    # - Zhou, Z-H., "Ensemble Methods - Foundations and Algorithms," CRC Press Taylor and Francis Group, Chapman and Hall/CRC, 2012.
+    class StackingClassifier < ::Rumale::Base::Estimator
+      include ::Rumale::Base::Classifier
+
+      # Return the base classifiers.
+      # @return [Hash<Symbol,Classifier>]
+      attr_reader :estimators
+
+      # Return the meta classifier.
+      # @return [Classifier]
+      attr_reader :meta_estimator
+
+      # Return the class labels.
+      # @return [Numo::Int32] (size: n_classes)
+      attr_reader :classes
+
+      # Return the method used by each base classifier.
+      # @return [Hash<Symbol,Symbol>]
+      attr_reader :stack_method
+
+      # Create a new classifier with stacking method.
+      #
+      # @param estimators [Hash<Symbol,Classifier>] The base classifiers for extracting meta features.
+      # @param meta_estimator [Classifier/Nil] The meta classifier that predicts class label.
+      #   If nil is given, LogisticRegression is used.
+      # @param n_splits [Integer] The number of folds for cross validation with stratified k-fold on meta feature extraction in training phase.
+      # @param shuffle [Boolean] The flag indicating whether to shuffle the dataset on cross validation.
+      # @param stack_method [String] The method name of base classifier for using meta feature extraction.
+      #   If 'auto' is given, it searches the callable method in the order 'predict_proba', 'decision_function', and 'predict'
+      #   on each classifier.
+      # @param passthrough [Boolean] The flag indicating whether to concatenate the original features and meta features when training the meta classifier.
+      # @param random_seed [Integer/Nil] The seed value using to initialize the random generator on cross validation.
+      def initialize(estimators:, meta_estimator: nil, n_splits: 5, shuffle: true, stack_method: 'auto', passthrough: false,
+                     random_seed: nil)
+        super()
+        @estimators = estimators
+        @meta_estimator = meta_estimator || ::Rumale::LinearModel::LogisticRegression.new
+        @params = {
+          n_splits: n_splits,
+          shuffle: shuffle,
+          stack_method: stack_method,
+          passthrough: passthrough,
+          random_seed: random_seed || srand
+        }
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [StackedClassifier] The learned classifier itself.
+      def fit(x, y)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+        y = ::Rumale::Validation.check_convert_label_array(y)
+        ::Rumale::Validation.check_sample_size(x, y)
+
+        n_samples, n_features = x.shape
+
+        @encoder = ::Rumale::Preprocessing::LabelEncoder.new
+        y_encoded = @encoder.fit_transform(y)
+        @classes = Numo::NArray[*@encoder.classes]
+
+        # training base classifiers with all training data.
+        @estimators.each_key { |name| @estimators[name].fit(x, y_encoded) }
+
+        # detecting feature extraction method and its size of output for each base classifier.
+        @stack_method = detect_stack_method
+        @output_size = detect_output_size(n_features)
+
+        # extracting meta features with base classifiers.
+        n_components = @output_size.values.sum
+        z = Numo::DFloat.zeros(n_samples, n_components)
+
+        kf = ::Rumale::ModelSelection::StratifiedKFold.new(
+          n_splits: @params[:n_splits], shuffle: @params[:shuffle], random_seed: @params[:random_seed]
+        )
+
+        kf.split(x, y_encoded).each do |train_ids, valid_ids|
+          x_train = x[train_ids, true]
+          y_train = y_encoded[train_ids]
+          x_valid = x[valid_ids, true]
+          f_start = 0
+          @estimators.each_key do |name|
+            est_fold = Marshal.load(Marshal.dump(@estimators[name]))
+            f_last = f_start + @output_size[name]
+            f_position = @output_size[name] == 1 ? f_start : f_start...f_last
+            z[valid_ids, f_position] = est_fold.fit(x_train, y_train).public_send(@stack_method[name], x_valid)
+            f_start = f_last
+          end
+        end
+
+        # concatenating original features.
+        z = Numo::NArray.hstack([z, x]) if @params[:passthrough]
+
+        # training meta classifier.
+        @meta_estimator.fit(z, y_encoded)
+
+        self
+      end
+
+      # Calculate confidence scores for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) The confidence score per sample.
+      def decision_function(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        z = transform(x)
+        @meta_estimator.decision_function(z)
+      end
+
+      # Predict class labels for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) The predicted class label per sample.
+      def predict(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        z = transform(x)
+        Numo::Int32.cast(@encoder.inverse_transform(@meta_estimator.predict(z)))
+      end
+
+      # Predict probability for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probabilities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) The predicted probability of each class per sample.
+      def predict_proba(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        z = transform(x)
+        @meta_estimator.predict_proba(z)
+      end
+
+      # Transform the given data with the learned model.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to be transformed with the learned model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The meta features for samples.
+      def transform(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        n_samples = x.shape[0]
+        n_components = @output_size.values.sum
+        z = Numo::DFloat.zeros(n_samples, n_components)
+        f_start = 0
+        @estimators.each_key do |name|
+          f_last = f_start + @output_size[name]
+          f_position = @output_size[name] == 1 ? f_start : f_start...f_last
+          z[true, f_position] = @estimators[name].public_send(@stack_method[name], x)
+          f_start = f_last
+        end
+        z = Numo::NArray.hstack([z, x]) if @params[:passthrough]
+        z
+      end
+
+      # Fit the model with training data, and then transform them with the learned model.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The meta features for training data.
+      def fit_transform(x, y)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+        y = ::Rumale::Validation.check_convert_label_array(y)
+        ::Rumale::Validation.check_sample_size(x, y)
+
+        fit(x, y).transform(x)
+      end
+
+      private
+
+      STACK_METHODS = %i[predict_proba decision_function predict].freeze
+
+      private_constant :STACK_METHODS
+
+      def detect_stack_method
+        if @params[:stack_method] == 'auto'
+          @estimators.each_key.with_object({}) do |name, obj|
+            obj[name] = STACK_METHODS.detect do |m|
+              @estimators[name].respond_to?(m)
+            end
+          end
+        else
+          @estimators.each_key.with_object({}) { |name, obj| obj[name] = @params[:stack_method].to_sym }
+        end
+      end
+
+      def detect_output_size(n_features)
+        x_dummy = Numo::DFloat.new(2, n_features).rand
+        @estimators.each_key.with_object({}) do |name, obj|
+          output_dummy = @estimators[name].public_send(@stack_method[name], x_dummy)
+          obj[name] = output_dummy.ndim == 1 ? 1 : output_dummy.shape[1]
+        end
+      end
+    end
+  end
+end