rumale-ensemble 0.24.0

data/lib/rumale/ensemble/stacking_regressor.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require 'rumale/validation'
+require 'rumale/base/estimator'
+require 'rumale/base/regressor'
+require 'rumale/linear_model/ridge'
+require 'rumale/model_selection/k_fold'
+
+module Rumale
+  module Ensemble
+    # StackingRegressor is a class that implements regressor with stacking method.
+    #
+    # @example
+    #   require 'rumale/ensemble/stacking_regressor'
+    #
+    #   estimators = {
+    #     las: Rumale::LinearModel::Lasso.new(reg_param: 1e-2, random_seed: 1),
+    #     mlp: Rumale::NeuralNetwork::MLPRegressor.new(hidden_units: [256], random_seed: 1),
+    #     rnd: Rumale::Ensemble::RandomForestRegressor.new(random_seed: 1)
+    #   }
+    #   meta_estimator = Rumale::LinearModel::Ridge.new(random_seed: 1)
+    #   regressor = Rumale::Ensemble::StackedRegressor.new(
+    #     estimators: estimators, meta_estimator: meta_estimator, random_seed: 1
+    #   )
+    #   regressor.fit(training_samples, training_values)
+    #   results = regressor.predict(testing_samples)
+    #
+    # *Reference*
+    # - Zhou, Z-H., "Ensemble Methods - Foundations and Algorithms," CRC Press Taylor and Francis Group, Chapman and Hall/CRC, 2012.
+    class StackingRegressor < ::Rumale::Base::Estimator
+      include ::Rumale::Base::Regressor
+
+      # Return the base regressors.
+      # @return [Hash<Symbol,Regressor>]
+      attr_reader :estimators
+
+      # Return the meta regressor.
+      # @return [Regressor]
+      attr_reader :meta_estimator
+
+      # Create a new regressor with stacking method.
+      #
+      # @param estimators [Hash<Symbol,Regressor>] The base regressors for extracting meta features.
+      # @param meta_estimator [Regressor/Nil] The meta regressor that predicts values.
+      #   If nil is given, Ridge is used.
+      # @param n_splits [Integer] The number of folds for cross validation with k-fold on meta feature extraction in training phase.
+      # @param shuffle [Boolean] The flag indicating whether to shuffle the dataset on cross validation.
+      # @param passthrough [Boolean] The flag indicating whether to concatenate the original features and meta features when training the meta regressor.
+      # @param random_seed [Integer/Nil] The seed value using to initialize the random generator on cross validation.
+      def initialize(estimators:, meta_estimator: nil, n_splits: 5, shuffle: true, passthrough: false, random_seed: nil)
+        super()
+        @estimators = estimators
+        @meta_estimator = meta_estimator || ::Rumale::LinearModel::Ridge.new
+        @params = {
+          n_splits: n_splits,
+          shuffle: shuffle,
+          passthrough: passthrough,
+          random_seed: random_seed || srand
+        }
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target variables to be used for fitting the model.
+      # @return [StackedRegressor] The learned regressor itself.
+      def fit(x, y)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+        y = ::Rumale::Validation.check_convert_target_value_array(y)
+        ::Rumale::Validation.check_sample_size(x, y)
+
+        n_samples, n_features = x.shape
+        n_outputs = y.ndim == 1 ? 1 : y.shape[1]
+
+        # training base regressors with all training data.
+        @estimators.each_key { |name| @estimators[name].fit(x, y) }
+
+        # detecting size of output for each base regressor.
+        @output_size = detect_output_size(n_features)
+
+        # extracting meta features with base regressors.
+        n_components = @output_size.values.sum
+        z = Numo::DFloat.zeros(n_samples, n_components)
+
+        kf = ::Rumale::ModelSelection::KFold.new(
+          n_splits: @params[:n_splits], shuffle: @params[:shuffle], random_seed: @params[:random_seed]
+        )
+
+        kf.split(x, y).each do |train_ids, valid_ids|
+          x_train = x[train_ids, true]
+          y_train = n_outputs == 1 ? y[train_ids] : y[train_ids, true]
+          x_valid = x[valid_ids, true]
+          f_start = 0
+          @estimators.each_key do |name|
+            est_fold = Marshal.load(Marshal.dump(@estimators[name]))
+            f_last = f_start + @output_size[name]
+            f_position = @output_size[name] == 1 ? f_start : f_start...f_last
+            z[valid_ids, f_position] = est_fold.fit(x_train, y_train).predict(x_valid)
+            f_start = f_last
+          end
+        end
+
+        # concatenating original features.
+        z = Numo::NArray.hstack([z, x]) if @params[:passthrough]
+
+        # training meta regressor.
+        @meta_estimator.fit(z, y)
+
+        self
+      end
+
+      # Predict values for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
+      # @return [Numo::DFloat] (shape: [n_samples, n_outputs]) The predicted values per sample.
+      def predict(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        z = transform(x)
+        @meta_estimator.predict(z)
+      end
+
+      # Transform the given data with the learned model.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to be transformed with the learned model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The meta features for samples.
+      def transform(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        n_samples = x.shape[0]
+        n_components = @output_size.values.sum
+        z = Numo::DFloat.zeros(n_samples, n_components)
+        f_start = 0
+        @estimators.each_key do |name|
+          f_last = f_start + @output_size[name]
+          f_position = @output_size[name] == 1 ? f_start : f_start...f_last
+          z[true, f_position] = @estimators[name].predict(x)
+          f_start = f_last
+        end
+        z = Numo::NArray.hstack([z, x]) if @params[:passthrough]
+        z
+      end
+
+      # Fit the model with training data, and then transform them with the learned model.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target variables to be used for fitting the model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The meta features for training data.
+      def fit_transform(x, y)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+        y = ::Rumale::Validation.check_convert_target_value_array(y)
+        ::Rumale::Validation.check_sample_size(x, y)
+
+        fit(x, y).transform(x)
+      end
+
+      private
+
+      def detect_output_size(n_features)
+        x_dummy = Numo::DFloat.new(2, n_features).rand
+        @estimators.each_key.with_object({}) do |name, obj|
+          output_dummy = @estimators[name].predict(x_dummy)
+          obj[name] = output_dummy.ndim == 1 ? 1 : output_dummy.shape[1]
+        end
+      end
+    end
+  end
+end

data/lib/rumale/ensemble/value.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Rumale
+  module Ensemble
+    # @!visibility private
+    module Value
+      # @!visibility private
+      N_BITS = 1.size * 8
+      # @!visibility private
+      SEED_BASE = 2**(N_BITS - 1) - 1
+    end
+  end
+end

data/lib/rumale/ensemble/version.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Rumale is a machine learning library in Ruby.
+module Rumale
+  # This module consists of the classes that implement ensemble-based methods.
+  module Ensemble
+    # @!visibility private
+    VERSION = '0.24.0'
+  end
+end

data/lib/rumale/ensemble/voting_classifier.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require 'rumale/validation'
+require 'rumale/base/estimator'
+require 'rumale/base/classifier'
+require 'rumale/preprocessing/label_encoder'
+
+module Rumale
+  module Ensemble
+    # VotingClassifier is a class that implements classifier with voting ensemble method.
+    #
+    # @example
+    #   require 'rumale/ensemble/voting_classifier'
+    #
+    #   estimators = {
+    #     lgr: Rumale::LinearModel::LogisticRegression.new(reg_param: 1e-2, random_seed: 1),
+    #     mlp: Rumale::NeuralNetwork::MLPClassifier.new(hidden_units: [256], random_seed: 1),
+    #     rnd: Rumale::Ensemble::RandomForestClassifier.new(random_seed: 1)
+    #   }
+    #   weights = { lgr: 0.2, mlp: 0.3, rnd: 0.5 }
+    #
+    #   classifier = Rumale::Ensemble::VotingClassifier.new(estimators: estimators, weights: weights, voting: 'soft')
+    #   classifier.fit(x_train, y_train)
+    #   results = classifier.predict(x_test)
+    #
+    # *Reference*
+    # - Zhou, Z-H., "Ensemble Methods - Foundations and Algorithms," CRC Press Taylor and Francis Group, Chapman and Hall/CRC, 2012.
+    class VotingClassifier < ::Rumale::Base::Estimator
+      include ::Rumale::Base::Classifier
+
+      # Return the sub-classifiers that voted.
+      # @return [Hash<Symbol,Classifier>]
+      attr_reader :estimators
+
+      # Return the class labels.
+      # @return [Numo::Int32] (size: n_classes)
+      attr_reader :classes
+
+      # Create a new ensembled classifier with voting rule.
+      #
+      # @param estimators [Hash<Symbol,Classifier>] The sub-classifiers to vote.
+      # @param weights [Hash<Symbol,Float>] The weight value for each classifier.
+      # @param voting [String] The voting rule for the predicted results of each classifier.
+      #   If 'hard' is given, the ensembled classifier predicts the class label by majority vote.
+      #   If 'soft' is given, the ensembled classifier uses the weighted average of predicted probabilities for the prediction.
+      def initialize(estimators:, weights: nil, voting: 'hard')
+        super()
+        @estimators = estimators
+        @params = {
+          weights: weights || estimators.each_key.with_object({}) { |name, w| w[name] = 1.0 },
+          voting: voting
+        }
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [VotingClassifier] The learned classifier itself.
+      def fit(x, y)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+        y = ::Rumale::Validation.check_convert_label_array(y)
+        ::Rumale::Validation.check_sample_size(x, y)
+
+        @encoder = ::Rumale::Preprocessing::LabelEncoder.new
+        y_encoded = @encoder.fit_transform(y)
+        @classes = Numo::NArray[*@encoder.classes]
+        @estimators.each_key { |name| @estimators[name].fit(x, y_encoded) }
+
+        self
+      end
+
+      # Calculate confidence scores for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) The confidence score per sample.
+      def decision_function(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        return predict_proba(x) if soft_voting?
+
+        n_samples = x.shape[0]
+        n_classes = @classes.size
+        z = Numo::DFloat.zeros(n_samples, n_classes)
+        @estimators.each do |name, estimator|
+          estimator.predict(x).to_a.each_with_index { |c, i| z[i, c] += @params[:weights][name] }
+        end
+        z
+      end
+
+      # Predict class labels for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) The predicted class label per sample.
+      def predict(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        n_samples = x.shape[0]
+        n_classes = @classes.size
+        z = decision_function(x)
+        predicted = z.max_index(axis: 1) - Numo::Int32.new(n_samples).seq * n_classes
+        Numo::Int32.cast(@encoder.inverse_transform(predicted))
+      end
+
+      # Predict probability for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probabilities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
+      def predict_proba(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        n_samples = x.shape[0]
+        n_classes = @classes.size
+        z = Numo::DFloat.zeros(n_samples, n_classes)
+        sum_weight = @params[:weights].each_value.sum
+        @estimators.each do |name, estimator|
+          z += @params[:weights][name] * estimator.predict_proba(x)
+        end
+        z /= sum_weight
+      end
+
+      private
+
+      def soft_voting?
+        @params[:voting] == 'soft'
+      end
+    end
+  end
+end

data/lib/rumale/ensemble/voting_regressor.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require 'rumale/validation'
+require 'rumale/base/estimator'
+require 'rumale/base/regressor'
+
+module Rumale
+  module Ensemble
+    # VotingRegressor is a class that implements regressor with voting ensemble method.
+    #
+    # @example
+    #   require 'rumale/ensemble/voting_regressor'
+    #
+    #   estimators = {
+    #     rdg: Rumale::LinearModel::Ridge.new(reg_param: 1e-2, random_seed: 1),
+    #     mlp: Rumale::NeuralNetwork::MLPRegressor.new(hidden_units: [256], random_seed: 1),
+    #     rnd: Rumale::Ensemble::RandomForestRegressor.new(random_seed: 1)
+    #   }
+    #   weights = { rdg: 0.2, mlp: 0.3, rnd: 0.5 }
+    #
+    #   regressor = Rumale::Ensemble::VotingRegressor.new(estimators: estimators, weights: weights, voting: 'soft')
+    #   regressor.fit(x_train, y_train)
+    #   results = regressor.predict(x_test)
+    #
+    # *Reference*
+    # - Zhou, Z-H., "Ensemble Methods - Foundations and Algorithms," CRC Press Taylor and Francis Group, Chapman and Hall/CRC, 2012.
+    class VotingRegressor < ::Rumale::Base::Estimator
+      include ::Rumale::Base::Regressor
+
+      # Return the sub-regressors that voted.
+      # @return [Hash<Symbol,Regressor>]
+      attr_reader :estimators
+
+      # Create a new ensembled regressor with voting rule.
+      #
+      # @param estimators [Hash<Symbol,Regressor>] The sub-regressors to vote.
+      # @param weights [Hash<Symbol,Float>] The weight value for each regressor.
+      def initialize(estimators:, weights: nil)
+        super()
+        @estimators = estimators
+        @params = {
+          weights: weights || estimators.each_key.with_object({}) { |name, w| w[name] = 1.0 }
+        }
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [VotingRegressor] The learned regressor itself.
+      def fit(x, y)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+        y = ::Rumale::Validation.check_convert_target_value_array(y)
+        ::Rumale::Validation.check_sample_size(x, y)
+
+        @n_outputs = y.ndim > 1 ? y.shape[1] : 1
+        @estimators.each_key { |name| @estimators[name].fit(x, y) }
+
+        self
+      end
+
+      # Predict values for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
+      # @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted value per sample.
+      def predict(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        z = single_target? ? Numo::DFloat.zeros(x.shape[0]) : Numo::DFloat.zeros(x.shape[0], @n_outputs)
+        sum_weight = @params[:weights].each_value.sum
+        @estimators.each do |name, estimator|
+          z += @params[:weights][name] * estimator.predict(x)
+        end
+        z / sum_weight
+      end
+
+      private
+
+      def single_target?
+        @n_outputs == 1
+      end
+    end
+  end
+end

data/lib/rumale/ensemble.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'numo/narray'
+
+require_relative 'ensemble/version'
+
+require_relative 'ensemble/value'
+
+require_relative 'ensemble/ada_boost_classifier'
+require_relative 'ensemble/ada_boost_regressor'
+require_relative 'ensemble/extra_trees_classifier'
+require_relative 'ensemble/extra_trees_regressor'
+require_relative 'ensemble/gradient_boosting_classifier'
+require_relative 'ensemble/gradient_boosting_regressor'
+require_relative 'ensemble/random_forest_classifier'
+require_relative 'ensemble/random_forest_regressor'
+require_relative 'ensemble/stacking_classifier'
+require_relative 'ensemble/stacking_regressor'
+require_relative 'ensemble/voting_classifier'
+require_relative 'ensemble/voting_regressor'

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: rumale-ensemble
+version: !ruby/object:Gem::Version
+  version: 0.24.0
+platform: ruby
+authors:
+- yoshoku
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2022-12-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: numo-narray
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+- !ruby/object:Gem::Dependency
+  name: rumale-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+- !ruby/object:Gem::Dependency
+  name: rumale-linear_model
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+- !ruby/object:Gem::Dependency
+  name: rumale-model_selection
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+- !ruby/object:Gem::Dependency
+  name: rumale-preprocessing
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+- !ruby/object:Gem::Dependency
+  name: rumale-tree
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+description: |
+  Rumale::Ensemble provides ensemble learning algorithms,
+  such as AdaBoost, Gradient Tree Boosting, and Random Forest,
+  with Rumale interface.
+email:
+- yoshoku@outlook.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- lib/rumale/ensemble.rb
+- lib/rumale/ensemble/ada_boost_classifier.rb
+- lib/rumale/ensemble/ada_boost_regressor.rb
+- lib/rumale/ensemble/extra_trees_classifier.rb
+- lib/rumale/ensemble/extra_trees_regressor.rb
+- lib/rumale/ensemble/gradient_boosting_classifier.rb
+- lib/rumale/ensemble/gradient_boosting_regressor.rb
+- lib/rumale/ensemble/random_forest_classifier.rb
+- lib/rumale/ensemble/random_forest_regressor.rb
+- lib/rumale/ensemble/stacking_classifier.rb
+- lib/rumale/ensemble/stacking_regressor.rb
+- lib/rumale/ensemble/value.rb
+- lib/rumale/ensemble/version.rb
+- lib/rumale/ensemble/voting_classifier.rb
+- lib/rumale/ensemble/voting_regressor.rb
+homepage: https://github.com/yoshoku/rumale
+licenses:
+- BSD-3-Clause
+metadata:
+  homepage_uri: https://github.com/yoshoku/rumale
+  source_code_uri: https://github.com/yoshoku/rumale/tree/main/rumale-ensemble
+  changelog_uri: https://github.com/yoshoku/rumale/blob/main/CHANGELOG.md
+  documentation_uri: https://yoshoku.github.io/rumale/doc/
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.26
+signing_key:
+specification_version: 4
+summary: Rumale::Ensemble provides ensemble learning algorithms with Rumale interface.
+test_files: []