rumale 0.8.0

data/lib/rumale/model_selection/k_fold.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'rumale/base/splitter'
+
+module Rumale
+  # This module consists of the classes for model validation techniques.
+  module ModelSelection
+    # KFold is a class that generates the set of data indices for K-fold cross-validation.
+    #
+    # @example
+    #   kf = Rumale::ModelSelection::KFold.new(n_splits: 3, shuffle: true, random_seed: 1)
+    #   kf.split(samples, labels).each do |train_ids, test_ids|
+    #     train_samples = samples[train_ids, true]
+    #     test_samples = samples[test_ids, true]
+    #     ...
+    #   end
+    #
+    class KFold
+      include Base::Splitter
+
+      # Return the flag indicating whether to shuffle the dataset.
+      # @return [Boolean]
+      attr_reader :shuffle
+
+      # Return the random generator for shuffling the dataset.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new data splitter for K-fold cross validation.
+      #
+      # @param n_splits [Integer] The number of folds.
+      # @param shuffle [Boolean] The flag indicating whether to shuffle the dataset.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(n_splits: 3, shuffle: false, random_seed: nil)
+        check_params_integer(n_splits: n_splits)
+        check_params_boolean(shuffle: shuffle)
+        check_params_type_or_nil(Integer, random_seed: random_seed)
+        check_params_positive(n_splits: n_splits)
+        @n_splits = n_splits
+        @shuffle = shuffle
+        @random_seed = random_seed
+        @random_seed ||= srand
+        @rng = Random.new(@random_seed)
+      end
+
+      # Generate data indices for K-fold cross validation.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features])
+      #   The dataset to be used to generate data indices for K-fold cross validation.
+      # @return [Array] The set of data indices for constructing the training and testing dataset in each fold.
+      def split(x, _y = nil)
+        check_sample_array(x)
+        # Initialize and check some variables.
+        n_samples, = x.shape
+        unless @n_splits.between?(2, n_samples)
+          raise ArgumentError,
+                'The value of n_splits must be not less than 2 and not more than the number of samples.'
+        end
+        # Splits dataset ids to each fold.
+        dataset_ids = [*0...n_samples]
+        dataset_ids.shuffle!(random: @rng) if @shuffle
+        fold_sets = Array.new(@n_splits) do |n|
+          n_fold_samples = n_samples / @n_splits
+          n_fold_samples += 1 if n < n_samples % @n_splits
+          dataset_ids.shift(n_fold_samples)
+        end
+        # Returns array consisting of the training and testing ids for each fold.
+        Array.new(@n_splits) do |n|
+          train_ids = fold_sets.select.with_index { |_, id| id != n }.flatten
+          test_ids = fold_sets[n]
+          [train_ids, test_ids]
+        end
+      end
+    end
+  end
+end

data/lib/rumale/model_selection/stratified_k_fold.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'rumale/base/splitter'
+
+module Rumale
+  module ModelSelection
+    # StratifiedKFold is a class that generates the set of data indices for K-fold cross-validation.
+    # The proportion of the number of samples in each class will be almost equal for each fold.
+    #
+    # @example
+    #   kf = Rumale::ModelSelection::StratifiedKFold.new(n_splits: 3, shuffle: true, random_seed: 1)
+    #   kf.split(samples, labels).each do |train_ids, test_ids|
+    #     train_samples = samples[train_ids, true]
+    #     test_samples = samples[test_ids, true]
+    #     ...
+    #   end
+    #
+    class StratifiedKFold
+      include Base::Splitter
+
+      # Return the flag indicating whether to shuffle the dataset.
+      # @return [Boolean]
+      attr_reader :shuffle
+
+      # Return the random generator for shuffling the dataset.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new data splitter for K-fold cross validation.
+      #
+      # @param n_splits [Integer] The number of folds.
+      # @param shuffle [Boolean] The flag indicating whether to shuffle the dataset.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(n_splits: 3, shuffle: false, random_seed: nil)
+        check_params_integer(n_splits: n_splits)
+        check_params_boolean(shuffle: shuffle)
+        check_params_type_or_nil(Integer, random_seed: random_seed)
+        check_params_positive(n_splits: n_splits)
+        @n_splits = n_splits
+        @shuffle = shuffle
+        @random_seed = random_seed
+        @random_seed ||= srand
+        @rng = Random.new(@random_seed)
+      end
+
+      # Generate data indices for stratified K-fold cross validation.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features])
+      #   The dataset to be used to generate data indices for stratified K-fold cross validation.
+      #   This argument exists to unify the interface between the K-fold methods, it is not used in the method.
+      # @param y [Numo::Int32] (shape: [n_samples])
+      #   The labels to be used to generate data indices for stratified K-fold cross validation.
+      # @return [Array] The set of data indices for constructing the training and testing dataset in each fold.
+      def split(x, y)
+        check_sample_array(x)
+        check_label_array(y)
+        check_sample_label_size(x, y)
+        # Check the number of samples in each class.
+        unless valid_n_splits?(y)
+          raise ArgumentError,
+                'The value of n_splits must be not less than 2 and not more than the number of samples in each class.'
+        end
+        # Splits dataset ids of each class to each fold.
+        fold_sets_each_class = y.to_a.uniq.map { |label| fold_sets(y, label) }
+        # Returns array consisting of the training and testing ids for each fold.
+        Array.new(@n_splits) { |fold_id| train_test_sets(fold_sets_each_class, fold_id) }
+      end
+
+      private
+
+      def valid_n_splits?(y)
+        y.to_a.uniq.map { |label| y.eq(label).where.size }.all? { |n_samples| @n_splits.between?(2, n_samples) }
+      end
+
+      def fold_sets(y, label)
+        sample_ids = y.eq(label).where.to_a
+        sample_ids.shuffle!(random: @rng) if @shuffle
+        n_samples = sample_ids.size
+        Array.new(@n_splits) do |n|
+          n_fold_samples = n_samples / @n_splits
+          n_fold_samples += 1 if n < n_samples % @n_splits
+          sample_ids.shift(n_fold_samples)
+        end
+      end
+
+      def train_test_sets(fold_sets_each_class, fold_id)
+        train_test_sets_each_class = fold_sets_each_class.map do |folds|
+          folds.partition.with_index { |_, id| id != fold_id }.map(&:flatten)
+        end
+        train_test_sets_each_class.transpose.map(&:flatten)
+      end
+    end
+  end
+end

data/lib/rumale/multiclass/one_vs_rest_classifier.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator.rb'
+require 'rumale/base/classifier.rb'
+
+module Rumale
+  # This module consists of the classes that implement multi-class classification strategy.
+  module Multiclass
+    # @note
+    #   All classifier in Rumale support multi-class classifiction since version 0.2.7.
+    #   There is no need to explicitly use this class for multiclass classifiction.
+    #
+    # OneVsRestClassifier is a class that implements One-vs-Rest (OvR) strategy for multi-class classification.
+    #
+    # @example
+    #   base_estimator = Rumale::LinearModel::LogisticRegression.new
+    #   estimator = Rumale::Multiclass::OneVsRestClassifier.new(estimator: base_estimator)
+    #   estimator.fit(training_samples, training_labels)
+    #   results = estimator.predict(testing_samples)
+    class OneVsRestClassifier
+      include Base::BaseEstimator
+      include Base::Classifier
+
+      # Return the set of estimators.
+      # @return [Array<Classifier>]
+      attr_reader :estimators
+
+      # Return the class labels.
+      # @return [Numo::Int32] (shape: [n_classes])
+      attr_reader :classes
+
+      # Create a new multi-class classifier with the one-vs-rest startegy.
+      #
+      # @param estimator [Classifier] The (binary) classifier for construction a multi-class classifier.
+      def initialize(estimator: nil)
+        check_params_type(Rumale::Base::BaseEstimator, estimator: estimator)
+        @params = {}
+        @params[:estimator] = estimator
+        @estimators = nil
+        @classes = nil
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [OneVsRestClassifier] The learned classifier itself.
+      def fit(x, y)
+        check_sample_array(x)
+        check_label_array(y)
+        check_sample_label_size(x, y)
+        y_arr = y.to_a
+        @classes = Numo::Int32.asarray(y_arr.uniq.sort)
+        @estimators = @classes.to_a.map do |label|
+          bin_y = Numo::Int32.asarray(y_arr.map { |l| l == label ? 1 : -1 })
+          @params[:estimator].dup.fit(x, bin_y)
+        end
+        self
+      end
+
+      # Calculate confidence scores for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence scores per sample for each class.
+      def decision_function(x)
+        check_sample_array(x)
+        n_classes = @classes.size
+        Numo::DFloat.asarray(Array.new(n_classes) { |m| @estimators[m].decision_function(x).to_a }).transpose
+      end
+
+      # Predict class labels for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
+      def predict(x)
+        check_sample_array(x)
+        n_samples, = x.shape
+        decision_values = decision_function(x)
+        Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data about OneVsRestClassifier.
+      def marshal_dump
+        { params: @params,
+          classes: @classes,
+          estimators: @estimators.map { |e| Marshal.dump(e) } }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @classes = obj[:classes]
+        @estimators = obj[:estimators].map { |e| Marshal.load(e) }
+        nil
+      end
+    end
+  end
+end

data/lib/rumale/naive_bayes/naive_bayes.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator'
+require 'rumale/base/classifier'
+
+module Rumale
+  # This module consists of the classes that implement naive bayes models.
+  module NaiveBayes
+    # BaseNaiveBayes is a class that has methods for common processes of naive bayes classifier.
+    class BaseNaiveBayes
+      include Base::BaseEstimator
+      include Base::Classifier
+
+      # Predict class labels for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
+      def predict(x)
+        check_sample_array(x)
+        n_samples = x.shape.first
+        decision_values = decision_function(x)
+        Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })
+      end
+
+      # Predict log-probability for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the log-probailities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted log-probability of each class per sample.
+      def predict_log_proba(x)
+        check_sample_array(x)
+        n_samples, = x.shape
+        log_likelihoods = decision_function(x)
+        log_likelihoods - Numo::NMath.log(Numo::NMath.exp(log_likelihoods).sum(1)).reshape(n_samples, 1)
+      end
+
+      # Predict probability for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
+      def predict_proba(x)
+        check_sample_array(x)
+        Numo::NMath.exp(predict_log_proba(x)).abs
+      end
+    end
+
+    # GaussianNB is a class that implements Gaussian Naive Bayes classifier.
+    #
+    # @example
+    #   estimator = Rumale::NaiveBayes::GaussianNB.new
+    #   estimator.fit(training_samples, training_labels)
+    #   results = estimator.predict(testing_samples)
+    class GaussianNB < BaseNaiveBayes
+      # Return the class labels.
+      # @return [Numo::Int32] (size: n_classes)
+      attr_reader :classes
+
+      # Return the prior probabilities of the classes.
+      # @return [Numo::DFloat] (shape: [n_classes])
+      attr_reader :class_priors
+
+      # Return the mean vectors of the classes.
+      # @return [Numo::DFloat] (shape: [n_classes, n_features])
+      attr_reader :means
+
+      # Return the variance vectors of the classes.
+      # @return [Numo::DFloat] (shape: [n_classes, n_features])
+      attr_reader :variances
+
+      # Create a new classifier with Gaussian Naive Bayes.
+      def initialize
+        @params = {}
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The categorical variables (e.g. labels)
+      #   to be used for fitting the model.
+      # @return [GaussianNB] The learned classifier itself.
+      def fit(x, y)
+        check_sample_array(x)
+        check_label_array(y)
+        check_sample_label_size(x, y)
+        n_samples, = x.shape
+        @classes = Numo::Int32[*y.to_a.uniq.sort]
+        @class_priors = Numo::DFloat[*@classes.to_a.map { |l| y.eq(l).count / n_samples.to_f }]
+        @means = Numo::DFloat[*@classes.to_a.map { |l| x[y.eq(l).where, true].mean(0) }]
+        @variances = Numo::DFloat[*@classes.to_a.map { |l| x[y.eq(l).where, true].var(0) }]
+        self
+      end
+
+      # Calculate confidence scores for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence scores per sample for each class.
+      def decision_function(x)
+        check_sample_array(x)
+        n_classes = @classes.size
+        log_likelihoods = Array.new(n_classes) do |l|
+          Math.log(@class_priors[l]) - 0.5 * (
+            Numo::NMath.log(2.0 * Math::PI * @variances[l, true]) +
+            ((x - @means[l, true])**2 / @variances[l, true])).sum(1)
+        end
+        Numo::DFloat[*log_likelihoods].transpose
+      end
+
+      # Dump marshal data.
+      #
+      # @return [Hash] The marshal data about GaussianNB.
+      def marshal_dump
+        { params: @params,
+          classes: @classes,
+          class_priors: @class_priors,
+          means: @means,
+          variances: @variances }
+      end
+
+      # Load marshal data.
+      #
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @classes = obj[:classes]
+        @class_priors = obj[:class_priors]
+        @means = obj[:means]
+        @variances = obj[:variances]
+        nil
+      end
+    end
+
+    # MultinomialNB is a class that implements Multinomial Naive Bayes classifier.
+    #
+    # @example
+    #   estimator = Rumale::NaiveBayes::MultinomialNB.new(smoothing_param: 1.0)
+    #   estimator.fit(training_samples, training_labels)
+    #   results = estimator.predict(testing_samples)
+    #
+    # *Reference*
+    # - C D. Manning, P. Raghavan, and H. Schutze, "Introduction to Information Retrieval," Cambridge University Press., 2008.
+    class MultinomialNB < BaseNaiveBayes
+      # Return the class labels.
+      # @return [Numo::Int32] (size: n_classes)
+      attr_reader :classes
+
+      # Return the prior probabilities of the classes.
+      # @return [Numo::DFloat] (shape: [n_classes])
+      attr_reader :class_priors
+
+      # Return the conditional probabilities for features of each class.
+      # @return [Numo::DFloat] (shape: [n_classes, n_features])
+      attr_reader :feature_probs
+
+      # Create a new classifier with Multinomial Naive Bayes.
+      #
+      # @param smoothing_param [Float] The Laplace smoothing parameter.
+      def initialize(smoothing_param: 1.0)
+        check_params_float(smoothing_param: smoothing_param)
+        check_params_positive(smoothing_param: smoothing_param)
+        @params = {}
+        @params[:smoothing_param] = smoothing_param
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The categorical variables (e.g. labels)
+      #   to be used for fitting the model.
+      # @return [MultinomialNB] The learned classifier itself.
+      def fit(x, y)
+        check_sample_array(x)
+        check_label_array(y)
+        check_sample_label_size(x, y)
+        n_samples, = x.shape
+        @classes = Numo::Int32[*y.to_a.uniq.sort]
+        @class_priors = Numo::DFloat[*@classes.to_a.map { |l| y.eq(l).count / n_samples.to_f }]
+        count_features = Numo::DFloat[*@classes.to_a.map { |l| x[y.eq(l).where, true].sum(0) }]
+        count_features += @params[:smoothing_param]
+        n_classes = @classes.size
+        @feature_probs = count_features / count_features.sum(1).reshape(n_classes, 1)
+        self
+      end
+
+      # Calculate confidence scores for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence scores per sample for each class.
+      def decision_function(x)
+        check_sample_array(x)
+        n_classes = @classes.size
+        bin_x = x.gt(0)
+        log_likelihoods = Array.new(n_classes) do |l|
+          Math.log(@class_priors[l]) + (Numo::DFloat[*bin_x] * Numo::NMath.log(@feature_probs[l, true])).sum(1)
+        end
+        Numo::DFloat[*log_likelihoods].transpose
+      end
+
+      # Dump marshal data.
+      #
+      # @return [Hash] The marshal data about MultinomialNB.
+      def marshal_dump
+        { params: @params,
+          classes: @classes,
+          class_priors: @class_priors,
+          feature_probs: @feature_probs }
+      end
+
+      # Load marshal data.
+      #
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @classes = obj[:classes]
+        @class_priors = obj[:class_priors]
+        @feature_probs = obj[:feature_probs]
+        nil
+      end
+    end
+
+    # BernoulliNB is a class that implements Bernoulli Naive Bayes classifier.
+    #
+    # @example
+    #   estimator = Rumale::NaiveBayes::BernoulliNB.new(smoothing_param: 1.0, bin_threshold: 0.0)
+    #   estimator.fit(training_samples, training_labels)
+    #   results = estimator.predict(testing_samples)
+    #
+    # *Reference*
+    # - C D. Manning, P. Raghavan, and H. Schutze, "Introduction to Information Retrieval," Cambridge University Press., 2008.
+    class BernoulliNB < BaseNaiveBayes
+      # Return the class labels.
+      # @return [Numo::Int32] (size: n_classes)
+      attr_reader :classes
+
+      # Return the prior probabilities of the classes.
+      # @return [Numo::DFloat] (shape: [n_classes])
+      attr_reader :class_priors
+
+      # Return the conditional probabilities for features of each class.
+      # @return [Numo::DFloat] (shape: [n_classes, n_features])
+      attr_reader :feature_probs
+
+      # Create a new classifier with Bernoulli Naive Bayes.
+      #
+      # @param smoothing_param [Float] The Laplace smoothing parameter.
+      # @param bin_threshold [Float] The threshold for binarizing of features.
+      def initialize(smoothing_param: 1.0, bin_threshold: 0.0)
+        check_params_float(smoothing_param: smoothing_param, bin_threshold: bin_threshold)
+        check_params_positive(smoothing_param: smoothing_param)
+        @params = {}
+        @params[:smoothing_param] = smoothing_param
+        @params[:bin_threshold] = bin_threshold
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The categorical variables (e.g. labels)
+      #   to be used for fitting the model.
+      # @return [BernoulliNB] The learned classifier itself.
+      def fit(x, y)
+        check_sample_array(x)
+        check_label_array(y)
+        check_sample_label_size(x, y)
+        n_samples, = x.shape
+        bin_x = Numo::DFloat[*x.gt(@params[:bin_threshold])]
+        @classes = Numo::Int32[*y.to_a.uniq.sort]
+        n_samples_each_class = Numo::DFloat[*@classes.to_a.map { |l| y.eq(l).count.to_f }]
+        @class_priors = n_samples_each_class / n_samples
+        count_features = Numo::DFloat[*@classes.to_a.map { |l| bin_x[y.eq(l).where, true].sum(0) }]
+        count_features += @params[:smoothing_param]
+        n_samples_each_class += 2.0 * @params[:smoothing_param]
+        n_classes = @classes.size
+        @feature_probs = count_features / n_samples_each_class.reshape(n_classes, 1)
+        self
+      end
+
+      # Calculate confidence scores for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence scores per sample for each class.
+      def decision_function(x)
+        check_sample_array(x)
+        n_classes = @classes.size
+        bin_x = Numo::DFloat[*x.gt(@params[:bin_threshold])]
+        not_bin_x = Numo::DFloat[*x.le(@params[:bin_threshold])]
+        log_likelihoods = Array.new(n_classes) do |l|
+          Math.log(@class_priors[l]) + (
+            (Numo::DFloat[*bin_x] * Numo::NMath.log(@feature_probs[l, true])).sum(1)
+            (Numo::DFloat[*not_bin_x] * Numo::NMath.log(1.0 - @feature_probs[l, true])).sum(1))
+        end
+        Numo::DFloat[*log_likelihoods].transpose
+      end
+
+      # Dump marshal data.
+      #
+      # @return [Hash] The marshal data about BernoulliNB.
+      def marshal_dump
+        { params: @params,
+          classes: @classes,
+          class_priors: @class_priors,
+          feature_probs: @feature_probs }
+      end
+
+      # Load marshal data.
+      #
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @classes = obj[:classes]
+        @class_priors = obj[:class_priors]
+        @feature_probs = obj[:feature_probs]
+        nil
+      end
+    end
+  end
+end