rumale 0.13.8 → 0.14.0

data/lib/rumale/model_selection/stratified_k_fold.rb

@@ -36,9 +36,9 @@ module Rumale
       # @param shuffle [Boolean] The flag indicating whether to shuffle the dataset.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(n_splits: 3, shuffle: false, random_seed: nil)
-        check_params_integer(n_splits: n_splits)
+        check_params_numeric(n_splits: n_splits)
         check_params_boolean(shuffle: shuffle)
-        check_params_type_or_nil(Integer, random_seed: random_seed)
+        check_params_numeric_or_nil(random_seed: random_seed)
         check_params_positive(n_splits: n_splits)
         @n_splits = n_splits
         @shuffle = shuffle
@@ -56,8 +56,8 @@ module Rumale
       #   The labels to be used to generate data indices for stratified K-fold cross validation.
       # @return [Array] The set of data indices for constructing the training and testing dataset in each fold.
       def split(x, y)
-        check_sample_array(x)
-        check_label_array(y)
+        x = check_convert_sample_array(x)
+        y = check_convert_label_array(y)
         check_sample_label_size(x, y)
         # Check the number of samples in each class.
         unless valid_n_splits?(y)

data/lib/rumale/model_selection/stratified_shuffle_split.rb

@@ -33,10 +33,8 @@ module Rumale
       # @param train_size [Float] The ratio of number of samples for train data.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(n_splits: 3, test_size: 0.1, train_size: nil, random_seed: nil)
-        check_params_integer(n_splits: n_splits)
-        check_params_float(test_size: test_size)
-        check_params_type_or_nil(Float, train_size: train_size)
-        check_params_type_or_nil(Integer, random_seed: random_seed)
+        check_params_numeric(n_splits: n_splits, test_size: test_size)
+        check_params_numeric_or_nil(train_size: train_size, random_seed: random_seed)
         check_params_positive(n_splits: n_splits)
         check_params_positive(test_size: test_size)
         check_params_positive(train_size: train_size) unless train_size.nil?
@@ -57,8 +55,8 @@ module Rumale
       #   The labels to be used to generate data indices for stratified random permutation cross validation.
       # @return [Array] The set of data indices for constructing the training and testing dataset in each fold.
       def split(x, y)
-        check_sample_array(x)
-        check_label_array(y)
+        x = check_convert_sample_array(x)
+        y = check_convert_label_array(y)
         check_sample_label_size(x, y)
         # Initialize and check some variables.
         train_sz = @train_size.nil? ? 1.0 - @test_size : @train_size

data/lib/rumale/multiclass/one_vs_rest_classifier.rb

@@ -46,8 +46,8 @@ module Rumale
       # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
       # @return [OneVsRestClassifier] The learned classifier itself.
       def fit(x, y)
-        check_sample_array(x)
-        check_label_array(y)
+        x = check_convert_sample_array(x)
+        y = check_convert_label_array(y)
         check_sample_label_size(x, y)
         y_arr = y.to_a
         @classes = Numo::Int32.asarray(y_arr.uniq.sort)
@@ -63,7 +63,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
       # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence scores per sample for each class.
       def decision_function(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         n_classes = @classes.size
         Numo::DFloat.asarray(Array.new(n_classes) { |m| @estimators[m].decision_function(x).to_a }).transpose
       end
@@ -73,7 +73,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
       # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
       def predict(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         n_samples, = x.shape
         decision_values = decision_function(x)
         Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })

data/lib/rumale/naive_bayes/naive_bayes.rb

@@ -16,7 +16,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
       # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
       def predict(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         n_samples = x.shape.first
         decision_values = decision_function(x)
         Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })
@@ -27,7 +27,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the log-probailities.
       # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted log-probability of each class per sample.
       def predict_log_proba(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         n_samples, = x.shape
         log_likelihoods = decision_function(x)
         log_likelihoods - Numo::NMath.log(Numo::NMath.exp(log_likelihoods).sum(1)).reshape(n_samples, 1)
@@ -38,7 +38,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
       # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
       def predict_proba(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         Numo::NMath.exp(predict_log_proba(x)).abs
       end
     end
@@ -78,8 +78,8 @@ module Rumale
       #   to be used for fitting the model.
       # @return [GaussianNB] The learned classifier itself.
       def fit(x, y)
-        check_sample_array(x)
-        check_label_array(y)
+        x = check_convert_sample_array(x)
+        y = check_convert_label_array(y)
         check_sample_label_size(x, y)
         n_samples, = x.shape
         @classes = Numo::Int32[*y.to_a.uniq.sort]
@@ -94,7 +94,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
       # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence scores per sample for each class.
       def decision_function(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         n_classes = @classes.size
         log_likelihoods = Array.new(n_classes) do |l|
           Math.log(@class_priors[l]) - 0.5 * (
@@ -154,7 +154,7 @@ module Rumale
       #
       # @param smoothing_param [Float] The Laplace smoothing parameter.
       def initialize(smoothing_param: 1.0)
-        check_params_float(smoothing_param: smoothing_param)
+        check_params_numeric(smoothing_param: smoothing_param)
         check_params_positive(smoothing_param: smoothing_param)
         @params = {}
         @params[:smoothing_param] = smoothing_param
@@ -167,8 +167,8 @@ module Rumale
       #   to be used for fitting the model.
       # @return [MultinomialNB] The learned classifier itself.
       def fit(x, y)
-        check_sample_array(x)
-        check_label_array(y)
+        x = check_convert_sample_array(x)
+        y = check_convert_label_array(y)
         check_sample_label_size(x, y)
         n_samples, = x.shape
         @classes = Numo::Int32[*y.to_a.uniq.sort]
@@ -185,7 +185,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
       # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence scores per sample for each class.
       def decision_function(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         n_classes = @classes.size
         bin_x = x.gt(0)
         log_likelihoods = Array.new(n_classes) do |l|
@@ -243,7 +243,7 @@ module Rumale
       # @param smoothing_param [Float] The Laplace smoothing parameter.
       # @param bin_threshold [Float] The threshold for binarizing of features.
       def initialize(smoothing_param: 1.0, bin_threshold: 0.0)
-        check_params_float(smoothing_param: smoothing_param, bin_threshold: bin_threshold)
+        check_params_numeric(smoothing_param: smoothing_param, bin_threshold: bin_threshold)
         check_params_positive(smoothing_param: smoothing_param)
         @params = {}
         @params[:smoothing_param] = smoothing_param
@@ -257,8 +257,8 @@ module Rumale
       #   to be used for fitting the model.
       # @return [BernoulliNB] The learned classifier itself.
       def fit(x, y)
-        check_sample_array(x)
-        check_label_array(y)
+        x = check_convert_sample_array(x)
+        y = check_convert_label_array(y)
         check_sample_label_size(x, y)
         n_samples, = x.shape
         bin_x = Numo::DFloat[*x.gt(@params[:bin_threshold])]
@@ -278,7 +278,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
       # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence scores per sample for each class.
       def decision_function(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         n_classes = @classes.size
         bin_x = Numo::DFloat[*x.gt(@params[:bin_threshold])]
         not_bin_x = Numo::DFloat[*x.le(@params[:bin_threshold])]

data/lib/rumale/nearest_neighbors/k_neighbors_classifier.rb

@@ -35,7 +35,7 @@ module Rumale
       #
       # @param n_neighbors [Integer] The number of neighbors.
       def initialize(n_neighbors: 5)
-        check_params_integer(n_neighbors: n_neighbors)
+        check_params_numeric(n_neighbors: n_neighbors)
         check_params_positive(n_neighbors: n_neighbors)
         @params = {}
         @params[:n_neighbors] = n_neighbors
@@ -50,8 +50,8 @@ module Rumale
       # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
       # @return [KNeighborsClassifier] The learned classifier itself.
       def fit(x, y)
-        check_sample_array(x)
-        check_label_array(y)
+        x = check_convert_sample_array(x)
+        y = check_convert_label_array(y)
         check_sample_label_size(x, y)
         @prototypes = Numo::DFloat.asarray(x.to_a)
         @labels = Numo::Int32.asarray(y.to_a)
@@ -64,7 +64,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
       # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence scores per sample for each class.
       def decision_function(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         distance_matrix = PairwiseMetric.euclidean_distance(x, @prototypes)
         n_samples, n_prototypes = distance_matrix.shape
         n_classes = @classes.size
@@ -82,7 +82,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
       # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
       def predict(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         n_samples = x.shape.first
         decision_values = decision_function(x)
         Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })

data/lib/rumale/nearest_neighbors/k_neighbors_regressor.rb

@@ -30,7 +30,7 @@ module Rumale
       #
       # @param n_neighbors [Integer] The number of neighbors.
       def initialize(n_neighbors: 5)
-        check_params_integer(n_neighbors: n_neighbors)
+        check_params_numeric(n_neighbors: n_neighbors)
         check_params_positive(n_neighbors: n_neighbors)
         @params = {}
         @params[:n_neighbors] = n_neighbors
@@ -44,8 +44,8 @@ module Rumale
       # @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values to be used for fitting the model.
       # @return [KNeighborsRegressor] The learned regressor itself.
       def fit(x, y)
-        check_sample_array(x)
-        check_tvalue_array(y)
+        x = check_convert_sample_array(x)
+        y = check_convert_tvalue_array(y)
         check_sample_tvalue_size(x, y)
         @prototypes = x.dup
         @values = y.dup
@@ -57,7 +57,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
       # @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted values per sample.
       def predict(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         # Initialize some variables.
         n_samples, = x.shape
         n_prototypes, n_outputs = @values.shape

data/lib/rumale/neural_network/base_mlp.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator'
+
+module Rumale
+  # This module consists of the modules and classes for implementation multi-layer perceptron estimator.
+  module NeuralNetwork
+    # @!visibility private
+    # This module consists of the classes that implement layer functions of neural network.
+    module Layer
+      # @!visibility private
+      # Affine is a class that calculates the linear transform.
+      # This class is used internally.
+      class Affine
+        # @!visibility private
+        def initialize(n_inputs: nil, n_outputs: nil, optimizer: nil, rng: nil)
+          @weight = 0.01 * Rumale::Utils.rand_normal([n_inputs, n_outputs], rng)
+          @bias = Numo::DFloat.zeros(n_outputs)
+          @optimizer_weight = optimizer.dup
+          @optimizer_bias = optimizer.dup
+        end
+
+        # @!visibility private
+        def forward(x)
+          out = x.dot(@weight) + @bias
+
+          backward = proc do |dout|
+            dx = dout.dot(@weight.transpose)
+            dw = x.transpose.dot(dout)
+            db = dout.sum(0)
+
+            @weight = @optimizer_weight.call(@weight, dw)
+            @bias = @optimizer_bias.call(@bias, db)
+
+            dx
+          end
+
+          [out, backward]
+        end
+      end
+
+      # @!visibility private
+      # Dropout is a class that performs dropout regularization.
+      # This class is used internally.
+      class Dropout
+        # @!visibility private
+        def initialize(rate: 0.3, rng: nil)
+          @rate = rate
+          @rng = rng
+        end
+
+        # @!visibility private
+        def forward(x)
+          rand_mat = Rumale::Utils.rand_uniform(x.shape, @rng)
+          mask = rand_mat.ge(@rate)
+          out = x * mask
+          out *= 1.fdiv(1 - @rate) if @rate < 1.0
+
+          backward = proc { |dout| dout * mask }
+
+          [out, backward]
+        end
+      end
+
+      # @!visibility private
+      # ReLU is a class that calculates rectified linear function.
+      # This class is used internally.
+      class Relu
+        # @!visibility private
+        def forward(x)
+          mask = x.gt(0)
+          out = x * mask
+
+          backward = proc { |dout| dout * mask }
+
+          [out, backward]
+        end
+      end
+    end
+
+    # @!visibility private
+    # This module consists of the classes that implement loss function for neural network.
+    module Loss
+      # @!visibility private
+      # MeanSquaredError is a class that calculates mean squared error for regression task.
+      # This class is used internally.
+      class MeanSquaredError
+        # @!visibility private
+        def call(out, y)
+          sz_batch = y.shape[0]
+          diff = out - y
+          loss = (diff**2).sum.fdiv(sz_batch)
+          dout = 2.fdiv(sz_batch) * diff
+          [loss, dout]
+        end
+      end
+
+      # @!visibility private
+      # SoftmaxCrossEntropy is a class that calculates softmax cross-entropy for classification task.
+      # This class is used internally.
+      class SoftmaxCrossEntropy
+        # @!visibility private
+        def call(out, y)
+          sz_batch = y.shape[0]
+          z = softmax(out)
+          loss = -(y * Numo::NMath.log(z + 1e-8)).sum.fdiv(sz_batch)
+          dout = (z - y) / sz_batch
+          [loss, dout]
+        end
+
+        private
+
+        def softmax(x)
+          clip = x.max(-1).expand_dims(-1)
+          exp_x = Numo::NMath.exp(x - clip)
+          exp_x / exp_x.sum(-1).expand_dims(-1)
+        end
+      end
+    end
+
+    # @!visibility private
+    # This module consists of the classes for implementing neural network model.
+    module Model
+      # @!visibility private
+      attr_reader :layers
+
+      # @!visibility private
+      # Sequential is a class that implements linear stack model.
+      # This class is used internally.
+      class Sequential
+        # @!visibility private
+        def initialize
+          @layers = []
+        end
+
+        # @!visibility private
+        def push(ops)
+          @layers.push(ops)
+          self
+        end
+
+        # @!visibility private
+        def delete_dropout
+          @layers.delete_if { |node| node.is_a?(Layer::Dropout) }
+          self
+        end
+
+        # @!visibility private
+        def forward(x)
+          backprops = []
+          out = x.dup
+
+          @layers.each do |l|
+            out, bw = l.forward(out)
+            backprops.push(bw)
+          end
+
+          backward = proc do |dout|
+            backprops.reverse_each { |bw| dout = bw.call(dout) }
+            dout
+          end
+
+          [out, backward]
+        end
+      end
+    end
+
+    # BaseMLP is an abstract class for implementation of multi-layer peceptron estimator.
+    # This class is used internally.
+    class BaseMLP
+      include Base::BaseEstimator
+
+      # Create a multi-layer perceptron estimator.
+      #
+      # @param hidden_units [Array] The number of units in the i-th hidden layer.
+      # @param dropout_rate [Float] The rate of the units to drop.
+      # @param learning_rate [Float] The initial value of learning rate in Adam optimizer.
+      # @param decay1 [Float] The smoothing parameter for the first moment in Adam optimizer.
+      # @param decay2 [Float] The smoothing parameter for the second moment in Adam optimizer.
+      # @param max_iter [Integer] The maximum number of iterations.
+      # @param batch_size [Intger] The size of the mini batches.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(hidden_units: [128, 128], dropout_rate: 0.4, learning_rate: 0.001, decay1: 0.9, decay2: 0.999,
+                     max_iter: 10000, batch_size: 50, tol: 1e-4, verbose: false, random_seed: nil)
+        @params = {}
+        @params[:hidden_units] = hidden_units
+        @params[:dropout_rate] = dropout_rate
+        @params[:learning_rate] = learning_rate
+        @params[:decay1] = decay1
+        @params[:decay2] = decay2
+        @params[:max_iter] = max_iter
+        @params[:batch_size] = batch_size
+        @params[:tol] = tol
+        @params[:verbose] = verbose
+        @params[:random_seed] = random_seed
+        @params[:random_seed] ||= srand
+        @n_iter = nil
+        @rng = Random.new(@params[:random_seed])
+      end
+
+      private
+
+      def buld_network(n_inputs, n_outputs, srng = nil)
+        adam = Rumale::Optimizer::Adam.new(learning_rate: @params[:learning_rate], decay1: @params[:decay1], decay2: @params[:decay2])
+        model = Model::Sequential.new
+        n_units = [n_inputs, *@params[:hidden_units]]
+        n_units.each_cons(2) do |n_in, n_out|
+          model.push(Layer::Affine.new(n_inputs: n_in, n_outputs: n_out, optimizer: adam, rng: srng))
+          model.push(Layer::Relu.new)
+          model.push(Layer::Dropout.new(rate: @params[:dropout_rate], rng: srng))
+        end
+        model.push(Layer::Affine.new(n_inputs: n_units[-1], n_outputs: n_outputs, optimizer: adam, rng: srng))
+      end
+
+      def train(x, y, network, loss_func, srng = nil)
+        class_name = self.class.to_s.split('::').last
+
+        n_samples = x.shape[0]
+        rand_ids = [*0...n_samples].shuffle(random: srng)
+
+        @params[:max_iter].times do |t|
+          # random sampling
+          subset_ids = rand_ids.shift(@params[:batch_size])
+          rand_ids.concat(subset_ids)
+          sub_x = x[subset_ids, true].dup
+          sub_y = y[subset_ids, true].dup
+          # forward
+          out, backward = network.forward(sub_x)
+          # calc loss function
+          loss, dout = loss_func.call(out, sub_y)
+          @n_iter = t + 1
+          puts "[#{class_name}] Loss after #{@n_iter} iterations: #{loss}" if @params[:verbose] && (@n_iter % 10).zero?
+          break if loss < @params[:tol]
+          # backward
+          backward.call(dout)
+        end
+
+        network
+      end
+    end
+  end
+end