rumale 0.13.8 → 0.14.0

data/lib/rumale/neural_network/mlp_classifier.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'rumale/base/classifier'
+require 'rumale/neural_network/base_mlp'
+require 'rumale/preprocessing/label_binarizer'
+
+module Rumale
+  module NeuralNetwork
+    # MLPClassifier is a class that implements classifier based on multi-layer perceptron.
+    # MLPClassifier use ReLu as the activation function and Adam as the optimization method
+    # and softmax cross entropy as the loss function.
+    #
+    # @example
+    #   estimator = Rumale::NeuralNetwork::MLPClassifier.new(hidden_units: [100, 100], dropout_rate: 0.3)
+    #   estimator.fit(training_samples, traininig_labels)
+    #   results = estimator.predict(testing_samples)
+    class MLPClassifier < BaseMLP
+      include Base::Classifier
+
+      # Return the network.
+      # @return [Rumale::NeuralNetwork::Model::Sequential]
+      attr_reader :network
+
+      # Return the class labels.
+      # @return [Numo::Int32] (size: n_classes)
+      attr_reader :classes
+
+      # Return the number of iterations run for optimization
+      # @return [Integer]
+      attr_reader :n_iter
+
+      # Return the random generator.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new classifier with multi-layer preceptron.
+      #
+      # @param hidden_units [Array] The number of units in the i-th hidden layer.
+      # @param dropout_rate [Float] The rate of the units to drop.
+      # @param learning_rate [Float] The initial value of learning rate in Adam optimizer.
+      # @param decay1 [Float] The smoothing parameter for the first moment in Adam optimizer.
+      # @param decay2 [Float] The smoothing parameter for the second moment in Adam optimizer.
+      # @param max_iter [Integer] The maximum number of iterations.
+      # @param batch_size [Intger] The size of the mini batches.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(hidden_units: [128, 128], dropout_rate: 0.4, learning_rate: 0.001, decay1: 0.9, decay2: 0.999,
+                     max_iter: 10000, batch_size: 50, tol: 1e-4, verbose: false, random_seed: nil)
+        check_params_type(Array, hidden_units: hidden_units)
+        check_params_numeric(dropout_rate: dropout_rate, learning_rate: learning_rate, decay1: decay1, decay2: decay2,
+                             max_iter: max_iter, batch_size: batch_size, tol: tol)
+        check_params_boolean(verbose: verbose)
+        check_params_numeric_or_nil(random_seed: random_seed)
+        super
+        @classes = nil
+        @network = nil
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [MLPClassifier] The learned classifier itself.
+      def fit(x, y)
+        x = check_convert_sample_array(x)
+        y = check_convert_label_array(y)
+        check_sample_label_size(x, y)
+
+        @classes = Numo::Int32[*y.to_a.uniq.sort]
+        n_labels = @classes.size
+        n_features = x.shape[1]
+        sub_rng = @rng.dup
+
+        loss = Loss::SoftmaxCrossEntropy.new
+        @network = buld_network(n_features, n_labels, sub_rng)
+        @network = train(x, one_hot_encode(y), @network, loss, sub_rng)
+        @network.delete_dropout
+
+        self
+      end
+
+      # Predict class labels for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
+      def predict(x)
+        x = check_convert_sample_array(x)
+        n_samples = x.shape[0]
+        decision_values = predict_proba(x)
+        predicted = Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] }
+        Numo::Int32.asarray(predicted)
+      end
+
+      # Predict probability for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
+      def predict_proba(x)
+        x = check_convert_sample_array(x)
+        out, = @network.forward(x)
+        softmax(out)
+      end
+
+      private
+
+      def one_hot_encode(y)
+        encoder = Rumale::Preprocessing::LabelBinarizer.new
+        encoder.fit_transform(y)
+      end
+
+      def softmax(x)
+        clip = x.max(-1).expand_dims(-1)
+        exp_x = Numo::NMath.exp(x - clip)
+        exp_x / exp_x.sum(-1).expand_dims(-1)
+      end
+    end
+  end
+end

data/lib/rumale/neural_network/mlp_regressor.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'rumale/base/regressor'
+require 'rumale/neural_network/base_mlp'
+
+module Rumale
+  module NeuralNetwork
+    # MLPRegressor is a class that implements regressor based on multi-layer perceptron.
+    # MLPRegressor use ReLu as the activation function and Adam as the optimization method
+    # and mean squared error as the loss function.
+    #
+    # @example
+    #   estimator = Rumale::NeuralNetwork::MLPRegressor.new(hidden_units: [100, 100], dropout_rate: 0.3)
+    #   estimator.fit(training_samples, traininig_labels)
+    #   results = estimator.predict(testing_samples)
+    class MLPRegressor < BaseMLP
+      include Base::Regressor
+
+      # Return the network.
+      # @return [Rumale::NeuralNetwork::Model::Sequential]
+      attr_reader :network
+
+      # Return the number of iterations run for optimization
+      # @return [Integer]
+      attr_reader :n_iter
+
+      # Return the random generator.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new regressor with multi-layer perceptron.
+      #
+      # @param hidden_units [Array] The number of units in the i-th hidden layer.
+      # @param dropout_rate [Float] The rate of the units to drop.
+      # @param learning_rate [Float] The initial value of learning rate in Adam optimizer.
+      # @param decay1 [Float] The smoothing parameter for the first moment in Adam optimizer.
+      # @param decay2 [Float] The smoothing parameter for the second moment in Adam optimizer.
+      # @param max_iter [Integer] The maximum number of iterations.
+      # @param batch_size [Intger] The size of the mini batches.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(hidden_units: [128, 128], dropout_rate: 0.4, learning_rate: 0.001, decay1: 0.9, decay2: 0.999,
+                     max_iter: 10000, batch_size: 50, tol: 1e-4, verbose: false, random_seed: nil)
+        check_params_type(Array, hidden_units: hidden_units)
+        check_params_numeric(dropout_rate: dropout_rate, learning_rate: learning_rate, decay1: decay1, decay2: decay2,
+                             max_iter: max_iter, batch_size: batch_size, tol: tol)
+        check_params_boolean(verbose: verbose)
+        check_params_numeric_or_nil(random_seed: random_seed)
+        super
+        @network = nil
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The taget values to be used for fitting the model.
+      # @return [MLPRegressor] The learned regressor itself.
+      def fit(x, y)
+        x = check_convert_sample_array(x)
+        y = check_convert_tvalue_array(y)
+        check_sample_tvalue_size(x, y)
+
+        y = y.expand_dims(1) if y.ndim == 1
+        n_targets = y.shape[1]
+        n_features = x.shape[1]
+        sub_rng = @rng.dup
+
+        loss = Loss::MeanSquaredError.new
+        @network = buld_network(n_features, n_targets, sub_rng)
+        @network = train(x, y, @network, loss, sub_rng)
+        @network.delete_dropout
+
+        self
+      end
+
+      # Predict values for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
+      # @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted values per sample.
+      def predict(x)
+        x = check_convert_sample_array(x)
+        out, = @network.forward(x)
+        out = out[true, 0] if out.shape[1] == 1
+        out
+      end
+    end
+  end
+end

data/lib/rumale/optimizer/ada_grad.rb

@@ -22,7 +22,7 @@ module Rumale
       #
       # @param learning_rate [Float] The initial value of learning rate.
       def initialize(learning_rate: 0.01)
-        check_params_float(learning_rate: learning_rate)
+        check_params_numeric(learning_rate: learning_rate)
         check_params_positive(learning_rate: learning_rate)
         @params = {}
         @params[:learning_rate] = learning_rate

data/lib/rumale/optimizer/adam.rb

@@ -24,7 +24,7 @@ module Rumale
       # @param decay1 [Float] The smoothing parameter for the first moment.
       # @param decay2 [Float] The smoothing parameter for the second moment.
       def initialize(learning_rate: 0.001, decay1: 0.9, decay2: 0.999)
-        check_params_float(learning_rate: learning_rate, decay1: decay1, decay2: decay2)
+        check_params_numeric(learning_rate: learning_rate, decay1: decay1, decay2: decay2)
         check_params_positive(learning_rate: learning_rate, decay1: decay1, decay2: decay2)
         @params = {}
         @params[:learning_rate] = learning_rate
@@ -41,8 +41,8 @@ module Rumale
       # @param gradient [Numo::DFloat] (shape: [n_features]) The gradient for updating the weight.
       # @return [Numo::DFloat] (shape: [n_feautres]) The updated weight.
       def call(weight, gradient)
-        @fst_moment ||= Numo::DFloat.zeros(weight.shape[0])
-        @sec_moment ||= Numo::DFloat.zeros(weight.shape[0])
+        @fst_moment ||= Numo::DFloat.zeros(weight.shape)
+        @sec_moment ||= Numo::DFloat.zeros(weight.shape)
 
         @iter += 1
 

data/lib/rumale/optimizer/nadam.rb

@@ -25,7 +25,7 @@ module Rumale
       # @param decay1 [Float] The smoothing parameter for the first moment.
       # @param decay2 [Float] The smoothing parameter for the second moment.
       def initialize(learning_rate: 0.01, decay1: 0.9, decay2: 0.999)
-        check_params_float(learning_rate: learning_rate, decay1: decay1, decay2: decay2)
+        check_params_numeric(learning_rate: learning_rate, decay1: decay1, decay2: decay2)
         check_params_positive(learning_rate: learning_rate, decay1: decay1, decay2: decay2)
         @params = {}
         @params[:learning_rate] = learning_rate

data/lib/rumale/optimizer/rmsprop.rb

@@ -25,7 +25,7 @@ module Rumale
       # @param momentum [Float] The initial value of momentum.
       # @param decay [Float] The smooting parameter.
       def initialize(learning_rate: 0.01, momentum: 0.9, decay: 0.9)
-        check_params_float(learning_rate: learning_rate, momentum: momentum, decay: decay)
+        check_params_numeric(learning_rate: learning_rate, momentum: momentum, decay: decay)
         check_params_positive(learning_rate: learning_rate, momentum: momentum, decay: decay)
         @params = {}
         @params[:learning_rate] = learning_rate

data/lib/rumale/optimizer/sgd.rb

@@ -21,7 +21,7 @@ module Rumale
       # @param momentum [Float] The initial value of momentum.
       # @param decay [Float] The smooting parameter.
       def initialize(learning_rate: 0.01, momentum: 0.0, decay: 0.0)
-        check_params_float(learning_rate: learning_rate, momentum: momentum, decay: decay)
+        check_params_numeric(learning_rate: learning_rate, momentum: momentum, decay: decay)
         check_params_positive(learning_rate: learning_rate, momentum: momentum, decay: decay)
         @params = {}
         @params[:learning_rate] = learning_rate

data/lib/rumale/optimizer/yellow_fin.rb

@@ -25,8 +25,7 @@ module Rumale
       # @param decay [Float] The smooting parameter.
       # @param window_width [Integer] The sliding window width for searching curvature range.
       def initialize(learning_rate: 0.01, momentum: 0.9, decay: 0.999, window_width: 20)
-        check_params_float(learning_rate: learning_rate, momentum: momentum, decay: decay)
-        check_params_integer(window_width: window_width)
+        check_params_numeric(learning_rate: learning_rate, momentum: momentum, decay: decay, window_width: window_width)
         check_params_positive(learning_rate: learning_rate, momentum: momentum, decay: decay, window_width: window_width)
         @params = {}
         @params[:learning_rate] = learning_rate

data/lib/rumale/pairwise_metric.rb

@@ -13,8 +13,8 @@ module Rumale
       # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
       def euclidean_distance(x, y = nil)
         y = x if y.nil?
-        Rumale::Validation.check_sample_array(x)
-        Rumale::Validation.check_sample_array(y)
+        x = Rumale::Validation.check_convert_sample_array(x)
+        y = Rumale::Validation.check_convert_sample_array(y)
         Numo::NMath.sqrt(squared_error(x, y).abs)
       end
 
@@ -25,8 +25,8 @@ module Rumale
       # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
       def manhattan_distance(x, y = nil)
         y = x if y.nil?
-        Rumale::Validation.check_sample_array(x)
-        Rumale::Validation.check_sample_array(y)
+        x = Rumale::Validation.check_convert_sample_array(x)
+        y = Rumale::Validation.check_convert_sample_array(y)
         n_samples_x = x.shape[0]
         n_samples_y = y.shape[0]
         distance_mat = Numo::DFloat.zeros(n_samples_x, n_samples_y)
@@ -43,8 +43,8 @@ module Rumale
       # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
       def squared_error(x, y = nil)
         y = x if y.nil?
-        Rumale::Validation.check_sample_array(x)
-        Rumale::Validation.check_sample_array(y)
+        x = Rumale::Validation.check_convert_sample_array(x)
+        y = Rumale::Validation.check_convert_sample_array(y)
         n_features = x.shape[1]
         one_vec = Numo::DFloat.ones(n_features).expand_dims(1)
         sum_x_vec = (x**2).dot(one_vec)
@@ -62,9 +62,9 @@ module Rumale
       def rbf_kernel(x, y = nil, gamma = nil)
         y = x if y.nil?
         gamma ||= 1.0 / x.shape[1]
-        Rumale::Validation.check_sample_array(x)
-        Rumale::Validation.check_sample_array(y)
-        Rumale::Validation.check_params_float(gamma: gamma)
+        x = Rumale::Validation.check_convert_sample_array(x)
+        y = Rumale::Validation.check_convert_sample_array(y)
+        Rumale::Validation.check_params_numeric(gamma: gamma)
         Numo::NMath.exp(-gamma * squared_error(x, y).abs)
       end
 
@@ -75,8 +75,8 @@ module Rumale
       # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
       def linear_kernel(x, y = nil)
         y = x if y.nil?
-        Rumale::Validation.check_sample_array(x)
-        Rumale::Validation.check_sample_array(y)
+        x = Rumale::Validation.check_convert_sample_array(x)
+        y = Rumale::Validation.check_convert_sample_array(y)
         x.dot(y.transpose)
       end
 
@@ -91,10 +91,9 @@ module Rumale
       def polynomial_kernel(x, y = nil, degree = 3, gamma = nil, coef = 1)
         y = x if y.nil?
         gamma ||= 1.0 / x.shape[1]
-        Rumale::Validation.check_sample_array(x)
-        Rumale::Validation.check_sample_array(y)
-        Rumale::Validation.check_params_float(gamma: gamma)
-        Rumale::Validation.check_params_integer(degree: degree, coef: coef)
+        x = Rumale::Validation.check_convert_sample_array(x)
+        y = Rumale::Validation.check_convert_sample_array(y)
+        Rumale::Validation.check_params_numeric(gamma: gamma, degree: degree, coef: coef)
         (x.dot(y.transpose) * gamma + coef)**degree
       end
 
@@ -108,10 +107,9 @@ module Rumale
       def sigmoid_kernel(x, y = nil, gamma = nil, coef = 1)
         y = x if y.nil?
         gamma ||= 1.0 / x.shape[1]
-        Rumale::Validation.check_sample_array(x)
-        Rumale::Validation.check_sample_array(y)
-        Rumale::Validation.check_params_float(gamma: gamma)
-        Rumale::Validation.check_params_integer(coef: coef)
+        x = Rumale::Validation.check_convert_sample_array(x)
+        y = Rumale::Validation.check_convert_sample_array(y)
+        Rumale::Validation.check_params_numeric(gamma: gamma, coef: coef)
         Numo::NMath.tanh(x.dot(y.transpose) * gamma + coef)
       end
     end

data/lib/rumale/pipeline/pipeline.rb

@@ -40,7 +40,7 @@ module Rumale
       # @param y [Numo::NArray] (shape: [n_samples, n_outputs]) The target values or labels to be used for fitting the model.
       # @return [Pipeline] The learned pipeline itself.
       def fit(x, y)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         trans_x = apply_transforms(x, y, fit: true)
         last_estimator&.fit(trans_x, y)
         self
@@ -52,7 +52,7 @@ module Rumale
       # @param y [Numo::NArray] (shape: [n_samples, n_outputs], default: nil) The target values or labels to be used for fitting the model.
       # @return [Numo::NArray] The predicted results by last estimator.
       def fit_predict(x, y = nil)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         trans_x = apply_transforms(x, y, fit: true)
         last_estimator.fit_predict(trans_x)
       end
@@ -63,7 +63,7 @@ module Rumale
       # @param y [Numo::NArray] (shape: [n_samples, n_outputs], default: nil) The target values or labels to be used for fitting the model.
       # @return [Numo::NArray] The predicted results by last estimator.
       def fit_transform(x, y = nil)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         trans_x = apply_transforms(x, y, fit: true)
         last_estimator.fit_transform(trans_x, y)
       end
@@ -73,7 +73,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
       # @return [Numo::DFloat] (shape: [n_samples]) Confidence score per sample.
       def decision_function(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         trans_x = apply_transforms(x)
         last_estimator.decision_function(trans_x)
       end
@@ -83,7 +83,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to obtain prediction result.
       # @return [Numo::NArray] The predicted results by last estimator.
       def predict(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         trans_x = apply_transforms(x)
         last_estimator.predict(trans_x)
       end
@@ -93,7 +93,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the log-probailities.
       # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted log-probability of each class per sample.
       def predict_log_proba(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         trans_x = apply_transforms(x)
         last_estimator.predict_log_proba(trans_x)
       end
@@ -103,7 +103,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
       # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
       def predict_proba(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         trans_x = apply_transforms(x)
         last_estimator.predict_proba(trans_x)
       end
@@ -113,7 +113,7 @@ module Rumale
       # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to be transformed.
       # @return [Numo::DFloat] (shape: [n_samples, n_components]) The transformed samples.
       def transform(x)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         trans_x = apply_transforms(x)
         last_estimator.nil? ? trans_x : last_estimator.transform(trans_x)
       end
@@ -123,7 +123,7 @@ module Rumale
       # @param z [Numo::DFloat] (shape: [n_samples, n_components]) The transformed samples to be restored into original space.
       # @return [Numo::DFloat] (shape: [n_samples, n_featuress]) The restored samples.
       def inverse_transform(z)
-        check_sample_array(z)
+        z = check_convert_sample_array(z)
         itrans_z = z
         @steps.keys.reverse_each do |name|
           transformer = @steps[name]
@@ -139,7 +139,7 @@ module Rumale
       # @param y [Numo::NArray] (shape: [n_samples, n_outputs]) True target values or labels for testing data.
       # @return [Float] The score of last estimator
       def score(x, y)
-        check_sample_array(x)
+        x = check_convert_sample_array(x)
         trans_x = apply_transforms(x)
         last_estimator.score(trans_x, y)
       end