rumale 0.16.1 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30476b58c5c5b39567f1cb3c8346a7c354fbf8d30401555fa2e02995021b759d
-  data.tar.gz: 6f664b0c279e0fef2dc47e608cdc2737318274b45017d6d60f0dd516aa2ebb48
+  metadata.gz: d1071dfdccfc177ea5902e5e1b09fce084fd4b6ce403fae6797e6b93c3f826ad
+  data.tar.gz: 30768881f5c826f59dbcca0b17a1192dbdc17ca835c8bcc626e874391131bf92
 SHA512:
-  metadata.gz: aa51f865e4995901e5587e3089fae724a57022d96c95d2b852cfde99f85f9aae7035c4edfe6c4a7899c22674778e1bfc0332ef83b6f234a8c9e8aa982e55e833
-  data.tar.gz: 55e209725a0c716b1f450bed025fceefe36dafe278b96648ac60079e3968778840bc4e1e75ff4181abafc4f98eb93cc40d8e2e0e3b5bf078bc94cf8b9a5dc50d
+  metadata.gz: e748eedf78b040a7dbe1a1b744f87f1c1e9c3ae751417711eccd5f69dca68335f1a206b9258503da70a8486c3d8588d61ae78081ebdecc6a8ee40f85383a319f
+  data.tar.gz: 2abae603660179e05f8341ab5351fb9e028549674bb13901e4cae4dfd13c99995de0de3c63f8c75182acd155a1d2171b02e4db74fcaa08c1108a1a0e92ad3eee

data/.rubocop.yml

@@ -15,7 +15,7 @@ AllCops:
 Style/Documentation:
   Enabled: false
 
-Metrics/LineLength:
+Layout/LineLength:
   Max: 145
   IgnoredPatterns: ['(\A|\s)#']
 
@@ -43,7 +43,7 @@ Metrics/BlockLength:
     - 'spec/**/*'
 
 Metrics/ParameterLists:
-  Max: 10
+  Max: 15
 
 Security/MarshalLoad:
   Enabled: false

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+# 0.17.0
+## Breaking changes
+- Fix all linear model estimators to use the new abstract class ([BaseSGD](https://yoshoku.github.io/rumale/doc/Rumale/LinearModel/BaseSGD.html)) introduced in version 0.16.1.
+  The major differences from the old abstract class are that
+  the optimizer of LinearModel estimators is fixed to mini-batch SGD with momentum term,
+  the max_iter parameter indicates the number of epochs instead of the maximum number of iterations,
+  the fit_bias parameter is true by default, and elastic-net style regularization can be used.
+  Note that there are additions and changes to hyperparameters.
+  Existing trained linear models may need to re-train the model and adjust the hyperparameters.
+  - [LogisticRegression](https://yoshoku.github.io/rumale/doc/Rumale/LinearModel/LogisticRegression.html)
+  - [SVC](https://yoshoku.github.io/rumale/doc/Rumale/LinearModel/SVC.html)
+  - [LinearRegression](https://yoshoku.github.io/rumale/doc/Rumale/LinearModel/LinearRegression.html)
+  - [Rdige](https://yoshoku.github.io/rumale/doc/Rumale/LinearModel/Ridge.html)
+  - [Lasso](https://yoshoku.github.io/rumale/doc/Rumale/LinearModel/Lasso.html)
+  - [SVR](https://yoshoku.github.io/rumale/doc/Rumale/LinearModel/SVR.html)
+- Change the default value of solver parameter on LinearRegression and Ridge to 'auto'.
+If Numo::Linalg is loaded, 'svd' is selected for the solver, otherwise 'sgd' is selected.
+- The meaning of the `max_iter` parameter of the factorization machine estimators
+has been changed from the maximum number of iterations to the number of epochs.
+  - [FactorizationMachineClassifier](https://yoshoku.github.io/rumale/doc/Rumale/PolynomialModel/FactorizationMachineClassifier.html)
+  - [FactorizationMachineRegressor](https://yoshoku.github.io/rumale/doc/Rumale/PolynomialModel/FactorizationMachineRegressor.html)
+
 # 0.16.1
 - Add regressor class for [ElasticNet](https://yoshoku.github.io/rumale/doc/Rumale/LinearModel/ElasticNet.html).
 - Add new linear model abstract class.

data/README.md

@@ -6,7 +6,7 @@
 [![Coverage Status](https://coveralls.io/repos/github/yoshoku/rumale/badge.svg?branch=master)](https://coveralls.io/github/yoshoku/rumale?branch=master)
 [![Gem Version](https://badge.fury.io/rb/rumale.svg)](https://badge.fury.io/rb/rumale)
 [![BSD 2-Clause License](https://img.shields.io/badge/License-BSD%202--Clause-orange.svg)](https://github.com/yoshoku/rumale/blob/master/LICENSE.txt)
-[![Documentation](http://img.shields.io/badge/docs-rdoc.info-blue.svg)](https://yoshoku.github.io/rumale/doc/)
+[![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://yoshoku.github.io/rumale/doc/)
 
 Rumale (**Ru**by **ma**chine **le**arning) is a machine learning library in Ruby.
 Rumale provides machine learning algorithms with interfaces similar to Scikit-Learn in Python.
@@ -37,6 +37,10 @@ Or install it yourself as:
 
     $ gem install rumale
 
+## Documentation
+
+- [Rumale API Documentation](https://yoshoku.github.io/rumale/doc/)
+
 ## Usage
 
 ### Example 1. XOR data
@@ -95,7 +99,7 @@ transformer = Rumale::KernelApproximation::RBF.new(gamma: 0.0001, n_components:
 transformed = transformer.fit_transform(samples)
 
 # Train linear SVM classifier.
-classifier = Rumale::LinearModel::SVC.new(reg_param: 0.0001, max_iter: 1000, batch_size: 50, random_seed: 1)
+classifier = Rumale::LinearModel::SVC.new(reg_param: 0.0001, random_seed: 1)
 classifier.fit(transformed, labels)
 
 # Save the model.
@@ -132,7 +136,7 @@ Execution of the above scripts result in the following.
 ```bash
 $ ruby train.rb
 $ ruby test.rb
-Accuracy: 98.4%
+Accuracy: 98.7%
 ```
 
 ### Example 3. Cross-validation
@@ -144,7 +148,7 @@ require 'rumale'
 samples, labels = Rumale::Dataset.load_libsvm_file('pendigits')
 
 # Define the estimator to be evaluated.
-lr = Rumale::LinearModel::LogisticRegression.new(reg_param: 0.0001, random_seed: 1)
+lr = Rumale::LinearModel::LogisticRegression.new(learning_rate: 0.00001, reg_param: 0.0001, random_seed: 1)
 
 # Define the evaluation measure, splitting strategy, and cross validation.
 ev = Rumale::EvaluationMeasure::LogLoss.new
@@ -163,7 +167,7 @@ Execution of the above scripts result in the following.
 
 ```bash
 $ ruby cross_validation.rb
-5-CV mean log-loss: 0.476
+5-CV mean log-loss: 0.355
 ```
 
 ### Example 4. Pipeline
@@ -176,7 +180,7 @@ samples, labels = Rumale::Dataset.load_libsvm_file('pendigits')
 
 # Construct pipeline with kernel approximation and SVC.
 rbf = Rumale::KernelApproximation::RBF.new(gamma: 0.0001, n_components: 800, random_seed: 1)
-svc = Rumale::LinearModel::SVC.new(reg_param: 0.0001, max_iter: 1000, random_seed: 1)
+svc = Rumale::LinearModel::SVC.new(reg_param: 0.0001, random_seed: 1)
 pipeline = Rumale::Pipeline::Pipeline.new(steps: { trns: rbf, clsf: svc })
 
 # Define the splitting strategy and cross validation.
@@ -195,7 +199,7 @@ Execution of the above scripts result in the following.
 
 ```bash
 $ ruby pipeline.rb
-5-CV mean accuracy: 99.2 %
+5-CV mean accuracy: 99.6 %
 ```
 
 ## Speeding up

data/lib/rumale/linear_model/base_linear_model.rb

@@ -5,10 +5,15 @@ require 'rumale/optimizer/nadam'
 
 module Rumale
   module LinearModel
+    # @note
+    #   In version 0.17.0, a new linear model abstract class called BaseSGD is introduced.
+    #   BaseLienarModel is deprecated and will be removed in the future.
+    #
     # BaseLinearModel is an abstract class for implementation of linear estimator
     # with mini-batch stochastic gradient descent optimization.
     # This class is used for internal process.
     class BaseLinearModel
+      # :nocov:
       include Base::BaseEstimator
 
       # Initialize a linear estimator.
@@ -26,6 +31,7 @@ module Rumale
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0,
                      max_iter: 1000, batch_size: 10, optimizer: nil, n_jobs: nil, random_seed: nil)
+        warn 'warning: BaseLinearModel is deprecated. Use BaseSGD instead.'
         @params = {}
         @params[:reg_param] = reg_param
         @params[:fit_bias] = fit_bias
@@ -88,6 +94,7 @@ module Rumale
           [weight, 0.0]
         end
       end
+      # :nocov:
     end
   end
 end

data/lib/rumale/linear_model/base_sgd.rb

@@ -99,6 +99,61 @@ module Rumale
           2.fdiv(y.shape[0]) * (out - y)
         end
       end
+
+      # @!visibility private
+      # LogLoss is a class that calculates logistic loss for logistic regression.
+      class LogLoss
+        # @!visibility private
+        def loss(out, y)
+          Numo::NMath.log(1 + Numo::NMath.exp(-y * out)).sum.fdiv(y.shape[0])
+        end
+
+        # @!visibility private
+        def dloss(out, y)
+          y / (1 + Numo::NMath.exp(-y * out)) - y
+        end
+      end
+
+      # @!visibility private
+      # HingeLoss is a class that calculates hinge loss for support vector classifier.
+      class HingeLoss
+        # @!visibility private
+        def loss(out, y)
+          out.class.maximum(0.0, 1 - y * out).sum.fdiv(y.shape[0])
+        end
+
+        # @!visibility private
+        def dloss(out, y)
+          tids = (y * out).lt(1)
+          d = Numo::DFloat.zeros(y.shape[0])
+          d[tids] = -y[tids] if tids.count.positive?
+          d
+        end
+      end
+
+      # @!visibility private
+      # EpsilonInsensitive is a class that calculates epsilon insensitive for support vector regressor.
+      class EpsilonInsensitive
+        # @!visibility private
+        def initialize(epsilon: 0.1)
+          @epsilon = epsilon
+        end
+
+        # @!visibility private
+        def loss(out, y)
+          out.class.maximum(0.0, (y - out).abs - @epsilon).sum.fdiv(y.shape[0])
+        end
+
+        # @!visibility private
+        def dloss(out, y)
+          d = Numo::DFloat.zeros(y.shape[0])
+          tids = (out - y).gt(@epsilon)
+          d[tids] = 1 if tids.count.positive?
+          tids = (y - out).gt(@epsilon)
+          d[tids] = -1 if tids.count.positive?
+          d
+        end
+      end
     end
 
     # BaseSGD is an abstract class for implementation of linear model with mini-batch stochastic gradient descent (SGD) optimization.

data/lib/rumale/linear_model/elastic_net.rb

@@ -59,13 +59,13 @@ module Rumale
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
                      reg_param: 1.0, l1_ratio: 0.5, fit_bias: true, bias_scale: 1.0,
-                     max_iter: 100, batch_size: 50, tol: 1e-4,
+                     max_iter: 200, batch_size: 50, tol: 1e-4,
                      n_jobs: nil, verbose: false, random_seed: nil)
         check_params_numeric(learning_rate: learning_rate, momentum: momentum,
                              reg_param: reg_param, l1_ratio: l1_ratio, bias_scale: bias_scale,
                              max_iter: max_iter, batch_size: batch_size, tol: tol)
         check_params_boolean(fit_bias: fit_bias, verbose: verbose)
-        check_params_numeric_or_nil(decay: nil, n_jobs: n_jobs, random_seed: random_seed)
+        check_params_numeric_or_nil(decay: decay, n_jobs: n_jobs, random_seed: random_seed)
         check_params_positive(learning_rate: learning_rate, reg_param: reg_param, max_iter: max_iter, batch_size: batch_size)
         super()
         @params.merge!(method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h)

data/lib/rumale/linear_model/lasso.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'rumale/linear_model/base_linear_model'
+require 'rumale/linear_model/base_sgd'
 require 'rumale/base/regressor'
 
 module Rumale
@@ -10,14 +10,15 @@ module Rumale
     #
     # @example
     #   estimator =
-    #     Rumale::LinearModel::Lasso.new(reg_param: 0.1, max_iter: 1000, batch_size: 20, random_seed: 1)
+    #     Rumale::LinearModel::Lasso.new(reg_param: 0.1, max_iter: 500, batch_size: 20, random_seed: 1)
     #   estimator.fit(training_samples, traininig_values)
     #   results = estimator.predict(testing_samples)
     #
     # *Reference*
     # - S. Shalev-Shwartz and Y. Singer, "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
+    # - Y. Tsuruoka, J. Tsujii, and S. Ananiadou, "Stochastic Gradient Descent Training for L1-regularized Log-linear Models with Cumulative Penalty," Proc. ACL'09, pp. 477--485, 2009.
     # - L. Bottou, "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
-    class Lasso < BaseLinearModel
+    class Lasso < BaseSGD
       include Base::Regressor
 
       # Return the weight vector.
@@ -34,25 +35,43 @@ module Rumale
 
       # Create a new Lasso regressor.
       #
+      # @param learning_rate [Float] The initial value of learning rate.
+      #   The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
+      # @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
+      #   If nil is given, the decay sets to 'reg_param * learning_rate'.
+      # @param momentum [Float] The momentum factor.
       # @param reg_param [Float] The regularization parameter.
       # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
       # @param bias_scale [Float] The scale of the bias term.
-      # @param max_iter [Integer] The maximum number of iterations.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
       # @param batch_size [Integer] The size of the mini batches.
-      # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
-      #   If nil is given, Nadam is used.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
       # @param n_jobs [Integer] The number of jobs for running the fit method in parallel.
       #   If nil is given, the method does not execute in parallel.
       #   If zero or less is given, it becomes equal to the number of processors.
       #   This parameter is ignored if the Parallel gem is not loaded.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0, max_iter: 1000, batch_size: 10, optimizer: nil,
-                     n_jobs: nil, random_seed: nil)
-        check_params_numeric(reg_param: reg_param, bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
-        check_params_boolean(fit_bias: fit_bias)
-        check_params_numeric_or_nil(n_jobs: n_jobs, random_seed: random_seed)
-        check_params_positive(reg_param: reg_param, max_iter: max_iter, batch_size: batch_size)
-        super
+      def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
+                     reg_param: 1.0, fit_bias: true, bias_scale: 1.0,
+                     max_iter: 200, batch_size: 50, tol: 1e-4,
+                     n_jobs: nil, verbose: false, random_seed: nil)
+        check_params_numeric(learning_rate: learning_rate, momentum: momentum,
+                             reg_param: reg_param, bias_scale: bias_scale,
+                             max_iter: max_iter, batch_size: batch_size, tol: tol)
+        check_params_boolean(fit_bias: fit_bias, verbose: verbose)
+        check_params_numeric_or_nil(decay: decay, n_jobs: n_jobs, random_seed: random_seed)
+        check_params_positive(learning_rate: learning_rate, reg_param: reg_param, max_iter: max_iter, batch_size: batch_size)
+        super()
+        @params.merge!(method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h)
+        @params[:decay] ||= @params[:reg_param] * @params[:learning_rate]
+        @params[:random_seed] ||= srand
+        @rng = Random.new(@params[:random_seed])
+        @penalty_type = L1_PENALTY
+        @loss_func = LinearModel::Loss::MeanSquaredError.new
+        @weight_vec = nil
+        @bias_term = nil
       end
 
       # Fit the model with given training data.
@@ -91,54 +110,6 @@ module Rumale
         x = check_convert_sample_array(x)
         x.dot(@weight_vec.transpose) + @bias_term
       end
-
-      # Dump marshal data.
-      # @return [Hash] The marshal data about Lasso.
-      def marshal_dump
-        { params: @params,
-          weight_vec: @weight_vec,
-          bias_term: @bias_term,
-          rng: @rng }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @weight_vec = obj[:weight_vec]
-        @bias_term = obj[:bias_term]
-        @rng = obj[:rng]
-        nil
-      end
-
-      private
-
-      def partial_fit(x, y)
-        n_features = @params[:fit_bias] ? x.shape[1] + 1 : x.shape[1]
-        @left_weight = Numo::DFloat.zeros(n_features)
-        @right_weight = Numo::DFloat.zeros(n_features)
-        @left_optimizer = @params[:optimizer].dup
-        @right_optimizer = @params[:optimizer].dup
-        super
-      end
-
-      def calc_loss_gradient(x, y, weight)
-        2.0 * (x.dot(weight) - y)
-      end
-
-      def calc_new_weight(_optimizer, x, _weight, loss_gradient)
-        @left_weight = round_weight(@left_optimizer.call(@left_weight, calc_weight_gradient(loss_gradient, x)))
-        @right_weight = round_weight(@right_optimizer.call(@right_weight, calc_weight_gradient(-loss_gradient, x)))
-        @left_weight - @right_weight
-      end
-
-      def calc_weight_gradient(loss_gradient, data)
-        ((@params[:reg_param] + loss_gradient).expand_dims(1) * data).mean(0)
-      end
-
-      def round_weight(weight)
-        0.5 * (weight + weight.abs)
-      end
     end
   end
 end

data/lib/rumale/linear_model/linear_regression.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
-require 'rumale/linear_model/base_linear_model'
+require 'rumale/linear_model/base_sgd'
 require 'rumale/base/regressor'
 
 module Rumale
   module LinearModel
     # LinearRegression is a class that implements ordinary least square linear regression
-    # with mini-batch stochastic gradient descent optimization or singular value decomposition.
+    # with stochastic gradient descent (SGD) optimization or singular value decomposition (SVD).
     #
     # @example
     #   estimator =
-    #     Rumale::LinearModel::LinearRegression.new(max_iter: 1000, batch_size: 20, random_seed: 1)
+    #     Rumale::LinearModel::LinearRegression.new(max_iter: 500, batch_size: 20, random_seed: 1)
     #   estimator.fit(training_samples, traininig_values)
     #   results = estimator.predict(testing_samples)
     #
@@ -19,7 +19,10 @@ module Rumale
     #   estimator = Rumale::LinearModel::LinearRegression.new(solver: 'svd')
     #   estimator.fit(training_samples, traininig_values)
     #   results = estimator.predict(testing_samples)
-    class LinearRegression < BaseLinearModel
+    #
+    # *Reference*
+    # - L. Bottou, "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
+    class LinearRegression < BaseSGD
       include Base::Regressor
 
       # Return the weight vector.
@@ -36,34 +39,57 @@ module Rumale
 
       # Create a new ordinary least square linear regressor.
       #
+      # @param learning_rate [Float] The initial value of learning rate.
+      #   The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
+      #   If solver = 'svd', this parameter is ignored.
+      # @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
+      #   If nil is given, the decay sets to 'learning_rate'.
+      #   If solver = 'svd', this parameter is ignored.
+      # @param momentum [Float] The momentum factor.
+      #   If solver = 'svd', this parameter is ignored.
       # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
       # @param bias_scale [Float] The scale of the bias term.
-      # @param max_iter [Integer] The maximum number of iterations.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
       #   If solver = 'svd', this parameter is ignored.
       # @param batch_size [Integer] The size of the mini batches.
       #   If solver = 'svd', this parameter is ignored.
-      # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
-      #   If nil is given, Nadam is used.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
       #   If solver = 'svd', this parameter is ignored.
-      # @param solver [String] The algorithm to calculate weights. ('sgd' or 'svd').
+      # @param solver [String] The algorithm to calculate weights. ('auto', 'sgd' or 'svd').
+      #   'auto' chooses the 'svd' solver if Numo::Linalg is loaded. Otherwise, it chooses the 'sgd' solver.
       #   'sgd' uses the stochastic gradient descent optimization.
       #   'svd' performs singular value decomposition of samples.
       # @param n_jobs [Integer] The number of jobs for running the fit method in parallel.
       #   If nil is given, the method does not execute in parallel.
       #   If zero or less is given, it becomes equal to the number of processors.
       #   This parameter is ignored if the Parallel gem is not loaded.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
+      #   If solver = 'svd', this parameter is ignored.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(fit_bias: false, bias_scale: 1.0, max_iter: 1000, batch_size: 10, optimizer: nil,
-                     solver: 'sgd', n_jobs: nil, random_seed: nil)
-        check_params_numeric(bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
-        check_params_boolean(fit_bias: fit_bias)
+      def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
+                     fit_bias: true, bias_scale: 1.0, max_iter: 200, batch_size: 50, tol: 1e-4,
+                     solver: 'auto',
+                     n_jobs: nil, verbose: false, random_seed: nil)
+        check_params_numeric(learning_rate: learning_rate, momentum: momentum,
+                             bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
+        check_params_boolean(fit_bias: fit_bias, verbose: verbose)
         check_params_string(solver: solver)
-        check_params_numeric_or_nil(n_jobs: n_jobs, random_seed: random_seed)
-        check_params_positive(max_iter: max_iter, batch_size: batch_size)
-        keywd_args = method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h.merge(reg_param: 0.0)
-        keywd_args.delete(:solver)
-        super(**keywd_args)
-        @params[:solver] = solver != 'svd' ? 'sgd' : 'svd'
+        check_params_numeric_or_nil(decay: decay, n_jobs: n_jobs, random_seed: random_seed)
+        check_params_positive(learning_rate: learning_rate, max_iter: max_iter, batch_size: batch_size)
+        super()
+        @params.merge!(method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h)
+        @params[:solver] = if solver == 'auto'
+                             load_linalg? ? 'svd' : 'sgd'
+                           else
+                             solver != 'svd' ? 'sgd' : 'svd'
+                           end
+        @params[:decay] ||= @params[:learning_rate]
+        @params[:random_seed] ||= srand
+        @rng = Random.new(@params[:random_seed])
+        @loss_func = LinearModel::Loss::MeanSquaredError.new
+        @weight_vec = nil
+        @bias_term = nil
       end
 
       # Fit the model with given training data.
@@ -94,33 +120,12 @@ module Rumale
         x.dot(@weight_vec.transpose) + @bias_term
       end
 
-      # Dump marshal data.
-      # @return [Hash] The marshal data about LinearRegression.
-      def marshal_dump
-        { params: @params,
-          weight_vec: @weight_vec,
-          bias_term: @bias_term,
-          rng: @rng }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @weight_vec = obj[:weight_vec]
-        @bias_term = obj[:bias_term]
-        @rng = obj[:rng]
-        nil
-      end
-
       private
 
       def fit_svd(x, y)
-        samples = @params[:fit_bias] ? expand_feature(x) : x
+        x = expand_feature(x) if fit_bias?
 
-        s, u, vt = Numo::Linalg.svd(samples, driver: 'sdd', job: 'S')
-        d = (s / s**2).diag
-        w = vt.transpose.dot(d).dot(u.transpose).dot(y)
+        w = Numo::Linalg.pinv(x, driver: 'svd').dot(y)
 
         is_single_target_vals = y.shape[1].nil?
         if @params[:fit_bias]
@@ -150,8 +155,14 @@ module Rumale
         end
       end
 
-      def calc_loss_gradient(x, y, weight)
-        2.0 * (x.dot(weight) - y)
+      def fit_bias?
+        @params[:fit_bias] == true
+      end
+
+      def load_linalg?
+        return false if defined?(Numo::Linalg).nil?
+        return false if Numo::Linalg::VERSION < '0.1.4'
+        true
       end
     end
   end

data/lib/rumale/linear_model/logistic_regression.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-require 'rumale/linear_model/base_linear_model'
+require 'rumale/linear_model/base_sgd'
 require 'rumale/base/classifier'
 
 module Rumale
   module LinearModel
     # LogisticRegression is a class that implements Logistic Regression
-    # with mini-batch stochastic gradient descent optimization.
+    # with stochastic gradient descent optimization.
     # For multiclass classification problem, it uses one-vs-the-rest strategy.
     #
     # Rumale::SVM provides Logistic Regression based on LIBLINEAR.
@@ -15,13 +15,15 @@ module Rumale
     #
     # @example
     #   estimator =
-    #     Rumale::LinearModel::LogisticRegression.new(reg_param: 1.0, max_iter: 1000, batch_size: 20, random_seed: 1)
+    #     Rumale::LinearModel::LogisticRegression.new(reg_param: 1.0, max_iter: 200, batch_size: 50, random_seed: 1)
     #   estimator.fit(training_samples, traininig_labels)
     #   results = estimator.predict(testing_samples)
     #
     # *Reference*
     # - S. Shalev-Shwartz, Y. Singer, N. Srebro, and A. Cotter, "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Mathematical Programming, vol. 127 (1), pp. 3--30, 2011.
-    class LogisticRegression < BaseLinearModel
+    # - Y. Tsuruoka, J. Tsujii, and S. Ananiadou, "Stochastic Gradient Descent Training for L1-regularized Log-linear Models with Cumulative Penalty," Proc. ACL'09, pp. 477--485, 2009.
+    # - L. Bottou, "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
+    class LogisticRegression < BaseSGD
       include Base::Classifier
 
       # Return the weight vector for Logistic Regression.
@@ -42,26 +44,53 @@ module Rumale
 
       # Create a new classifier with Logisitc Regression by the SGD optimization.
       #
+      # @param learning_rate [Float] The initial value of learning rate.
+      #   The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
+      # @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
+      #   If nil is given, the decay sets to 'reg_param * learning_rate'.
+      # @param momentum [Float] The momentum factor.
+      # @param penalty [String] The regularization type to be used ('l1', 'l2', and 'elasticnet').
+      # @param l1_ratio [Float] The elastic-net type regularization mixing parameter.
+      #   If penalty set to 'l2' or 'l1', this parameter is ignored.
+      #   If l1_ratio = 1, the regularization is similar to Lasso.
+      #   If l1_ratio = 0, the regularization is similar to Ridge.
+      #   If 0 < l1_ratio < 1, the regularization is a combination of L1 and L2.
       # @param reg_param [Float] The regularization parameter.
       # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
       # @param bias_scale [Float] The scale of the bias term.
       #   If fit_bias is true, the feature vector v becoms [v; bias_scale].
-      # @param max_iter [Integer] The maximum number of iterations.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
       # @param batch_size [Integer] The size of the mini batches.
-      # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
-      #   If nil is given, Nadam is used.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
       # @param n_jobs [Integer] The number of jobs for running the fit and predict methods in parallel.
       #   If nil is given, the methods do not execute in parallel.
       #   If zero or less is given, it becomes equal to the number of processors.
       #   This parameter is ignored if the Parallel gem is not loaded.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0,
-                     max_iter: 1000, batch_size: 20, optimizer: nil, n_jobs: nil, random_seed: nil)
-        check_params_numeric(reg_param: reg_param, bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
-        check_params_boolean(fit_bias: fit_bias)
-        check_params_numeric_or_nil(n_jobs: n_jobs, random_seed: random_seed)
-        check_params_positive(reg_param: reg_param, bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
-        super
+      def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
+                     penalty: 'l2', reg_param: 1.0, l1_ratio: 0.5,
+                     fit_bias: true, bias_scale: 1.0,
+                     max_iter: 200, batch_size: 50, tol: 1e-4,
+                     n_jobs: nil, verbose: false, random_seed: nil)
+        check_params_numeric(learning_rate: learning_rate, momentum: momentum,
+                             reg_param: reg_param, l1_ratio: l1_ratio, bias_scale: bias_scale,
+                             max_iter: max_iter, batch_size: batch_size, tol: tol)
+        check_params_boolean(fit_bias: fit_bias, verbose: verbose)
+        check_params_string(penalty: penalty)
+        check_params_numeric_or_nil(decay: decay, n_jobs: n_jobs, random_seed: random_seed)
+        check_params_positive(learning_rate: learning_rate, reg_param: reg_param,
+                              bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
+        super()
+        @params.merge!(method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h)
+        @params[:decay] ||= @params[:reg_param] * @params[:learning_rate]
+        @params[:random_seed] ||= srand
+        @rng = Random.new(@params[:random_seed])
+        @penalty_type = @params[:penalty]
+        @loss_func = LinearModel::Loss::LogLoss.new
+        @weight_vec = nil
+        @bias_term = nil
         @classes = nil
       end
 
@@ -148,33 +177,8 @@ module Rumale
         probs
       end
 
-      # Dump marshal data.
-      # @return [Hash] The marshal data about LogisticRegression.
-      def marshal_dump
-        { params: @params,
-          weight_vec: @weight_vec,
-          bias_term: @bias_term,
-          classes: @classes,
-          rng: @rng }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @weight_vec = obj[:weight_vec]
-        @bias_term = obj[:bias_term]
-        @classes = obj[:classes]
-        @rng = obj[:rng]
-        nil
-      end
-
       private
 
-      def calc_loss_gradient(x, y, weight)
-        y / (Numo::NMath.exp(-y * x.dot(weight)) + 1.0) - y
-      end
-
       def multiclass_problem?
         @classes.size > 2
       end

data/lib/rumale/linear_model/ridge.rb

@@ -1,16 +1,16 @@
 # frozen_string_literal: true
 
-require 'rumale/linear_model/base_linear_model'
+require 'rumale/linear_model/base_sgd'
 require 'rumale/base/regressor'
 
 module Rumale
   module LinearModel
     # Ridge is a class that implements Ridge Regression
-    # with mini-batch stochastic gradient descent optimization or singular value decomposition.
+    # with stochastic gradient descent (SGD) optimization or singular value decomposition (SVD).
     #
     # @example
     #   estimator =
-    #     Rumale::LinearModel::Ridge.new(reg_param: 0.1, max_iter: 1000, batch_size: 20, random_seed: 1)
+    #     Rumale::LinearModel::Ridge.new(reg_param: 0.1, max_iter: 500, batch_size: 20, random_seed: 1)
     #   estimator.fit(training_samples, traininig_values)
     #   results = estimator.predict(testing_samples)
     #
@@ -19,7 +19,10 @@ module Rumale
     #   estimator = Rumale::LinearModel::Ridge.new(reg_param: 0.1, solver: 'svd')
     #   estimator.fit(training_samples, traininig_values)
     #   results = estimator.predict(testing_samples)
-    class Ridge < BaseLinearModel
+    #
+    # *Reference*
+    # - L. Bottou, "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
+    class Ridge < BaseSGD
       include Base::Regressor
 
       # Return the weight vector.
@@ -36,35 +39,61 @@ module Rumale
 
       # Create a new Ridge regressor.
       #
+      # @param learning_rate [Float] The initial value of learning rate.
+      #   The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
+      #   If solver = 'svd', this parameter is ignored.
+      # @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
+      #   If nil is given, the decay sets to 'reg_param * learning_rate'.
+      #   If solver = 'svd', this parameter is ignored.
+      # @param momentum [Float] The momentum factor.
+      #   If solver = 'svd', this parameter is ignored.
       # @param reg_param [Float] The regularization parameter.
       # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
       # @param bias_scale [Float] The scale of the bias term.
-      # @param max_iter [Integer] The maximum number of iterations.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
       #   If solver = 'svd', this parameter is ignored.
       # @param batch_size [Integer] The size of the mini batches.
       #   If solver = 'svd', this parameter is ignored.
-      # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
-      #   If nil is given, Nadam is used.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
       #   If solver = 'svd', this parameter is ignored.
-      # @param solver [String] The algorithm to calculate weights. ('sgd' or 'svd').
+      # @param solver [String] The algorithm to calculate weights. ('auto', 'sgd' or 'svd').
+      #   'auto' chooses the 'svd' solver if Numo::Linalg is loaded. Otherwise, it chooses the 'sgd' solver.
       #   'sgd' uses the stochastic gradient descent optimization.
       #   'svd' performs singular value decomposition of samples.
       # @param n_jobs [Integer] The number of jobs for running the fit method in parallel.
       #   If nil is given, the method does not execute in parallel.
       #   If zero or less is given, it becomes equal to the number of processors.
       #   This parameter is ignored if the Parallel gem is not loaded or the solver is 'svd'.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
+      #   If solver = 'svd', this parameter is ignored.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0, max_iter: 1000, batch_size: 10, optimizer: nil,
-                     solver: 'sgd', n_jobs: nil, random_seed: nil)
-        check_params_numeric(reg_param: reg_param, bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
-        check_params_boolean(fit_bias: fit_bias)
+      def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
+                     reg_param: 1.0, fit_bias: true, bias_scale: 1.0,
+                     max_iter: 200, batch_size: 50, tol: 1e-4,
+                     solver: 'auto',
+                     n_jobs: nil, verbose: false, random_seed: nil)
+        check_params_numeric(learning_rate: learning_rate, momentum: momentum,
+                             reg_param: reg_param, bias_scale: bias_scale,
+                             max_iter: max_iter, batch_size: batch_size, tol: tol)
+        check_params_boolean(fit_bias: fit_bias, verbose: verbose)
         check_params_string(solver: solver)
-        check_params_numeric_or_nil(n_jobs: n_jobs, random_seed: random_seed)
-        check_params_positive(reg_param: reg_param, max_iter: max_iter, batch_size: batch_size)
-        keywd_args = method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h
-        keywd_args.delete(:solver)
-        super(**keywd_args)
-        @params[:solver] = solver != 'svd' ? 'sgd' : 'svd'
+        check_params_numeric_or_nil(decay: decay, n_jobs: n_jobs, random_seed: random_seed)
+        check_params_positive(learning_rate: learning_rate, reg_param: reg_param, max_iter: max_iter, batch_size: batch_size)
+        super()
+        @params.merge!(method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h)
+        @params[:solver] = if solver == 'auto'
+                             load_linalg? ? 'svd' : 'sgd'
+                           else
+                             solver != 'svd' ? 'sgd' : 'svd'
+                           end
+        @params[:decay] ||= @params[:reg_param] * @params[:learning_rate]
+        @params[:random_seed] ||= srand
+        @rng = Random.new(@params[:random_seed])
+        @penalty_type = L2_PENALTY
+        @loss_func = LinearModel::Loss::MeanSquaredError.new
+        @weight_vec = nil
+        @bias_term = nil
       end
 
       # Fit the model with given training data.
@@ -95,25 +124,6 @@ module Rumale
         x.dot(@weight_vec.transpose) + @bias_term
       end
 
-      # Dump marshal data.
-      # @return [Hash] The marshal data about Ridge.
-      def marshal_dump
-        { params: @params,
-          weight_vec: @weight_vec,
-          bias_term: @bias_term,
-          rng: @rng }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @weight_vec = obj[:weight_vec]
-        @bias_term = obj[:bias_term]
-        @rng = obj[:rng]
-        nil
-      end
-
       private
 
       def fit_svd(x, y)
@@ -151,8 +161,10 @@ module Rumale
         end
       end
 
-      def calc_loss_gradient(x, y, weight)
-        2.0 * (x.dot(weight) - y)
+      def load_linalg?
+        return false if defined?(Numo::Linalg).nil?
+        return false if Numo::Linalg::VERSION < '0.1.4'
+        true
       end
     end
   end

data/lib/rumale/linear_model/svc.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'rumale/linear_model/base_linear_model'
+require 'rumale/linear_model/base_sgd'
 require 'rumale/base/classifier'
 require 'rumale/probabilistic_output'
 
@@ -8,7 +8,7 @@ module Rumale
   # This module consists of the classes that implement generalized linear models.
   module LinearModel
     # SVC is a class that implements Support Vector Classifier
-    # with mini-batch stochastic gradient descent optimization.
+    # with stochastic gradient descent optimization.
     # For multiclass classification problem, it uses one-vs-the-rest strategy.
     #
     # Rumale::SVM provides linear support vector classifier based on LIBLINEAR.
@@ -17,13 +17,15 @@ module Rumale
     #
     # @example
     #   estimator =
-    #     Rumale::LinearModel::SVC.new(reg_param: 1.0, max_iter: 1000, batch_size: 20, random_seed: 1)
+    #     Rumale::LinearModel::SVC.new(reg_param: 1.0, max_iter: 200, batch_size: 50, random_seed: 1)
     #   estimator.fit(training_samples, traininig_labels)
     #   results = estimator.predict(testing_samples)
     #
     # *Reference*
     # - S. Shalev-Shwartz and Y. Singer, "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
-    class SVC < BaseLinearModel
+    # - Y. Tsuruoka, J. Tsujii, and S. Ananiadou, "Stochastic Gradient Descent Training for L1-regularized Log-linear Models with Cumulative Penalty," Proc. ACL'09, pp. 477--485, 2009.
+    # - L. Bottou, "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
+    class SVC < BaseSGD
       include Base::Classifier
 
       # Return the weight vector for SVC.
@@ -44,31 +46,56 @@ module Rumale
 
       # Create a new classifier with Support Vector Machine by the SGD optimization.
       #
+      # @param learning_rate [Float] The initial value of learning rate.
+      #   The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
+      # @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
+      #   If nil is given, the decay sets to 'reg_param * learning_rate'.
+      # @param momentum [Float] The momentum factor.
+      # @param penalty [String] The regularization type to be used ('l1', 'l2', and 'elasticnet').
+      # @param l1_ratio [Float] The elastic-net type regularization mixing parameter.
+      #   If penalty set to 'l2' or 'l1', this parameter is ignored.
+      #   If l1_ratio = 1, the regularization is similar to Lasso.
+      #   If l1_ratio = 0, the regularization is similar to Ridge.
+      #   If 0 < l1_ratio < 1, the regularization is a combination of L1 and L2.
       # @param reg_param [Float] The regularization parameter.
       # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
       # @param bias_scale [Float] The scale of the bias term.
-      # @param max_iter [Integer] The maximum number of iterations.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
       # @param batch_size [Integer] The size of the mini batches.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
       # @param probability [Boolean] The flag indicating whether to perform probability estimation.
-      # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
-      #   If nil is given, Nadam is used.
       # @param n_jobs [Integer] The number of jobs for running the fit and predict methods in parallel.
       #   If nil is given, the methods do not execute in parallel.
       #   If zero or less is given, it becomes equal to the number of processors.
       #   This parameter is ignored if the Parallel gem is not loaded.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0,
-                     max_iter: 1000, batch_size: 20, probability: false, optimizer: nil, n_jobs: nil, random_seed: nil)
-        check_params_numeric(reg_param: reg_param, bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
-        check_params_boolean(fit_bias: fit_bias, probability: probability)
-        check_params_numeric_or_nil(n_jobs: n_jobs, random_seed: random_seed)
-        check_params_positive(reg_param: reg_param, bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
-        keywd_args = method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h
-        keywd_args.delete(:probability)
-        super(**keywd_args)
-        @params[:probability] = probability
-        @prob_param = nil
+      def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
+                     penalty: 'l2', reg_param: 1.0, l1_ratio: 0.5,
+                     fit_bias: true, bias_scale: 1.0,
+                     max_iter: 200, batch_size: 50, tol: 1e-4,
+                     probability: false,
+                     n_jobs: nil, verbose: false, random_seed: nil)
+        check_params_numeric(learning_rate: learning_rate, momentum: momentum,
+                             reg_param: reg_param, l1_ratio: l1_ratio, bias_scale: bias_scale,
+                             max_iter: max_iter, batch_size: batch_size, tol: tol)
+        check_params_boolean(fit_bias: fit_bias, verbose: verbose, probability: probability)
+        check_params_string(penalty: penalty)
+        check_params_numeric_or_nil(decay: decay, n_jobs: n_jobs, random_seed: random_seed)
+        check_params_positive(learning_rate: learning_rate, reg_param: reg_param,
+                              bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
+        super()
+        @params.merge!(method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h)
+        @params[:decay] ||= @params[:reg_param] * @params[:learning_rate]
+        @params[:random_seed] ||= srand
+        @rng = Random.new(@params[:random_seed])
+        @penalty_type = @params[:penalty]
+        @loss_func = LinearModel::Loss::HingeLoss.new
+        @weight_vec = nil
+        @bias_term = nil
         @classes = nil
+        @prob_param = nil
       end
 
       # Fit the model with given training data.
@@ -165,29 +192,6 @@ module Rumale
         end
       end
 
-      # Dump marshal data.
-      # @return [Hash] The marshal data about SVC.
-      def marshal_dump
-        { params: @params,
-          weight_vec: @weight_vec,
-          bias_term: @bias_term,
-          prob_param: @prob_param,
-          classes: @classes,
-          rng: @rng }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @weight_vec = obj[:weight_vec]
-        @bias_term = obj[:bias_term]
-        @prob_param = obj[:prob_param]
-        @classes = obj[:classes]
-        @rng = obj[:rng]
-        nil
-      end
-
       private
 
       def partial_fit(x, bin_y)
@@ -200,13 +204,6 @@ module Rumale
         [w, b, p]
       end
 
-      def calc_loss_gradient(x, y, weight)
-        target_ids = (x.dot(weight) * y).lt(1.0).where
-        grad = Numo::DFloat.zeros(@params[:batch_size])
-        grad[target_ids] = -y[target_ids]
-        grad
-      end
-
       def multiclass_problem?
         @classes.size > 2
       end

data/lib/rumale/linear_model/svr.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
-require 'rumale/linear_model/base_linear_model'
+require 'rumale/linear_model/base_sgd'
 require 'rumale/base/regressor'
 
 module Rumale
   module LinearModel
     # SVR is a class that implements Support Vector Regressor
-    # with mini-batch stochastic gradient descent optimization.
+    # with stochastic gradient descent optimization.
     #
     # Rumale::SVM provides linear and kernel support vector regressor based on LIBLINEAR and LIBSVM.
     # If you prefer execution speed, you should use Rumale::SVM::LinearSVR.
@@ -14,13 +14,15 @@ module Rumale
     #
     # @example
     #   estimator =
-    #     Rumale::LinearModel::SVR.new(reg_param: 1.0, epsilon: 0.1, max_iter: 1000, batch_size: 20, random_seed: 1)
+    #     Rumale::LinearModel::SVR.new(reg_param: 1.0, epsilon: 0.1, max_iter: 200, batch_size: 50, random_seed: 1)
     #   estimator.fit(training_samples, traininig_target_values)
     #   results = estimator.predict(testing_samples)
     #
     # *Reference*
-    # 1. S. Shalev-Shwartz and Y. Singer, "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
-    class SVR < BaseLinearModel
+    # - S. Shalev-Shwartz and Y. Singer, "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
+    # - Y. Tsuruoka, J. Tsujii, and S. Ananiadou, "Stochastic Gradient Descent Training for L1-regularized Log-linear Models with Cumulative Penalty," Proc. ACL'09, pp. 477--485, 2009.
+    # - L. Bottou, "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
+    class SVR < BaseSGD
       include Base::Regressor
 
       # Return the weight vector for SVR.
@@ -37,30 +39,54 @@ module Rumale
 
       # Create a new regressor with Support Vector Machine by the SGD optimization.
       #
+      # @param learning_rate [Float] The initial value of learning rate.
+      #   The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
+      # @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
+      #   If nil is given, the decay sets to 'reg_param * learning_rate'.
+      # @param momentum [Float] The momentum factor.
+      # @param penalty [String] The regularization type to be used ('l1', 'l2', and 'elasticnet').
+      # @param l1_ratio [Float] The elastic-net type regularization mixing parameter.
+      #   If penalty set to 'l2' or 'l1', this parameter is ignored.
+      #   If l1_ratio = 1, the regularization is similar to Lasso.
+      #   If l1_ratio = 0, the regularization is similar to Ridge.
+      #   If 0 < l1_ratio < 1, the regularization is a combination of L1 and L2.
       # @param reg_param [Float] The regularization parameter.
       # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
       # @param bias_scale [Float] The scale of the bias term.
       # @param epsilon [Float] The margin of tolerance.
-      # @param max_iter [Integer] The maximum number of iterations.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
       # @param batch_size [Integer] The size of the mini batches.
-      # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
-      #   If nil is given, Nadam is used.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
       # @param n_jobs [Integer] The number of jobs for running the fit method in parallel.
       #   If nil is given, the method does not execute in parallel.
       #   If zero or less is given, it becomes equal to the number of processors.
       #   This parameter is ignored if the Parallel gem is not loaded.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0, epsilon: 0.1,
-                     max_iter: 1000, batch_size: 20, optimizer: nil, n_jobs: nil, random_seed: nil)
-        check_params_numeric(reg_param: reg_param, bias_scale: bias_scale, epsilon: epsilon, max_iter: max_iter, batch_size: batch_size)
-        check_params_boolean(fit_bias: fit_bias)
-        check_params_numeric_or_nil(n_jobs: n_jobs, random_seed: random_seed)
-        check_params_positive(reg_param: reg_param, bias_scale: bias_scale, epsilon: epsilon,
+      def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
+                     penalty: 'l2', reg_param: 1.0, l1_ratio: 0.5,
+                     fit_bias: true, bias_scale: 1.0,
+                     epsilon: 0.1,
+                     max_iter: 200, batch_size: 50, tol: 1e-4,
+                     n_jobs: nil, verbose: false, random_seed: nil)
+        check_params_numeric(learning_rate: learning_rate, momentum: momentum,
+                             reg_param: reg_param, bias_scale: bias_scale, epsilon: epsilon,
+                             max_iter: max_iter, batch_size: batch_size, tol: tol)
+        check_params_boolean(fit_bias: fit_bias, verbose: verbose)
+        check_params_numeric_or_nil(decay: decay, n_jobs: n_jobs, random_seed: random_seed)
+        check_params_positive(learning_rate: learning_rate, reg_param: reg_param,
+                              bias_scale: bias_scale, epsilon: epsilon,
                               max_iter: max_iter, batch_size: batch_size)
-        keywd_args = method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h
-        keywd_args.delete(:epsilon)
-        super(**keywd_args)
-        @params[:epsilon] = epsilon
+        super()
+        @params.merge!(method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h)
+        @params[:decay] ||= @params[:reg_param] * @params[:learning_rate]
+        @params[:random_seed] ||= srand
+        @rng = Random.new(@params[:random_seed])
+        @penalty_type = @params[:penalty]
+        @loss_func = LinearModel::Loss::EpsilonInsensitive.new(epsilon: @params[:epsilon])
+        @weight_vec = nil
+        @bias_term = nil
       end
 
       # Fit the model with given training data.
@@ -100,35 +126,6 @@ module Rumale
         x = check_convert_sample_array(x)
         x.dot(@weight_vec.transpose) + @bias_term
       end
-
-      # Dump marshal data.
-      # @return [Hash] The marshal data about SVR.
-      def marshal_dump
-        { params: @params,
-          weight_vec: @weight_vec,
-          bias_term: @bias_term,
-          rng: @rng }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @weight_vec = obj[:weight_vec]
-        @bias_term = obj[:bias_term]
-        @rng = obj[:rng]
-        nil
-      end
-
-      private
-
-      def calc_loss_gradient(x, y, weight)
-        z = x.dot(weight)
-        grad = Numo::DFloat.zeros(@params[:batch_size])
-        grad[(z - y).gt(@params[:epsilon]).where] = 1
-        grad[(y - z).gt(@params[:epsilon]).where] = -1
-        grad
-      end
     end
   end
 end

data/lib/rumale/polynomial_model/base_factorization_machine.rb

@@ -17,7 +17,8 @@ module Rumale
       # @param loss [String] The loss function ('hinge' or 'logistic' or nil).
       # @param reg_param_linear [Float] The regularization parameter for linear model.
       # @param reg_param_factor [Float] The regularization parameter for factor matrix.
-      # @param max_iter [Integer] The maximum number of iterations.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
       # @param batch_size [Integer] The size of the mini batches.
       # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
       #   If nil is given, Nadam is used.
@@ -27,7 +28,7 @@ module Rumale
       #   This parameter is ignored if the Parallel gem is not loaded.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(n_factors: 2, loss: nil, reg_param_linear: 1.0, reg_param_factor: 1.0,
-                     max_iter: 1000, batch_size: 10, optimizer: nil, n_jobs: nil, random_seed: nil)
+                     max_iter: 200, batch_size: 50, optimizer: nil, n_jobs: nil, random_seed: nil)
         @params = {}
         @params[:n_factors] = n_factors
         @params[:loss] = loss unless loss.nil?
@@ -51,27 +52,29 @@ module Rumale
       def partial_fit(x, y)
         # Initialize some variables.
         n_samples, n_features = x.shape
-        rand_ids = [*0...n_samples].shuffle(random: @rng.dup)
+        sub_rng = @rng.dup
         weight_vec = Numo::DFloat.zeros(n_features + 1)
         factor_mat = Numo::DFloat.zeros(@params[:n_factors], n_features)
         weight_optimizer = @params[:optimizer].dup
         factor_optimizers = Array.new(@params[:n_factors]) { @params[:optimizer].dup }
         # Start optimization.
         @params[:max_iter].times do |_t|
-          # Random sampling.
-          subset_ids = rand_ids.shift(@params[:batch_size])
-          rand_ids.concat(subset_ids)
-          data = x[subset_ids, true]
-          ex_data = expand_feature(data)
-          targets = y[subset_ids]
-          # Calculate gradients for loss function.
-          loss_grad = loss_gradient(data, ex_data, targets, factor_mat, weight_vec)
-          next if loss_grad.ne(0.0).count.zero?
-          # Update each parameter.
-          weight_vec = weight_optimizer.call(weight_vec, weight_gradient(loss_grad, ex_data, weight_vec))
-          @params[:n_factors].times do |n|
-            factor_mat[n, true] = factor_optimizers[n].call(factor_mat[n, true],
-                                                            factor_gradient(loss_grad, data, factor_mat[n, true]))
+          sample_ids = [*0...n_samples]
+          sample_ids.shuffle!(random: sub_rng)
+          until (subset_ids = sample_ids.shift(@params[:batch_size])).empty?
+            # Sampling.
+            sub_x = x[subset_ids, true]
+            sub_y = y[subset_ids]
+            ex_sub_x = expand_feature(sub_x)
+            # Calculate gradients for loss function.
+            loss_grad = loss_gradient(sub_x, ex_sub_x, sub_y, factor_mat, weight_vec)
+            next if loss_grad.ne(0.0).count.zero?
+            # Update each parameter.
+            weight_vec = weight_optimizer.call(weight_vec, weight_gradient(loss_grad, ex_sub_x, weight_vec))
+            @params[:n_factors].times do |n|
+              factor_mat[n, true] = factor_optimizers[n].call(factor_mat[n, true],
+                                                              factor_gradient(loss_grad, sub_x, factor_mat[n, true]))
+            end
           end
         end
         [factor_mat, *split_weight_vec_bias(weight_vec)]

data/lib/rumale/polynomial_model/factorization_machine_classifier.rb

@@ -14,7 +14,7 @@ module Rumale
     #   estimator =
     #     Rumale::PolynomialModel::FactorizationMachineClassifier.new(
     #      n_factors: 10, loss: 'hinge', reg_param_linear: 0.001, reg_param_factor: 0.001,
-    #      max_iter: 5000, batch_size: 50, random_seed: 1)
+    #      max_iter: 500, batch_size: 50, random_seed: 1)
     #   estimator.fit(training_samples, traininig_labels)
     #   results = estimator.predict(testing_samples)
     #
@@ -50,7 +50,8 @@ module Rumale
       # @param loss [String] The loss function ('hinge' or 'logistic').
       # @param reg_param_linear [Float] The regularization parameter for linear model.
       # @param reg_param_factor [Float] The regularization parameter for factor matrix.
-      # @param max_iter [Integer] The maximum number of iterations.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
       # @param batch_size [Integer] The size of the mini batches.
       # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
       #   If nil is given, Nadam is used.
@@ -60,7 +61,7 @@ module Rumale
       #   This parameter is ignored if the Parallel gem is not loaded.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(n_factors: 2, loss: 'hinge', reg_param_linear: 1.0, reg_param_factor: 1.0,
-                     max_iter: 1000, batch_size: 10, optimizer: nil, n_jobs: nil, random_seed: nil)
+                     max_iter: 200, batch_size: 50, optimizer: nil, n_jobs: nil, random_seed: nil)
         check_params_numeric(reg_param_linear: reg_param_linear, reg_param_factor: reg_param_factor,
                              n_factors: n_factors, max_iter: max_iter, batch_size: batch_size)
         check_params_string(loss: loss)

data/lib/rumale/polynomial_model/factorization_machine_regressor.rb

@@ -12,7 +12,7 @@ module Rumale
     #   estimator =
     #     Rumale::PolynomialModel::FactorizationMachineRegressor.new(
     #      n_factors: 10, reg_param_linear: 0.1, reg_param_factor: 0.1,
-    #      max_iter: 5000, batch_size: 50, random_seed: 1)
+    #      max_iter: 500, batch_size: 50, random_seed: 1)
     #   estimator.fit(training_samples, traininig_values)
     #   results = estimator.predict(testing_samples)
     #
@@ -43,7 +43,8 @@ module Rumale
       # @param n_factors [Integer] The maximum number of iterations.
       # @param reg_param_linear [Float] The regularization parameter for linear model.
       # @param reg_param_factor [Float] The regularization parameter for factor matrix.
-      # @param max_iter [Integer] The maximum number of iterations.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
       # @param batch_size [Integer] The size of the mini batches.
       # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
       #   If nil is given, Nadam is used.
@@ -53,7 +54,7 @@ module Rumale
       #   This parameter is ignored if the Parallel gem is not loaded.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(n_factors: 2, reg_param_linear: 1.0, reg_param_factor: 1.0,
-                     max_iter: 1000, batch_size: 10, optimizer: nil, n_jobs: nil, random_seed: nil)
+                     max_iter: 200, batch_size: 50, optimizer: nil, n_jobs: nil, random_seed: nil)
         check_params_numeric(reg_param_linear: reg_param_linear, reg_param_factor: reg_param_factor,
                              n_factors: n_factors, max_iter: max_iter, batch_size: batch_size)
         check_params_numeric_or_nil(n_jobs: n_jobs, random_seed: random_seed)

data/lib/rumale/version.rb

@@ -3,5 +3,5 @@
 # Rumale is a machine learning library in Ruby.
 module Rumale
   # The version of Rumale you are using.
-  VERSION = '0.16.1'
+  VERSION = '0.17.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rumale
 version: !ruby/object:Gem::Version
-  version: 0.16.1
+  version: 0.17.0
 platform: ruby
 authors:
 - yoshoku
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-11 00:00:00.000000000 Z
+date: 2020-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray