rumale 0.23.3 → 0.24.0

data/lib/rumale/linear_model/base_sgd.rb

@@ -1,285 +0,0 @@
-# frozen_string_literal: true
-
-require 'rumale/base/base_estimator'
-
-module Rumale
-  module LinearModel
-    # @!visibility private
-    # This module consists of the classes that implement penalty (regularization) term.
-    module Penalty
-      # @!visibility private
-      # L2Penalty is a class that applies L2 penalty to weight vector of linear model.
-      # This class is used internally.
-      class L2Penalty
-        # @!visibility private
-        def initialize(reg_param:)
-          @reg_param = reg_param
-        end
-
-        # @!visibility private
-        def call(weight, lr)
-          weight - @reg_param * lr * weight
-        end
-      end
-
-      # @!visibility private
-      # L1Penalty is a class that applies L1 penalty to weight vector of linear model.
-      # This class is used internally.
-      class L1Penalty
-        # @!visibility private
-        def initialize(reg_param:)
-          @q_vec = nil
-          @u = 0
-          @reg_param = reg_param
-        end
-
-        # @!visibility private
-        def call(weight, lr)
-          @q_vec ||= Numo::DFloat.zeros(weight.shape[0])
-          @u += @reg_param * lr
-          z = weight.dup
-          gt = weight.gt(0)
-          lt = weight.lt(0)
-          weight[gt] = Numo::DFloat.maximum(0.0, weight[gt] - (@u + @q_vec[gt])) if gt.count.positive?
-          weight[lt] = Numo::DFloat.minimum(0.0, weight[lt] + (@u - @q_vec[lt])) if lt.count.positive?
-          @q_vec += weight - z
-          weight
-        end
-      end
-    end
-
-    # @!visibility private
-    # This module consists of the class that implements stochastic gradient descent (SGD) optimizer.
-    module Optimizer
-      # @!visibility private
-      # SGD is a class that implements SGD optimizer.
-      # This class is used internally.
-      class SGD
-        # @!visibility private
-        # Create a new SGD optimizer.
-        # @param learning_rate [Float] The initial value of learning rate.
-        # @param momentum [Float] The initial value of momentum.
-        # @param decay [Float] The smooting parameter.
-        def initialize(learning_rate: 0.01, momentum: 0.0, decay: 0.0)
-          @learning_rate = learning_rate
-          @momentum = momentum
-          @decay = decay
-          @update = nil
-          @iter = 0
-        end
-
-        # @!visibility private
-        def current_learning_rate
-          @learning_rate / (1.0 + @decay * @iter)
-        end
-
-        # @!visibility private
-        def call(weight, gradient)
-          @update ||= Numo::DFloat.zeros(weight.shape[0])
-          @update = @momentum * @update - current_learning_rate * gradient
-          @iter += 1
-          weight + @update
-        end
-      end
-    end
-
-    # @!visibility private
-    # This module consists of the classes that implement loss function for linear model.
-    module Loss
-      # @!visibility private
-      # MeanSquaredError is a class that calculates mean squared error for linear regression model.
-      class MeanSquaredError
-        # @!visibility private
-        def loss(out, y)
-          ((out - y)**2).sum.fdiv(y.shape[0])
-        end
-
-        # @!visibility private
-        def dloss(out, y)
-          2.fdiv(y.shape[0]) * (out - y)
-        end
-      end
-
-      # @!visibility private
-      # LogLoss is a class that calculates logistic loss for logistic regression.
-      class LogLoss
-        # @!visibility private
-        def loss(out, y)
-          Numo::NMath.log(1 + Numo::NMath.exp(-y * out)).sum.fdiv(y.shape[0])
-        end
-
-        # @!visibility private
-        def dloss(out, y)
-          y / (1 + Numo::NMath.exp(-y * out)) - y
-        end
-      end
-
-      # @!visibility private
-      # HingeLoss is a class that calculates hinge loss for support vector classifier.
-      class HingeLoss
-        # @!visibility private
-        def loss(out, y)
-          out.class.maximum(0.0, 1 - y * out).sum.fdiv(y.shape[0])
-        end
-
-        # @!visibility private
-        def dloss(out, y)
-          tids = (y * out).lt(1)
-          d = Numo::DFloat.zeros(y.shape[0])
-          d[tids] = -y[tids] if tids.count.positive?
-          d
-        end
-      end
-
-      # @!visibility private
-      # EpsilonInsensitive is a class that calculates epsilon insensitive for support vector regressor.
-      class EpsilonInsensitive
-        # @!visibility private
-        def initialize(epsilon: 0.1)
-          @epsilon = epsilon
-        end
-
-        # @!visibility private
-        def loss(out, y)
-          out.class.maximum(0.0, (y - out).abs - @epsilon).sum.fdiv(y.shape[0])
-        end
-
-        # @!visibility private
-        def dloss(out, y)
-          d = Numo::DFloat.zeros(y.shape[0])
-          tids = (out - y).gt(@epsilon)
-          d[tids] = 1 if tids.count.positive?
-          tids = (y - out).gt(@epsilon)
-          d[tids] = -1 if tids.count.positive?
-          d
-        end
-      end
-    end
-
-    # BaseSGD is an abstract class for implementation of linear model with mini-batch stochastic gradient descent (SGD) optimization.
-    # This class is used internally.
-    class BaseSGD
-      include Rumale::Base::BaseEstimator
-
-      # Create an initial linear model.
-      def initialize
-        @params = {}
-        @params[:learning_rate] = 0.01
-        @params[:decay] = nil
-        @params[:momentum] = 0.0
-        @params[:bias_scale] = 1.0
-        @params[:fit_bias] = true
-        @params[:reg_param] = 0.0
-        @params[:l1_ratio] = 0.0
-        @params[:max_iter] = 1000
-        @params[:batch_size] = 50
-        @params[:tol] = 0.0001
-        @params[:verbose] = false
-        @penalty_type = nil
-        @loss_func = nil
-        @weight_vec = nil
-        @bias_term = nil
-        @n_iter = nil
-        @rng = nil
-      end
-
-      private
-
-      L2_PENALTY = 'l2'
-      L1_PENALTY = 'l1'
-      ELASTICNET_PENALTY = 'elasticnet'
-
-      private_constant :L2_PENALTY, :L1_PENALTY, :ELASTICNET_PENALTY
-
-      def partial_fit(x, y)
-        class_name = self.class.to_s.split('::').last if @params[:verbose]
-        narr = x.class
-        # Expand feature vectors for bias term.
-        x = expand_feature(x) if fit_bias?
-        # Initialize some variables.
-        sub_rng = @rng.dup
-        n_samples, n_features = x.shape
-        weight = Numo::DFloat.zeros(n_features)
-        optimizer = LinearModel::Optimizer::SGD.new(
-          learning_rate: @params[:learning_rate],
-          momentum: @params[:momentum],
-          decay: @params[:decay]
-        )
-        l2_penalty = LinearModel::Penalty::L2Penalty.new(reg_param: l2_reg_param) if apply_l2_penalty?
-        l1_penalty = LinearModel::Penalty::L1Penalty.new(reg_param: l1_reg_param) if apply_l1_penalty?
-        # Optimization.
-        @params[:max_iter].times do |t|
-          sample_ids = Array(0...n_samples)
-          sample_ids.shuffle!(random: sub_rng)
-          until (subset_ids = sample_ids.shift(@params[:batch_size])).empty?
-            # sampling
-            sub_x = x[subset_ids, true]
-            sub_y = y[subset_ids]
-            # calculate gradient
-            dloss = @loss_func.dloss(sub_x.dot(weight), sub_y)
-            dloss = narr.minimum(1e12, narr.maximum(-1e12, dloss))
-            gradient = dloss.dot(sub_x)
-            # update weight
-            lr = optimizer.current_learning_rate
-            weight = optimizer.call(weight, gradient)
-            # l2 regularization
-            weight = l2_penalty.call(weight, lr) if apply_l2_penalty?
-            # l1 regularization
-            weight = l1_penalty.call(weight, lr) if apply_l1_penalty?
-          end
-          loss = @loss_func.loss(x.dot(weight), y)
-          puts "[#{class_name}] Loss after #{t + 1} epochs: #{loss}" if @params[:verbose]
-          break if loss < @params[:tol]
-        end
-        split_weight(weight)
-      end
-
-      def expand_feature(x)
-        n_samples = x.shape[0]
-        Numo::NArray.hstack([x, Numo::DFloat.ones([n_samples, 1]) * @params[:bias_scale]])
-      end
-
-      def split_weight(weight)
-        if fit_bias?
-          [weight[0...-1].dup, weight[-1]]
-        else
-          [weight, 0.0]
-        end
-      end
-
-      def fit_bias?
-        @params[:fit_bias] == true
-      end
-
-      def apply_l2_penalty?
-        @penalty_type == L2_PENALTY || @penalty_type == ELASTICNET_PENALTY
-      end
-
-      def apply_l1_penalty?
-        @penalty_type == L1_PENALTY || @penalty_type == ELASTICNET_PENALTY
-      end
-
-      def l2_reg_param
-        case @penalty_type
-        when ELASTICNET_PENALTY
-          @params[:reg_param] * (1.0 - @params[:l1_ratio])
-        when L2_PENALTY
-          @params[:reg_param]
-        else
-          0.0
-        end
-      end
-
-      def l1_reg_param
-        case @penalty_type
-        when ELASTICNET_PENALTY
-          @params[:reg_param] * @params[:l1_ratio]
-        when L1_PENALTY
-          @params[:reg_param]
-        else
-          0.0
-        end
-      end
-    end
-  end
-end

data/lib/rumale/linear_model/elastic_net.rb

@@ -1,119 +0,0 @@
-# frozen_string_literal: true
-
-require 'rumale/linear_model/base_sgd'
-require 'rumale/base/regressor'
-
-module Rumale
-  module LinearModel
-    # ElasticNet is a class that implements Elastic-net Regression
-    # with stochastic gradient descent (SGD) optimization.
-    #
-    # @example
-    #   estimator =
-    #     Rumale::LinearModel::ElasticNet.new(reg_param: 0.1, l1_ratio: 0.5, max_iter: 1000, batch_size: 50, random_seed: 1)
-    #   estimator.fit(training_samples, traininig_values)
-    #   results = estimator.predict(testing_samples)
-    #
-    # *Reference*
-    # - Shalev-Shwartz, S., and Singer, Y., "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
-    # - Tsuruoka, Y., Tsujii, J., and Ananiadou, S., "Stochastic Gradient Descent Training for L1-regularized Log-linear Models with Cumulative Penalty," Proc. ACL'09, pp. 477--485, 2009.
-    # - Bottou, L., "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
-    class ElasticNet < BaseSGD
-      include Base::Regressor
-
-      # Return the weight vector.
-      # @return [Numo::DFloat] (shape: [n_outputs, n_features])
-      attr_reader :weight_vec
-
-      # Return the bias term (a.k.a. intercept).
-      # @return [Numo::DFloat] (shape: [n_outputs])
-      attr_reader :bias_term
-
-      # Return the random generator for random sampling.
-      # @return [Random]
-      attr_reader :rng
-
-      # Create a new Elastic-net regressor.
-      #
-      # @param learning_rate [Float] The initial value of learning rate.
-      #   The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
-      # @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
-      #   If nil is given, the decay sets to 'reg_param * learning_rate'.
-      # @param momentum [Float] The momentum factor.
-      # @param reg_param [Float] The regularization parameter.
-      # @param l1_ratio [Float] The elastic-net mixing parameter.
-      #   If l1_ratio = 1, the regularization is similar to Lasso.
-      #   If l1_ratio = 0, the regularization is similar to Ridge.
-      #   If 0 < l1_ratio < 1, the regularization is a combination of L1 and L2.
-      # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
-      # @param bias_scale [Float] The scale of the bias term.
-      # @param max_iter [Integer] The maximum number of epochs that indicates
-      #   how many times the whole data is given to the training process.
-      # @param batch_size [Integer] The size of the mini batches.
-      # @param tol [Float] The tolerance of loss for terminating optimization.
-      # @param n_jobs [Integer] The number of jobs for running the fit method in parallel.
-      #   If nil is given, the method does not execute in parallel.
-      #   If zero or less is given, it becomes equal to the number of processors.
-      #   This parameter is ignored if the Parallel gem is not loaded.
-      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
-      # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
-                     reg_param: 1.0, l1_ratio: 0.5, fit_bias: true, bias_scale: 1.0,
-                     max_iter: 1000, batch_size: 50, tol: 1e-4,
-                     n_jobs: nil, verbose: false, random_seed: nil)
-        check_params_numeric(learning_rate: learning_rate, momentum: momentum,
-                             reg_param: reg_param, l1_ratio: l1_ratio, bias_scale: bias_scale,
-                             max_iter: max_iter, batch_size: batch_size, tol: tol)
-        check_params_boolean(fit_bias: fit_bias, verbose: verbose)
-        check_params_numeric_or_nil(decay: decay, n_jobs: n_jobs, random_seed: random_seed)
-        check_params_positive(learning_rate: learning_rate, reg_param: reg_param, max_iter: max_iter, batch_size: batch_size)
-        super()
-        @params.merge!(method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h)
-        @params[:decay] ||= @params[:reg_param] * @params[:learning_rate]
-        @params[:random_seed] ||= srand
-        @rng = Random.new(@params[:random_seed])
-        @penalty_type = ELASTICNET_PENALTY
-        @loss_func = LinearModel::Loss::MeanSquaredError.new
-        @weight_vec = nil
-        @bias_term = nil
-      end
-
-      # Fit the model with given training data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
-      # @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values to be used for fitting the model.
-      # @return [ElasticNet] The learned regressor itself.
-      def fit(x, y)
-        x = check_convert_sample_array(x)
-        y = check_convert_tvalue_array(y)
-        check_sample_tvalue_size(x, y)
-
-        n_outputs = y.shape[1].nil? ? 1 : y.shape[1]
-        n_features = x.shape[1]
-
-        if n_outputs > 1
-          @weight_vec = Numo::DFloat.zeros(n_outputs, n_features)
-          @bias_term = Numo::DFloat.zeros(n_outputs)
-          if enable_parallel?
-            models = parallel_map(n_outputs) { |n| partial_fit(x, y[true, n]) }
-            n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = models[n] }
-          else
-            n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = partial_fit(x, y[true, n]) }
-          end
-        else
-          @weight_vec, @bias_term = partial_fit(x, y)
-        end
-        self
-      end
-
-      # Predict values for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
-      # @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted values per sample.
-      def predict(x)
-        x = check_convert_sample_array(x)
-        x.dot(@weight_vec.transpose) + @bias_term
-      end
-    end
-  end
-end

data/lib/rumale/linear_model/lasso.rb

@@ -1,115 +0,0 @@
-# frozen_string_literal: true
-
-require 'rumale/linear_model/base_sgd'
-require 'rumale/base/regressor'
-
-module Rumale
-  module LinearModel
-    # Lasso is a class that implements Lasso Regression
-    # with stochastic gradient descent (SGD) optimization.
-    #
-    # @example
-    #   estimator =
-    #     Rumale::LinearModel::Lasso.new(reg_param: 0.1, max_iter: 1000, batch_size: 20, random_seed: 1)
-    #   estimator.fit(training_samples, traininig_values)
-    #   results = estimator.predict(testing_samples)
-    #
-    # *Reference*
-    # - Shalev-Shwartz, S., and Singer, Y., "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
-    # - Tsuruoka, Y., Tsujii, J., and Ananiadou, S., "Stochastic Gradient Descent Training for L1-regularized Log-linear Models with Cumulative Penalty," Proc. ACL'09, pp. 477--485, 2009.
-    # - Bottou, L., "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
-    class Lasso < BaseSGD
-      include Base::Regressor
-
-      # Return the weight vector.
-      # @return [Numo::DFloat] (shape: [n_outputs, n_features])
-      attr_reader :weight_vec
-
-      # Return the bias term (a.k.a. intercept).
-      # @return [Numo::DFloat] (shape: [n_outputs])
-      attr_reader :bias_term
-
-      # Return the random generator for random sampling.
-      # @return [Random]
-      attr_reader :rng
-
-      # Create a new Lasso regressor.
-      #
-      # @param learning_rate [Float] The initial value of learning rate.
-      #   The learning rate decreases as the iteration proceeds according to the equation: learning_rate / (1 + decay * t).
-      # @param decay [Float] The smoothing parameter for decreasing learning rate as the iteration proceeds.
-      #   If nil is given, the decay sets to 'reg_param * learning_rate'.
-      # @param momentum [Float] The momentum factor.
-      # @param reg_param [Float] The regularization parameter.
-      # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
-      # @param bias_scale [Float] The scale of the bias term.
-      # @param max_iter [Integer] The maximum number of epochs that indicates
-      #   how many times the whole data is given to the training process.
-      # @param batch_size [Integer] The size of the mini batches.
-      # @param tol [Float] The tolerance of loss for terminating optimization.
-      # @param n_jobs [Integer] The number of jobs for running the fit method in parallel.
-      #   If nil is given, the method does not execute in parallel.
-      #   If zero or less is given, it becomes equal to the number of processors.
-      #   This parameter is ignored if the Parallel gem is not loaded.
-      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
-      # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(learning_rate: 0.01, decay: nil, momentum: 0.9,
-                     reg_param: 1.0, fit_bias: true, bias_scale: 1.0,
-                     max_iter: 1000, batch_size: 50, tol: 1e-4,
-                     n_jobs: nil, verbose: false, random_seed: nil)
-        check_params_numeric(learning_rate: learning_rate, momentum: momentum,
-                             reg_param: reg_param, bias_scale: bias_scale,
-                             max_iter: max_iter, batch_size: batch_size, tol: tol)
-        check_params_boolean(fit_bias: fit_bias, verbose: verbose)
-        check_params_numeric_or_nil(decay: decay, n_jobs: n_jobs, random_seed: random_seed)
-        check_params_positive(learning_rate: learning_rate, reg_param: reg_param, max_iter: max_iter, batch_size: batch_size)
-        super()
-        @params.merge!(method(:initialize).parameters.map { |_t, arg| [arg, binding.local_variable_get(arg)] }.to_h)
-        @params[:decay] ||= @params[:reg_param] * @params[:learning_rate]
-        @params[:random_seed] ||= srand
-        @rng = Random.new(@params[:random_seed])
-        @penalty_type = L1_PENALTY
-        @loss_func = LinearModel::Loss::MeanSquaredError.new
-        @weight_vec = nil
-        @bias_term = nil
-      end
-
-      # Fit the model with given training data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
-      # @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values to be used for fitting the model.
-      # @return [Lasso] The learned regressor itself.
-      def fit(x, y)
-        x = check_convert_sample_array(x)
-        y = check_convert_tvalue_array(y)
-        check_sample_tvalue_size(x, y)
-
-        n_outputs = y.shape[1].nil? ? 1 : y.shape[1]
-        n_features = x.shape[1]
-
-        if n_outputs > 1
-          @weight_vec = Numo::DFloat.zeros(n_outputs, n_features)
-          @bias_term = Numo::DFloat.zeros(n_outputs)
-          if enable_parallel?
-            models = parallel_map(n_outputs) { |n| partial_fit(x, y[true, n]) }
-            n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = models[n] }
-          else
-            n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = partial_fit(x, y[true, n]) }
-          end
-        else
-          @weight_vec, @bias_term = partial_fit(x, y)
-        end
-        self
-      end
-
-      # Predict values for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
-      # @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted values per sample.
-      def predict(x)
-        x = check_convert_sample_array(x)
-        x.dot(@weight_vec.transpose) + @bias_term
-      end
-    end
-  end
-end