svmkit 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8486463886064e5aa5169dbed20101a01d01101483226cf96e38b377144e153
-  data.tar.gz: 79807cdbe9f10fba17cd91dbcd68c713180fd986e9b6fab575a849c762bb0d2e
+  metadata.gz: 1d52bf496a762b096a5f6dbeec278a1cae8079b53d6c91cc13c07dca7a799fde
+  data.tar.gz: e5ca2fed307b82e88dfe816691a4715d62a3187c1cad71421a48bea65037b19c
 SHA512:
-  metadata.gz: 2ec6c45c99bda82813644b7b4a043a4dda27810e24e10b4588182cbc3dea106eb05ac013354177861ab4abf552d89670ea1f0d39d50976a232feedc0daee9030
-  data.tar.gz: 373a566f294bab3d6fb232516f5720a877b6e20999c3253ae49044d759f83b06999fef4e259df91b950fe49c395042c1a74675fadb326460a7e65d54048ca325
+  metadata.gz: 620370c119300f3f419550609444eba4aa34561a954e8ec26cf6a0d3522cd32cabf1f6875092de5ab0dd202ebf7b772c1d6d6421cd05d90cfeeeeadea3cd0565
+  data.tar.gz: a0d8b5a7b91c4a8e2ffb4312a8082096a2e4fbd411e37bd36752ac19a66f6b9accf22be15894d6a88421b7296bde90ead33b55604085d139e5faec64b97f0f55

data/.rubocop.yml

@@ -1,7 +1,7 @@
 inherit_from: .rubocop_todo.yml
 
 AllCops:
-  TargetRubyVersion: 2.1
+  TargetRubyVersion: 2.2
   DisplayCopNames: true
   DisplayStyleGuide: true
 
@@ -39,3 +39,6 @@ Naming/UncommunicativeMethodParamName:
 
 Style/FormatStringToken:
   Enabled: false
+
+Style/NumericLiterals:
+  Enabled: false

data/HISTORY.md

@@ -1,3 +1,8 @@
+# 0.6.1
+- Add abstract class for linear estimator with stochastic gradient descent.
+- Refactor linear estimators to use linear esitmator abstract class.
+- Refactor decistion tree classes to avoid unneeded type conversion.
+
 # 0.6.0
 - Add class for Principal Component Analysis.
 - Add class for Non-negative Matrix Factorization.

data/lib/svmkit/decomposition/nmf.rb

@@ -121,11 +121,11 @@ module SVMKit
           # update
           if update_comps
             nume = coefficients.transpose.dot(x)
-            deno = (coefficients.transpose.dot(coefficients)).dot(@components) + @params[:eps]
+            deno = coefficients.transpose.dot(coefficients).dot(@components) + @params[:eps]
             @components *= (nume / deno)
           end
           nume = x.dot(@components.transpose)
-          deno = (coefficients.dot(@components)).dot(@components.transpose) + @params[:eps]
+          deno = coefficients.dot(@components).dot(@components.transpose) + @params[:eps]
           coefficients *= (nume / deno)
           # normalize
           norm = Numo::NMath.sqrt((@components**2).sum(1)) + @params[:eps]

data/lib/svmkit/linear_model/lasso.rb

@@ -1,9 +1,8 @@
 # frozen_string_literal: true
 
 require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
+require 'svmkit/linear_model/sgd_linear_estimator'
 require 'svmkit/base/regressor'
-require 'svmkit/optimizer/nadam'
 
 module SVMKit
   module LinearModel
@@ -19,8 +18,7 @@ module SVMKit
     # *Reference*
     # - S. Shalev-Shwartz and Y. Singer, "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
     # - L. Bottou, "Large-Scale Machine Learning with Stochastic Gradient Descent," Proc. COMPSTAT'10, pp. 177--186, 2010.
-    class Lasso
-      include Base::BaseEstimator
+    class Lasso < SGDLinearEstimator
       include Base::Regressor
       include Validation
 
@@ -40,29 +38,19 @@ module SVMKit
       #
       # @param reg_param [Float] The regularization parameter.
       # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
+      # @param bias_scale [Float] The scale of the bias term.
       # @param max_iter [Integer] The maximum number of iterations.
       # @param batch_size [Integer] The size of the mini batches.
       # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
       #   If nil is given, Nadam is used.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(reg_param: 1.0, fit_bias: false, max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
-        check_params_float(reg_param: reg_param)
+      def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0, max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
+        check_params_float(reg_param: reg_param, bias_scale: bias_scale)
         check_params_integer(max_iter: max_iter, batch_size: batch_size)
         check_params_boolean(fit_bias: fit_bias)
         check_params_type_or_nil(Integer, random_seed: random_seed)
         check_params_positive(reg_param: reg_param, max_iter: max_iter, batch_size: batch_size)
-        @params = {}
-        @params[:reg_param] = reg_param
-        @params[:fit_bias] = fit_bias
-        @params[:max_iter] = max_iter
-        @params[:batch_size] = batch_size
-        @params[:optimizer] = optimizer
-        @params[:optimizer] ||= Optimizer::Nadam.new
-        @params[:random_seed] = random_seed
-        @params[:random_seed] ||= srand
-        @weight_vec = nil
-        @bias_term = nil
-        @rng = Random.new(@params[:random_seed])
+        super
       end
 
       # Fit the model with given training data.
@@ -76,14 +64,14 @@ module SVMKit
         check_sample_tvalue_size(x, y)
 
         n_outputs = y.shape[1].nil? ? 1 : y.shape[1]
-        _n_samples, n_features = x.shape
+        n_features = x.shape[1]
 
         if n_outputs > 1
           @weight_vec = Numo::DFloat.zeros(n_outputs, n_features)
           @bias_term = Numo::DFloat.zeros(n_outputs)
-          n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = single_fit(x, y[true, n]) }
+          n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = partial_fit(x, y[true, n]) }
         else
-          @weight_vec, @bias_term = single_fit(x, y)
+          @weight_vec, @bias_term = partial_fit(x, y)
         end
 
         self
@@ -119,60 +107,32 @@ module SVMKit
 
       private
 
-      def single_fit(x, y)
-        # Expand feature vectors for bias term.
-        samples = @params[:fit_bias] ? expand_feature(x) : x
-        # Initialize some variables.
-        n_samples, n_features = samples.shape
-        rand_ids = [*0...n_samples].shuffle(random: @rng)
-        weight_vec = Numo::DFloat.zeros(n_features)
-        left_weight_vec = Numo::DFloat.zeros(n_features)
-        right_weight_vec = Numo::DFloat.zeros(n_features)
-        left_optimizer = @params[:optimizer].dup
-        right_optimizer = @params[:optimizer].dup
-        # Start optimization.
-        @params[:max_iter].times do |_t|
-          # Random sampling.
-          subset_ids = rand_ids.shift(@params[:batch_size])
-          rand_ids.concat(subset_ids)
-          data = samples[subset_ids, true]
-          values = y[subset_ids]
-          # Calculate gradients for loss function.
-          loss_grad = loss_gradient(data, values, weight_vec)
-          next if loss_grad.ne(0.0).count.zero?
-          # Update weight.
-          left_weight_vec = round_weight(left_optimizer.call(left_weight_vec, left_weight_gradient(loss_grad, data)))
-          right_weight_vec = round_weight(right_optimizer.call(right_weight_vec, right_weight_gradient(loss_grad, data)))
-          weight_vec = left_weight_vec - right_weight_vec
-        end
-        split_weight_vec_bias(weight_vec)
+      def partial_fit(x, y)
+        n_features = @params[:fit_bias] ? x.shape[1] + 1 : x.shape[1]
+        @left_weight = Numo::DFloat.zeros(n_features)
+        @right_weight = Numo::DFloat.zeros(n_features)
+        @left_optimizer = @params[:optimizer].dup
+        @right_optimizer = @params[:optimizer].dup
+        super
       end
 
-      def loss_gradient(x, y, weight)
+      def calc_loss_gradient(x, y, weight)
         2.0 * (x.dot(weight) - y)
       end
 
-      def left_weight_gradient(loss_grad, data)
-        ((@params[:reg_param] + loss_grad).expand_dims(1) * data).mean(0)
+      def calc_new_weight(_optimizer, x, _weight, loss_gradient)
+        @left_weight = round_weight(@left_optimizer.call(@left_weight, calc_weight_gradient(loss_gradient, x)))
+        @right_weight = round_weight(@right_optimizer.call(@right_weight, calc_weight_gradient(-loss_gradient, x)))
+        @left_weight - @right_weight
       end
 
-      def right_weight_gradient(loss_grad, data)
-        ((@params[:reg_param] - loss_grad).expand_dims(1) * data).mean(0)
+      def calc_weight_gradient(loss_gradient, data)
+        ((@params[:reg_param] + loss_gradient).expand_dims(1) * data).mean(0)
       end
 
       def round_weight(weight)
         0.5 * (weight + weight.abs)
       end
-
-      def expand_feature(x)
-        Numo::NArray.hstack([x, Numo::DFloat.ones([x.shape[0], 1])])
-      end
-
-      def split_weight_vec_bias(weight_vec)
-        weights = @params[:fit_bias] ? weight_vec[0...-1] : weight_vec
-        bias = @params[:fit_bias] ? weight_vec[-1] : 0.0
-        [weights, bias]
-      end
     end
   end
 end

data/lib/svmkit/linear_model/linear_regression.rb

@@ -1,9 +1,8 @@
 # frozen_string_literal: true
 
 require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
+require 'svmkit/linear_model/sgd_linear_estimator'
 require 'svmkit/base/regressor'
-require 'svmkit/optimizer/nadam'
 
 module SVMKit
   module LinearModel
@@ -16,8 +15,7 @@ module SVMKit
     #   estimator.fit(training_samples, traininig_values)
     #   results = estimator.predict(testing_samples)
     #
-    class LinearRegression
-      include Base::BaseEstimator
+    class LinearRegression < SGDLinearEstimator
       include Base::Regressor
       include Validation
 
@@ -36,27 +34,20 @@ module SVMKit
       # Create a new ordinary least square linear regressor.
       #
       # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
+      # @param bias_scale [Float] The scale of the bias term.
       # @param max_iter [Integer] The maximum number of iterations.
       # @param batch_size [Integer] The size of the mini batches.
       # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
       #   If nil is given, Nadam is used.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(fit_bias: false, max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
+      def initialize(fit_bias: false, bias_scale: 1.0, max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
+        check_params_float(bias_scale: bias_scale)
         check_params_integer(max_iter: max_iter, batch_size: batch_size)
         check_params_boolean(fit_bias: fit_bias)
         check_params_type_or_nil(Integer, random_seed: random_seed)
         check_params_positive(max_iter: max_iter, batch_size: batch_size)
-        @params = {}
-        @params[:fit_bias] = fit_bias
-        @params[:max_iter] = max_iter
-        @params[:batch_size] = batch_size
-        @params[:optimizer] = optimizer
-        @params[:optimizer] ||= Optimizer::Nadam.new
-        @params[:random_seed] = random_seed
-        @params[:random_seed] ||= srand
-        @weight_vec = nil
-        @bias_term = nil
-        @rng = Random.new(@params[:random_seed])
+        super(reg_param: 0.0, fit_bias: fit_bias, bias_scale: bias_scale,
+              max_iter: max_iter, batch_size: batch_size, optimizer: optimizer, random_seed: random_seed)
       end
 
       # Fit the model with given training data.
@@ -75,9 +66,9 @@ module SVMKit
         if n_outputs > 1
           @weight_vec = Numo::DFloat.zeros(n_outputs, n_features)
           @bias_term = Numo::DFloat.zeros(n_outputs)
-          n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = single_fit(x, y[true, n]) }
+          n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = partial_fit(x, y[true, n]) }
         else
-          @weight_vec, @bias_term = single_fit(x, y)
+          @weight_vec, @bias_term = partial_fit(x, y)
         end
 
         self
@@ -113,47 +104,9 @@ module SVMKit
 
       private
 
-      def single_fit(x, y)
-        # Expand feature vectors for bias term.
-        samples = @params[:fit_bias] ? expand_feature(x) : x
-        # Initialize some variables.
-        n_samples, n_features = samples.shape
-        rand_ids = [*0...n_samples].shuffle(random: @rng)
-        weight_vec = Numo::DFloat.zeros(n_features)
-        optimizer = @params[:optimizer].dup
-        # Start optimization.
-        @params[:max_iter].times do |_t|
-          # Random sampling.
-          subset_ids = rand_ids.shift(@params[:batch_size])
-          rand_ids.concat(subset_ids)
-          data = samples[subset_ids, true]
-          values = y[subset_ids]
-          # Calculate gradients for loss function.
-          loss_grad = loss_gradient(data, values, weight_vec)
-          next if loss_grad.ne(0.0).count.zero?
-          # Update weight.
-          weight_vec = optimizer.call(weight_vec, weight_gradient(loss_grad, data, weight_vec))
-        end
-        split_weight_vec_bias(weight_vec)
-      end
-
-      def loss_gradient(x, y, weight)
+      def calc_loss_gradient(x, y, weight)
         2.0 * (x.dot(weight) - y)
       end
-
-      def weight_gradient(loss_grad, data, _weight)
-        (loss_grad.expand_dims(1) * data).mean(0)
-      end
-
-      def expand_feature(x)
-        Numo::NArray.hstack([x, Numo::DFloat.ones([x.shape[0], 1])])
-      end
-
-      def split_weight_vec_bias(weight_vec)
-        weights = @params[:fit_bias] ? weight_vec[0...-1] : weight_vec
-        bias = @params[:fit_bias] ? weight_vec[-1] : 0.0
-        [weights, bias]
-      end
     end
   end
 end

data/lib/svmkit/linear_model/logistic_regression.rb

@@ -1,9 +1,8 @@
 # frozen_string_literal: true
 
 require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
+require 'svmkit/linear_model/sgd_linear_estimator'
 require 'svmkit/base/classifier'
-require 'svmkit/optimizer/nadam'
 
 module SVMKit
   module LinearModel
@@ -19,8 +18,7 @@ module SVMKit
     #
     # *Reference*
     # - S. Shalev-Shwartz, Y. Singer, N. Srebro, and A. Cotter, "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Mathematical Programming, vol. 127 (1), pp. 3--30, 2011.
-    class LogisticRegression
-      include Base::BaseEstimator
+    class LogisticRegression < SGDLinearEstimator
       include Base::Classifier
       include Validation
 
@@ -58,20 +56,8 @@ module SVMKit
         check_params_boolean(fit_bias: fit_bias)
         check_params_type_or_nil(Integer, random_seed: random_seed)
         check_params_positive(reg_param: reg_param, bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
-        @params = {}
-        @params[:reg_param] = reg_param
-        @params[:fit_bias] = fit_bias
-        @params[:bias_scale] = bias_scale
-        @params[:max_iter] = max_iter
-        @params[:batch_size] = batch_size
-        @params[:optimizer] = optimizer
-        @params[:optimizer] ||= Optimizer::Nadam.new
-        @params[:random_seed] = random_seed
-        @params[:random_seed] ||= srand
-        @weight_vec = nil
-        @bias_term = nil
+        super
         @classes = nil
-        @rng = Random.new(@params[:random_seed])
       end
 
       # Fit the model with given training data.
@@ -86,21 +72,19 @@ module SVMKit
 
         @classes = Numo::Int32[*y.to_a.uniq.sort]
         n_classes = @classes.size
-        _n_samples, n_features = x.shape
+        n_features = x.shape[1]
 
         if n_classes > 2
           @weight_vec = Numo::DFloat.zeros(n_classes, n_features)
           @bias_term = Numo::DFloat.zeros(n_classes)
           n_classes.times do |n|
             bin_y = Numo::Int32.cast(y.eq(@classes[n])) * 2 - 1
-            weight, bias = binary_fit(x, bin_y)
-            @weight_vec[n, true] = weight
-            @bias_term[n] = bias
+            @weight_vec[n, true], @bias_term[n] = partial_fit(x, bin_y)
           end
         else
           negative_label = y.to_a.uniq.min
           bin_y = Numo::Int32.cast(y.ne(negative_label)) * 2 - 1
-          @weight_vec, @bias_term = binary_fit(x, bin_y)
+          @weight_vec, @bias_term = partial_fit(x, bin_y)
         end
 
         self
@@ -169,47 +153,8 @@ module SVMKit
 
       private
 
-      def binary_fit(x, y)
-        # Expand feature vectors for bias term.
-        samples = @params[:fit_bias] ? expand_feature(x) : x
-        # Initialize some variables.
-        n_samples, n_features = samples.shape
-        rand_ids = [*0...n_samples].shuffle(random: @rng)
-        weight_vec = Numo::DFloat.zeros(n_features)
-        optimizer = @params[:optimizer].dup
-        # Start optimization.
-        @params[:max_iter].times do |_t|
-          # random sampling
-          subset_ids = rand_ids.shift(@params[:batch_size])
-          rand_ids.concat(subset_ids)
-          data = samples[subset_ids, true]
-          labels = y[subset_ids]
-          # calculate gradient for loss function.
-          loss_grad = loss_gradient(data, labels, weight_vec)
-          # update weight.
-          weight_vec = optimizer.call(weight_vec, weight_gradient(loss_grad, data, weight_vec))
-        end
-        split_weight_vec_bias(weight_vec)
-      end
-
-      def loss_gradient(x, y, weight)
-        z = x.dot(weight)
-        grad = y / (Numo::NMath.exp(-y * z) + 1.0) - y
-        grad
-      end
-
-      def weight_gradient(loss_grad, x, weight)
-        x.transpose.dot(loss_grad) / @params[:batch_size] + @params[:reg_param] * weight
-      end
-
-      def expand_feature(x)
-        Numo::NArray.hstack([x, Numo::DFloat.ones([x.shape[0], 1]) * @params[:bias_scale]])
-      end
-
-      def split_weight_vec_bias(weight_vec)
-        weights = @params[:fit_bias] ? weight_vec[0...-1] : weight_vec
-        bias = @params[:fit_bias] ? weight_vec[-1] : 0.0
-        [weights, bias]
+      def calc_loss_gradient(x, y, weight)
+        y / (Numo::NMath.exp(-y * x.dot(weight)) + 1.0) - y
       end
     end
   end

data/lib/svmkit/linear_model/ridge.rb

@@ -1,9 +1,8 @@
 # frozen_string_literal: true
 
 require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
+require 'svmkit/linear_model/sgd_linear_estimator'
 require 'svmkit/base/regressor'
-require 'svmkit/optimizer/nadam'
 
 module SVMKit
   module LinearModel
@@ -16,8 +15,7 @@ module SVMKit
     #   estimator.fit(training_samples, traininig_values)
     #   results = estimator.predict(testing_samples)
     #
-    class Ridge
-      include Base::BaseEstimator
+    class Ridge < SGDLinearEstimator
       include Base::Regressor
       include Validation
 
@@ -37,29 +35,19 @@ module SVMKit
       #
       # @param reg_param [Float] The regularization parameter.
       # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
+      # @param bias_scale [Float] The scale of the bias term.
       # @param max_iter [Integer] The maximum number of iterations.
       # @param batch_size [Integer] The size of the mini batches.
       # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
       #   If nil is given, Nadam is used.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(reg_param: 1.0, fit_bias: false, max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
-        check_params_float(reg_param: reg_param)
+      def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0, max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
+        check_params_float(reg_param: reg_param, bias_scale: bias_scale)
         check_params_integer(max_iter: max_iter, batch_size: batch_size)
         check_params_boolean(fit_bias: fit_bias)
         check_params_type_or_nil(Integer, random_seed: random_seed)
         check_params_positive(reg_param: reg_param, max_iter: max_iter, batch_size: batch_size)
-        @params = {}
-        @params[:reg_param] = reg_param
-        @params[:fit_bias] = fit_bias
-        @params[:max_iter] = max_iter
-        @params[:batch_size] = batch_size
-        @params[:optimizer] = optimizer
-        @params[:optimizer] ||= Optimizer::Nadam.new
-        @params[:random_seed] = random_seed
-        @params[:random_seed] ||= srand
-        @weight_vec = nil
-        @bias_term = nil
-        @rng = Random.new(@params[:random_seed])
+        super
       end
 
       # Fit the model with given training data.
@@ -73,14 +61,14 @@ module SVMKit
         check_sample_tvalue_size(x, y)
 
         n_outputs = y.shape[1].nil? ? 1 : y.shape[1]
-        _n_samples, n_features = x.shape
+        n_features = x.shape[1]
 
         if n_outputs > 1
           @weight_vec = Numo::DFloat.zeros(n_outputs, n_features)
           @bias_term = Numo::DFloat.zeros(n_outputs)
-          n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = single_fit(x, y[true, n]) }
+          n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = partial_fit(x, y[true, n]) }
         else
-          @weight_vec, @bias_term = single_fit(x, y)
+          @weight_vec, @bias_term = partial_fit(x, y)
         end
 
         self
@@ -116,47 +104,9 @@ module SVMKit
 
       private
 
-      def single_fit(x, y)
-        # Expand feature vectors for bias term.
-        samples = @params[:fit_bias] ? expand_feature(x) : x
-        # Initialize some variables.
-        n_samples, n_features = samples.shape
-        rand_ids = [*0...n_samples].shuffle(random: @rng)
-        weight_vec = Numo::DFloat.zeros(n_features)
-        optimizer = @params[:optimizer].dup
-        # Start optimization.
-        @params[:max_iter].times do |_t|
-          # Random sampling.
-          subset_ids = rand_ids.shift(@params[:batch_size])
-          rand_ids.concat(subset_ids)
-          data = samples[subset_ids, true]
-          values = y[subset_ids]
-          # Calculate gradients for loss function.
-          loss_grad = loss_gradient(data, values, weight_vec)
-          next if loss_grad.ne(0.0).count.zero?
-          # Update weight.
-          weight_vec = optimizer.call(weight_vec, weight_gradient(loss_grad, data, weight_vec))
-        end
-        split_weight_vec_bias(weight_vec)
-      end
-
-      def loss_gradient(x, y, weight)
+      def calc_loss_gradient(x, y, weight)
         2.0 * (x.dot(weight) - y)
       end
-
-      def weight_gradient(loss_grad, data, weight)
-        (loss_grad.expand_dims(1) * data).mean(0) + @params[:reg_param] * weight
-      end
-
-      def expand_feature(x)
-        Numo::NArray.hstack([x, Numo::DFloat.ones([x.shape[0], 1])])
-      end
-
-      def split_weight_vec_bias(weight_vec)
-        weights = @params[:fit_bias] ? weight_vec[0...-1] : weight_vec
-        bias = @params[:fit_bias] ? weight_vec[-1] : 0.0
-        [weights, bias]
-      end
     end
   end
 end

data/lib/svmkit/linear_model/sgd_linear_estimator.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'svmkit/base/base_estimator'
+require 'svmkit/optimizer/nadam'
+
+module SVMKit
+  module LinearModel
+    # SGDLinearEstimator is an abstract class for implementation of linear estimator
+    # with mini-batch stochastic gradient descent optimization.
+    # This class is used for internal process.
+    class SGDLinearEstimator
+      include Base::BaseEstimator
+
+      # Initialize a linear estimator.
+      #
+      # @param reg_param [Float] The regularization parameter.
+      # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
+      # @param bias_scale [Float] The scale of the bias term.
+      # @param max_iter [Integer] The maximum number of iterations.
+      # @param batch_size [Integer] The size of the mini batches.
+      # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
+      #   If nil is given, Nadam is used.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0,
+                     max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
+        @params = {}
+        @params[:reg_param] = reg_param
+        @params[:fit_bias] = fit_bias
+        @params[:bias_scale] = bias_scale
+        @params[:max_iter] = max_iter
+        @params[:batch_size] = batch_size
+        @params[:optimizer] = optimizer
+        @params[:optimizer] ||= Optimizer::Nadam.new
+        @params[:random_seed] = random_seed
+        @params[:random_seed] ||= srand
+        @weight_vec = nil
+        @bias_term = nil
+        @rng = Random.new(@params[:random_seed])
+      end
+
+      private
+
+      def partial_fit(x, y)
+        # Expand feature vectors for bias term.
+        samples = @params[:fit_bias] ? expand_feature(x) : x
+        # Initialize some variables.
+        n_samples, n_features = samples.shape
+        rand_ids = [*0...n_samples].shuffle(random: @rng)
+        weight = Numo::DFloat.zeros(n_features)
+        optimizer = @params[:optimizer].dup
+        # Optimization.
+        @params[:max_iter].times do |_t|
+          # Random sampling
+          subset_ids = rand_ids.shift(@params[:batch_size])
+          rand_ids.concat(subset_ids)
+          sub_samples = samples[subset_ids, true]
+          sub_targets = y[subset_ids]
+          # Update weight.
+          loss_gradient = calc_loss_gradient(sub_samples, sub_targets, weight)
+          next if loss_gradient.ne(0.0).count.zero?
+          weight = calc_new_weight(optimizer, sub_samples, weight, loss_gradient)
+        end
+        split_weight(weight)
+      end
+
+      def calc_loss_gradient(_x, _y, _weight)
+        raise NotImplementedError, "#{__method__} has to be implemented in #{self.class}."
+      end
+
+      def calc_new_weight(optimizer, x, weight, loss_gradient)
+        weight_gradient = x.transpose.dot(loss_gradient) / @params[:batch_size] + @params[:reg_param] * weight
+        optimizer.call(weight, weight_gradient)
+      end
+
+      def expand_feature(x)
+        n_samples = x.shape[0]
+        Numo::NArray.hstack([x, Numo::DFloat.ones([n_samples, 1]) * @params[:bias_scale]])
+      end
+
+      def split_weight(weight)
+        if @params[:fit_bias]
+          [weight[0...-1], weight[-1]]
+        else
+          [weight, 0.0]
+        end
+      end
+    end
+  end
+end

data/lib/svmkit/linear_model/svc.rb

@@ -1,9 +1,8 @@
 # frozen_string_literal: true
 
 require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
+require 'svmkit/linear_model/sgd_linear_estimator'
 require 'svmkit/base/classifier'
-require 'svmkit/optimizer/nadam'
 require 'svmkit/probabilistic_output'
 
 module SVMKit
@@ -21,8 +20,7 @@ module SVMKit
     #
     # *Reference*
     # - S. Shalev-Shwartz and Y. Singer, "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
-    class SVC
-      include Base::BaseEstimator
+    class SVC < SGDLinearEstimator
       include Base::Classifier
       include Validation
 
@@ -60,22 +58,11 @@ module SVMKit
         check_params_boolean(fit_bias: fit_bias, probability: probability)
         check_params_type_or_nil(Integer, random_seed: random_seed)
         check_params_positive(reg_param: reg_param, bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
-        @params = {}
-        @params[:reg_param] = reg_param
-        @params[:fit_bias] = fit_bias
-        @params[:bias_scale] = bias_scale
-        @params[:max_iter] = max_iter
-        @params[:batch_size] = batch_size
+        super(reg_param: reg_param, fit_bias: fit_bias, bias_scale: bias_scale,
+              max_iter: max_iter, batch_size: batch_size, optimizer: optimizer, random_seed: random_seed)
         @params[:probability] = probability
-        @params[:optimizer] = optimizer
-        @params[:optimizer] ||= Optimizer::Nadam.new
-        @params[:random_seed] = random_seed
-        @params[:random_seed] ||= srand
-        @weight_vec = nil
-        @bias_term = nil
         @prob_param = nil
         @classes = nil
-        @rng = Random.new(@params[:random_seed])
       end
 
       # Fit the model with given training data.
@@ -90,7 +77,7 @@ module SVMKit
 
         @classes = Numo::Int32[*y.to_a.uniq.sort]
         n_classes = @classes.size
-        _n_samples, n_features = x.shape
+        n_features = x.shape[1]
 
         if n_classes > 2
           @weight_vec = Numo::DFloat.zeros(n_classes, n_features)
@@ -98,11 +85,9 @@ module SVMKit
           @prob_param = Numo::DFloat.zeros(n_classes, 2)
           n_classes.times do |n|
             bin_y = Numo::Int32.cast(y.eq(@classes[n])) * 2 - 1
-            weight, bias = binary_fit(x, bin_y)
-            @weight_vec[n, true] = weight
-            @bias_term[n] = bias
+            @weight_vec[n, true], @bias_term[n] = partial_fit(x, bin_y)
             @prob_param[n, true] = if @params[:probability]
-                                     SVMKit::ProbabilisticOutput.fit_sigmoid(x.dot(weight.transpose) + bias, bin_y)
+                                     SVMKit::ProbabilisticOutput.fit_sigmoid(x.dot(@weight_vec[n, true].transpose) + @bias_term[n], bin_y)
                                    else
                                      Numo::DFloat[1, 0]
                                    end
@@ -110,7 +95,7 @@ module SVMKit
         else
           negative_label = y.to_a.uniq.min
           bin_y = Numo::Int32.cast(y.ne(negative_label)) * 2 - 1
-          @weight_vec, @bias_term = binary_fit(x, bin_y)
+          @weight_vec, @bias_term = partial_fit(x, bin_y)
           @prob_param = if @params[:probability]
                           SVMKit::ProbabilisticOutput.fit_sigmoid(x.dot(@weight_vec.transpose) + @bias_term, bin_y)
                         else
@@ -188,50 +173,12 @@ module SVMKit
 
       private
 
-      def binary_fit(x, y)
-        # Expand feature vectors for bias term.
-        samples = @params[:fit_bias] ? expand_feature(x) : x
-        # Initialize some variables.
-        n_samples, n_features = samples.shape
-        rand_ids = [*0...n_samples].shuffle(random: @rng)
-        weight_vec = Numo::DFloat.zeros(n_features)
-        optimizer = @params[:optimizer].dup
-        # Start optimization.
-        @params[:max_iter].times do |_t|
-          # random sampling.
-          subset_ids = rand_ids.shift(@params[:batch_size])
-          rand_ids.concat(subset_ids)
-          data = samples[subset_ids, true]
-          labels = y[subset_ids]
-          # calculate gradient for loss function.
-          loss_grad = loss_gradient(data, labels, weight_vec)
-          next if loss_grad.ne(0.0).count.zero?
-          # update weight.
-          weight_vec = optimizer.call(weight_vec, weight_gradient(loss_grad, data, weight_vec))
-        end
-        split_weight_vec_bias(weight_vec)
-      end
-
-      def loss_gradient(x, y, weight)
+      def calc_loss_gradient(x, y, weight)
         target_ids = (x.dot(weight) * y).lt(1.0).where
         grad = Numo::DFloat.zeros(@params[:batch_size])
         grad[target_ids] = -y[target_ids]
         grad
       end
-
-      def weight_gradient(loss_grad, x, weight)
-        x.transpose.dot(loss_grad) / @params[:batch_size] + @params[:reg_param] * weight
-      end
-
-      def expand_feature(x)
-        Numo::NArray.hstack([x, Numo::DFloat.ones([x.shape[0], 1]) * @params[:bias_scale]])
-      end
-
-      def split_weight_vec_bias(weight_vec)
-        weights = @params[:fit_bias] ? weight_vec[0...-1] : weight_vec
-        bias = @params[:fit_bias] ? weight_vec[-1] : 0.0
-        [weights, bias]
-      end
     end
   end
 end

data/lib/svmkit/linear_model/svr.rb

@@ -1,9 +1,8 @@
 # frozen_string_literal: true
 
 require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
+require 'svmkit/linear_model/sgd_linear_estimator'
 require 'svmkit/base/regressor'
-require 'svmkit/optimizer/nadam'
 
 module SVMKit
   module LinearModel
@@ -18,8 +17,7 @@ module SVMKit
     #
     # *Reference*
     # 1. S. Shalev-Shwartz and Y. Singer, "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
-    class SVR
-      include Base::BaseEstimator
+    class SVR < SGDLinearEstimator
       include Base::Regressor
       include Validation
 
@@ -54,20 +52,9 @@ module SVMKit
         check_params_type_or_nil(Integer, random_seed: random_seed)
         check_params_positive(reg_param: reg_param, bias_scale: bias_scale, epsilon: epsilon,
                               max_iter: max_iter, batch_size: batch_size)
-        @params = {}
-        @params[:reg_param] = reg_param
-        @params[:fit_bias] = fit_bias
-        @params[:bias_scale] = bias_scale
+        super(reg_param: reg_param, fit_bias: fit_bias, bias_scale: bias_scale,
+              max_iter: max_iter, batch_size: batch_size, optimizer: optimizer, random_seed: random_seed)
         @params[:epsilon] = epsilon
-        @params[:max_iter] = max_iter
-        @params[:batch_size] = batch_size
-        @params[:optimizer] = optimizer
-        @params[:optimizer] ||= Optimizer::Nadam.new
-        @params[:random_seed] = random_seed
-        @params[:random_seed] ||= srand
-        @weight_vec = nil
-        @bias_term = nil
-        @rng = Random.new(@params[:random_seed])
       end
 
       # Fit the model with given training data.
@@ -81,14 +68,14 @@ module SVMKit
         check_sample_tvalue_size(x, y)
 
         n_outputs = y.shape[1].nil? ? 1 : y.shape[1]
-        _n_samples, n_features = x.shape
+        n_features = x.shape[1]
 
         if n_outputs > 1
           @weight_vec = Numo::DFloat.zeros(n_outputs, n_features)
           @bias_term = Numo::DFloat.zeros(n_outputs)
-          n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = single_fit(x, y[true, n]) }
+          n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = partial_fit(x, y[true, n]) }
         else
-          @weight_vec, @bias_term = single_fit(x, y)
+          @weight_vec, @bias_term = partial_fit(x, y)
         end
 
         self
@@ -124,49 +111,13 @@ module SVMKit
 
       private
 
-      def single_fit(x, y)
-        # Expand feature vectors for bias term.
-        samples = @params[:fit_bias] ? expand_feature(x) : x
-        # Initialize some variables.
-        n_samples, n_features = samples.shape
-        rand_ids = [*0...n_samples].shuffle(random: @rng)
-        weight_vec = Numo::DFloat.zeros(n_features)
-        optimizer = @params[:optimizer].dup
-        # Start optimization.
-        @params[:max_iter].times do |_t|
-          # random sampling
-          subset_ids = rand_ids.shift(@params[:batch_size])
-          rand_ids.concat(subset_ids)
-          data = samples[subset_ids, true]
-          values = y[subset_ids]
-          # update the weight vector.
-          loss_grad = loss_gradient(data, values, weight_vec)
-          weight_vec = optimizer.call(weight_vec, weight_gradient(loss_grad, data, weight_vec))
-        end
-        split_weight_vec_bias(weight_vec)
-      end
-
-      def loss_gradient(x, y, weight)
+      def calc_loss_gradient(x, y, weight)
         z = x.dot(weight)
         grad = Numo::DFloat.zeros(@params[:batch_size])
         grad[(z - y).gt(@params[:epsilon]).where] = 1
         grad[(y - z).gt(@params[:epsilon]).where] = -1
         grad
       end
-
-      def weight_gradient(loss_grad, x, weight)
-        x.transpose.dot(loss_grad) / @params[:batch_size] + @params[:reg_param] * weight
-      end
-
-      def expand_feature(x)
-        Numo::NArray.hstack([x, Numo::DFloat.ones([x.shape[0], 1]) * @params[:bias_scale]])
-      end
-
-      def split_weight_vec_bias(weight_vec)
-        weights = @params[:fit_bias] ? weight_vec[0...-1] : weight_vec
-        bias = @params[:fit_bias] ? weight_vec[-1] : 0.0
-        [weights, bias]
-      end
     end
   end
 end

data/lib/svmkit/tree/decision_tree_classifier.rb

@@ -213,7 +213,7 @@ module SVMKit
       end
 
       def put_leaf(node, y)
-        node.probs = Numo::DFloat[*(@classes.to_a.map { |c| y.eq(c).count })] / node.n_samples
+        node.probs = Numo::DFloat.cast(@classes.map { |c| y.eq(c).count_true }) / node.n_samples
         node.leaf = true
         node.leaf_id = @n_leaves
         @n_leaves += 1
@@ -234,18 +234,18 @@ module SVMKit
       end
 
       def splited_ids(features, threshold)
-        [features.le(threshold).where.to_a, features.gt(threshold).where.to_a]
+        [features.le(threshold).where, features.gt(threshold).where]
       end
 
       def gain(labels, labels_left, labels_right)
-        prob_left = labels_left.size / labels.size.to_f
-        prob_right = labels_right.size / labels.size.to_f
+        prob_left = labels_left.size.fdiv(labels.size)
+        prob_right = labels_right.size.fdiv(labels.size)
         impurity(labels) - prob_left * impurity(labels_left) - prob_right * impurity(labels_right)
       end
 
       def impurity(labels)
-        posterior_probs = Numo::DFloat[*(labels.to_a.uniq.sort.map { |c| labels.eq(c).count })] / labels.size.to_f
-        send(@criterion, posterior_probs)
+        cls = labels.to_a.uniq.sort
+        cls.size == 1 ? 0.0 : send(@criterion, Numo::DFloat[*(cls.map { |c| labels.eq(c).count_true.fdiv(labels.size) })])
       end
 
       def gini(posterior_probs)

data/lib/svmkit/tree/decision_tree_regressor.rb

@@ -208,7 +208,7 @@ module SVMKit
       end
 
       def splited_ids(features, threshold)
-        [features.le(threshold).where.to_a, features.gt(threshold).where.to_a]
+        [features.le(threshold).where, features.gt(threshold).where]
       end
 
       def gain(values, values_left, values_right)

data/lib/svmkit/version.rb

@@ -3,5 +3,5 @@
 # SVMKit is a machine learning library in Ruby.
 module SVMKit
   # @!visibility private
-  VERSION = '0.6.0'.freeze
+  VERSION = '0.6.1'.freeze
 end

data/lib/svmkit.rb

@@ -19,6 +19,7 @@ require 'svmkit/optimizer/rmsprop'
 require 'svmkit/optimizer/nadam'
 require 'svmkit/optimizer/yellow_fin'
 require 'svmkit/kernel_approximation/rbf'
+require 'svmkit/linear_model/sgd_linear_estimator'
 require 'svmkit/linear_model/svc'
 require 'svmkit/linear_model/svr'
 require 'svmkit/linear_model/logistic_regression'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: svmkit
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - yoshoku
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-28 00:00:00.000000000 Z
+date: 2018-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -140,6 +140,7 @@ files:
 - lib/svmkit/linear_model/linear_regression.rb
 - lib/svmkit/linear_model/logistic_regression.rb
 - lib/svmkit/linear_model/ridge.rb
+- lib/svmkit/linear_model/sgd_linear_estimator.rb
 - lib/svmkit/linear_model/svc.rb
 - lib/svmkit/linear_model/svr.rb
 - lib/svmkit/model_selection/cross_validation.rb