svmkit 0.7.3 → 0.8.1

data/lib/svmkit/linear_model/sgd_linear_estimator.rb

@@ -1,89 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/base/base_estimator'
-require 'svmkit/optimizer/nadam'
-
-module SVMKit
-  module LinearModel
-    # SGDLinearEstimator is an abstract class for implementation of linear estimator
-    # with mini-batch stochastic gradient descent optimization.
-    # This class is used for internal process.
-    class SGDLinearEstimator
-      include Base::BaseEstimator
-
-      # Initialize a linear estimator.
-      #
-      # @param reg_param [Float] The regularization parameter.
-      # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
-      # @param bias_scale [Float] The scale of the bias term.
-      # @param max_iter [Integer] The maximum number of iterations.
-      # @param batch_size [Integer] The size of the mini batches.
-      # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
-      #   If nil is given, Nadam is used.
-      # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0,
-                     max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
-        @params = {}
-        @params[:reg_param] = reg_param
-        @params[:fit_bias] = fit_bias
-        @params[:bias_scale] = bias_scale
-        @params[:max_iter] = max_iter
-        @params[:batch_size] = batch_size
-        @params[:optimizer] = optimizer
-        @params[:optimizer] ||= Optimizer::Nadam.new
-        @params[:random_seed] = random_seed
-        @params[:random_seed] ||= srand
-        @weight_vec = nil
-        @bias_term = nil
-        @rng = Random.new(@params[:random_seed])
-      end
-
-      private
-
-      def partial_fit(x, y)
-        # Expand feature vectors for bias term.
-        samples = @params[:fit_bias] ? expand_feature(x) : x
-        # Initialize some variables.
-        n_samples, n_features = samples.shape
-        rand_ids = [*0...n_samples].shuffle(random: @rng)
-        weight = Numo::DFloat.zeros(n_features)
-        optimizer = @params[:optimizer].dup
-        # Optimization.
-        @params[:max_iter].times do |_t|
-          # Random sampling
-          subset_ids = rand_ids.shift(@params[:batch_size])
-          rand_ids.concat(subset_ids)
-          sub_samples = samples[subset_ids, true]
-          sub_targets = y[subset_ids]
-          # Update weight.
-          loss_gradient = calc_loss_gradient(sub_samples, sub_targets, weight)
-          next if loss_gradient.ne(0.0).count.zero?
-          weight = calc_new_weight(optimizer, sub_samples, weight, loss_gradient)
-        end
-        split_weight(weight)
-      end
-
-      def calc_loss_gradient(_x, _y, _weight)
-        raise NotImplementedError, "#{__method__} has to be implemented in #{self.class}."
-      end
-
-      def calc_new_weight(optimizer, x, weight, loss_gradient)
-        weight_gradient = x.transpose.dot(loss_gradient) / @params[:batch_size] + @params[:reg_param] * weight
-        optimizer.call(weight, weight_gradient)
-      end
-
-      def expand_feature(x)
-        n_samples = x.shape[0]
-        Numo::NArray.hstack([x, Numo::DFloat.ones([n_samples, 1]) * @params[:bias_scale]])
-      end
-
-      def split_weight(weight)
-        if @params[:fit_bias]
-          [weight[0...-1].dup, weight[-1]]
-        else
-          [weight, 0.0]
-        end
-      end
-    end
-  end
-end

data/lib/svmkit/linear_model/svc.rb

@@ -1,184 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/linear_model/sgd_linear_estimator'
-require 'svmkit/base/classifier'
-require 'svmkit/probabilistic_output'
-
-module SVMKit
-  # This module consists of the classes that implement generalized linear models.
-  module LinearModel
-    # SVC is a class that implements Support Vector Classifier
-    # with mini-batch stochastic gradient descent optimization.
-    # For multiclass classification problem, it uses one-vs-the-rest strategy.
-    #
-    # @example
-    #   estimator =
-    #     SVMKit::LinearModel::SVC.new(reg_param: 1.0, max_iter: 1000, batch_size: 20, random_seed: 1)
-    #   estimator.fit(training_samples, traininig_labels)
-    #   results = estimator.predict(testing_samples)
-    #
-    # *Reference*
-    # - S. Shalev-Shwartz and Y. Singer, "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
-    class SVC < SGDLinearEstimator
-      include Base::Classifier
-      include Validation
-
-      # Return the weight vector for SVC.
-      # @return [Numo::DFloat] (shape: [n_classes, n_features])
-      attr_reader :weight_vec
-
-      # Return the bias term (a.k.a. intercept) for SVC.
-      # @return [Numo::DFloat] (shape: [n_classes])
-      attr_reader :bias_term
-
-      # Return the class labels.
-      # @return [Numo::Int32] (shape: [n_classes])
-      attr_reader :classes
-
-      # Return the random generator for performing random sampling.
-      # @return [Random]
-      attr_reader :rng
-
-      # Create a new classifier with Support Vector Machine by the SGD optimization.
-      #
-      # @param reg_param [Float] The regularization parameter.
-      # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
-      # @param bias_scale [Float] The scale of the bias term.
-      # @param max_iter [Integer] The maximum number of iterations.
-      # @param batch_size [Integer] The size of the mini batches.
-      # @param probability [Boolean] The flag indicating whether to perform probability estimation.
-      # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
-      #   If nil is given, Nadam is used.
-      # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0,
-                     max_iter: 1000, batch_size: 20, probability: false, optimizer: nil, random_seed: nil)
-        check_params_float(reg_param: reg_param, bias_scale: bias_scale)
-        check_params_integer(max_iter: max_iter, batch_size: batch_size)
-        check_params_boolean(fit_bias: fit_bias, probability: probability)
-        check_params_type_or_nil(Integer, random_seed: random_seed)
-        check_params_positive(reg_param: reg_param, bias_scale: bias_scale, max_iter: max_iter, batch_size: batch_size)
-        super(reg_param: reg_param, fit_bias: fit_bias, bias_scale: bias_scale,
-              max_iter: max_iter, batch_size: batch_size, optimizer: optimizer, random_seed: random_seed)
-        @params[:probability] = probability
-        @prob_param = nil
-        @classes = nil
-      end
-
-      # Fit the model with given training data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
-      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
-      # @return [SVC] The learned classifier itself.
-      def fit(x, y)
-        check_sample_array(x)
-        check_label_array(y)
-        check_sample_label_size(x, y)
-
-        @classes = Numo::Int32[*y.to_a.uniq.sort]
-        n_classes = @classes.size
-        n_features = x.shape[1]
-
-        if n_classes > 2
-          @weight_vec = Numo::DFloat.zeros(n_classes, n_features)
-          @bias_term = Numo::DFloat.zeros(n_classes)
-          @prob_param = Numo::DFloat.zeros(n_classes, 2)
-          n_classes.times do |n|
-            bin_y = Numo::Int32.cast(y.eq(@classes[n])) * 2 - 1
-            @weight_vec[n, true], @bias_term[n] = partial_fit(x, bin_y)
-            @prob_param[n, true] = if @params[:probability]
-                                     SVMKit::ProbabilisticOutput.fit_sigmoid(x.dot(@weight_vec[n, true].transpose) + @bias_term[n], bin_y)
-                                   else
-                                     Numo::DFloat[1, 0]
-                                   end
-          end
-        else
-          negative_label = y.to_a.uniq.min
-          bin_y = Numo::Int32.cast(y.ne(negative_label)) * 2 - 1
-          @weight_vec, @bias_term = partial_fit(x, bin_y)
-          @prob_param = if @params[:probability]
-                          SVMKit::ProbabilisticOutput.fit_sigmoid(x.dot(@weight_vec.transpose) + @bias_term, bin_y)
-                        else
-                          Numo::DFloat[1, 0]
-                        end
-        end
-
-        self
-      end
-
-      # Calculate confidence scores for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
-      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence score per sample.
-      def decision_function(x)
-        check_sample_array(x)
-        x.dot(@weight_vec.transpose) + @bias_term
-      end
-
-      # Predict class labels for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
-      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
-      def predict(x)
-        check_sample_array(x)
-
-        return Numo::Int32.cast(decision_function(x).ge(0.0)) * 2 - 1 if @classes.size <= 2
-
-        n_samples, = x.shape
-        decision_values = decision_function(x)
-        Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })
-      end
-
-      # Predict probability for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
-      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
-      def predict_proba(x)
-        check_sample_array(x)
-
-        if @classes.size > 2
-          probs = 1.0 / (Numo::NMath.exp(@prob_param[true, 0] * decision_function(x) + @prob_param[true, 1]) + 1.0)
-          return (probs.transpose / probs.sum(axis: 1)).transpose
-        end
-
-        n_samples, = x.shape
-        probs = Numo::DFloat.zeros(n_samples, 2)
-        probs[true, 1] = 1.0 / (Numo::NMath.exp(@prob_param[0] * decision_function(x) + @prob_param[1]) + 1.0)
-        probs[true, 0] = 1.0 - probs[true, 1]
-        probs
-      end
-
-      # Dump marshal data.
-      # @return [Hash] The marshal data about SVC.
-      def marshal_dump
-        { params: @params,
-          weight_vec: @weight_vec,
-          bias_term: @bias_term,
-          prob_param: @prob_param,
-          classes: @classes,
-          rng: @rng }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @weight_vec = obj[:weight_vec]
-        @bias_term = obj[:bias_term]
-        @prob_param = obj[:prob_param]
-        @classes = obj[:classes]
-        @rng = obj[:rng]
-        nil
-      end
-
-      private
-
-      def calc_loss_gradient(x, y, weight)
-        target_ids = (x.dot(weight) * y).lt(1.0).where
-        grad = Numo::DFloat.zeros(@params[:batch_size])
-        grad[target_ids] = -y[target_ids]
-        grad
-      end
-    end
-  end
-end

data/lib/svmkit/linear_model/svr.rb

@@ -1,123 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/linear_model/sgd_linear_estimator'
-require 'svmkit/base/regressor'
-
-module SVMKit
-  module LinearModel
-    # SVR is a class that implements Support Vector Regressor
-    # with mini-batch stochastic gradient descent optimization.
-    #
-    # @example
-    #   estimator =
-    #     SVMKit::LinearModel::SVR.new(reg_param: 1.0, epsilon: 0.1, max_iter: 1000, batch_size: 20, random_seed: 1)
-    #   estimator.fit(training_samples, traininig_target_values)
-    #   results = estimator.predict(testing_samples)
-    #
-    # *Reference*
-    # 1. S. Shalev-Shwartz and Y. Singer, "Pegasos: Primal Estimated sub-GrAdient SOlver for SVM," Proc. ICML'07, pp. 807--814, 2007.
-    class SVR < SGDLinearEstimator
-      include Base::Regressor
-      include Validation
-
-      # Return the weight vector for SVR.
-      # @return [Numo::DFloat] (shape: [n_outputs, n_features])
-      attr_reader :weight_vec
-
-      # Return the bias term (a.k.a. intercept) for SVR.
-      # @return [Numo::DFloat] (shape: [n_outputs])
-      attr_reader :bias_term
-
-      # Return the random generator for performing random sampling.
-      # @return [Random]
-      attr_reader :rng
-
-      # Create a new regressor with Support Vector Machine by the SGD optimization.
-      #
-      # @param reg_param [Float] The regularization parameter.
-      # @param fit_bias [Boolean] The flag indicating whether to fit the bias term.
-      # @param bias_scale [Float] The scale of the bias term.
-      # @param epsilon [Float] The margin of tolerance.
-      # @param max_iter [Integer] The maximum number of iterations.
-      # @param batch_size [Integer] The size of the mini batches.
-      # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
-      #   If nil is given, Nadam is used.
-      # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(reg_param: 1.0, fit_bias: false, bias_scale: 1.0, epsilon: 0.1,
-                     max_iter: 1000, batch_size: 20, optimizer: nil, random_seed: nil)
-        check_params_float(reg_param: reg_param, bias_scale: bias_scale, epsilon: epsilon)
-        check_params_integer(max_iter: max_iter, batch_size: batch_size)
-        check_params_boolean(fit_bias: fit_bias)
-        check_params_type_or_nil(Integer, random_seed: random_seed)
-        check_params_positive(reg_param: reg_param, bias_scale: bias_scale, epsilon: epsilon,
-                              max_iter: max_iter, batch_size: batch_size)
-        super(reg_param: reg_param, fit_bias: fit_bias, bias_scale: bias_scale,
-              max_iter: max_iter, batch_size: batch_size, optimizer: optimizer, random_seed: random_seed)
-        @params[:epsilon] = epsilon
-      end
-
-      # Fit the model with given training data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
-      # @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values to be used for fitting the model.
-      # @return [SVR] The learned regressor itself.
-      def fit(x, y)
-        check_sample_array(x)
-        check_tvalue_array(y)
-        check_sample_tvalue_size(x, y)
-
-        n_outputs = y.shape[1].nil? ? 1 : y.shape[1]
-        n_features = x.shape[1]
-
-        if n_outputs > 1
-          @weight_vec = Numo::DFloat.zeros(n_outputs, n_features)
-          @bias_term = Numo::DFloat.zeros(n_outputs)
-          n_outputs.times { |n| @weight_vec[n, true], @bias_term[n] = partial_fit(x, y[true, n]) }
-        else
-          @weight_vec, @bias_term = partial_fit(x, y)
-        end
-
-        self
-      end
-
-      # Predict values for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
-      # @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted values per sample.
-      def predict(x)
-        check_sample_array(x)
-        x.dot(@weight_vec.transpose) + @bias_term
-      end
-
-      # Dump marshal data.
-      # @return [Hash] The marshal data about SVR.
-      def marshal_dump
-        { params: @params,
-          weight_vec: @weight_vec,
-          bias_term: @bias_term,
-          rng: @rng }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @weight_vec = obj[:weight_vec]
-        @bias_term = obj[:bias_term]
-        @rng = obj[:rng]
-        nil
-      end
-
-      private
-
-      def calc_loss_gradient(x, y, weight)
-        z = x.dot(weight)
-        grad = Numo::DFloat.zeros(@params[:batch_size])
-        grad[(z - y).gt(@params[:epsilon]).where] = 1
-        grad[(y - z).gt(@params[:epsilon]).where] = -1
-        grad
-      end
-    end
-  end
-end

data/lib/svmkit/model_selection/cross_validation.rb

@@ -1,121 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
-require 'svmkit/base/classifier'
-require 'svmkit/base/regressor'
-require 'svmkit/base/splitter'
-require 'svmkit/base/evaluator'
-require 'svmkit/evaluation_measure/log_loss'
-
-module SVMKit
-  # This module consists of the classes for model validation techniques.
-  module ModelSelection
-    # CrossValidation is a class that evaluates a given classifier with cross-validation method.
-    #
-    # @example
-    #   svc = SVMKit::LinearModel::SVC.new
-    #   kf = SVMKit::ModelSelection::StratifiedKFold.new(n_splits: 5)
-    #   cv = SVMKit::ModelSelection::CrossValidation.new(estimator: svc, splitter: kf)
-    #   report = cv.perform(samples, lables)
-    #   mean_test_score = report[:test_score].inject(:+) / kf.n_splits
-    #
-    class CrossValidation
-      # Return the classifier of which performance is evaluated.
-      # @return [Classifier]
-      attr_reader :estimator
-
-      # Return the splitter that divides dataset.
-      # @return [Splitter]
-      attr_reader :splitter
-
-      # Return the evaluator that calculates score.
-      # @return [Evaluator]
-      attr_reader :evaluator
-
-      # Return the flag indicating whether to caculate the score of training dataset.
-      # @return [Boolean]
-      attr_reader :return_train_score
-
-      # Create a new evaluator with cross-validation method.
-      #
-      # @param estimator [Classifier] The classifier of which performance is evaluated.
-      # @param splitter [Splitter] The splitter that divides dataset to training and testing dataset.
-      # @param evaluator [Evaluator] The evaluator that calculates score of estimator results.
-      # @param return_train_score [Boolean] The flag indicating whether to calculate the score of training dataset.
-      def initialize(estimator: nil, splitter: nil, evaluator: nil, return_train_score: false)
-        SVMKit::Validation.check_params_type(SVMKit::Base::BaseEstimator, estimator: estimator)
-        SVMKit::Validation.check_params_type(SVMKit::Base::Splitter, splitter: splitter)
-        SVMKit::Validation.check_params_type_or_nil(SVMKit::Base::Evaluator, evaluator: evaluator)
-        SVMKit::Validation.check_params_boolean(return_train_score: return_train_score)
-        @estimator = estimator
-        @splitter = splitter
-        @evaluator = evaluator
-        @return_train_score = return_train_score
-      end
-
-      # Perform the evalution of given classifier with cross-validation method.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features])
-      #   The dataset to be used to evaluate the estimator.
-      # @param y [Numo::Int32 / Numo::DFloat] (shape: [n_samples] / [n_samples, n_outputs])
-      #   The labels to be used to evaluate the classifier / The target values to be used to evaluate the regressor.
-      # @return [Hash] The report summarizing the results of cross-validation.
-      #   * :fit_time (Array<Float>) The calculation times of fitting the estimator for each split.
-      #   * :test_score (Array<Float>) The scores of testing dataset for each split.
-      #   * :train_score (Array<Float>) The scores of training dataset for each split. This option is nil if
-      #     the return_train_score is false.
-      def perform(x, y)
-        SVMKit::Validation.check_sample_array(x)
-        if @estimator.is_a?(SVMKit::Base::Classifier)
-          SVMKit::Validation.check_label_array(y)
-          SVMKit::Validation.check_sample_label_size(x, y)
-        end
-        if @estimator.is_a?(SVMKit::Base::Regressor)
-          SVMKit::Validation.check_tvalue_array(y)
-          SVMKit::Validation.check_sample_tvalue_size(x, y)
-        end
-        # Initialize the report of cross validation.
-        report = { test_score: [], train_score: nil, fit_time: [] }
-        report[:train_score] = [] if @return_train_score
-        # Evaluate the estimator on each split.
-        @splitter.split(x, y).each do |train_ids, test_ids|
-          # Split dataset into training and testing dataset.
-          feature_ids = !kernel_machine? || train_ids
-          train_x = x[train_ids, feature_ids]
-          train_y = y.shape[1].nil? ? y[train_ids] : y[train_ids, true]
-          test_x = x[test_ids, feature_ids]
-          test_y = y.shape[1].nil? ? y[test_ids] : y[test_ids, true]
-          # Fit the estimator.
-          start_time = Time.now.to_i
-          @estimator.fit(train_x, train_y)
-          # Calculate scores and prepare the report.
-          report[:fit_time].push(Time.now.to_i - start_time)
-          if @evaluator.nil?
-            report[:test_score].push(@estimator.score(test_x, test_y))
-            report[:train_score].push(@estimator.score(train_x, train_y)) if @return_train_score
-          elsif log_loss?
-            report[:test_score].push(@evaluator.score(test_y, @estimator.predict_proba(test_x)))
-            report[:train_score].push(@evaluator.score(train_y, @estimator.predict_proba(train_x))) if @return_train_score
-          else
-            report[:test_score].push(@evaluator.score(test_y, @estimator.predict(test_x)))
-            report[:train_score].push(@evaluator.score(train_y, @estimator.predict(train_x))) if @return_train_score
-          end
-        end
-        report
-      end
-
-      private
-
-      def kernel_machine?
-        class_name = @estimator.class.to_s
-        class_name = @estimator.params[:estimator].class.to_s if class_name.include?('Multiclass')
-        class_name.include?('KernelMachine')
-      end
-
-      def log_loss?
-        @evaluator.is_a?(SVMKit::EvaluationMeasure::LogLoss)
-      end
-    end
-  end
-end