svmkit 0.2.8 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: bbc648db53b4285bd15ae00e2464c4376d027893
-  data.tar.gz: 28d9db47ae3053031f1643329cf02cd4e7d9c135
+SHA256:
+  metadata.gz: 533508a3afd82d2bae3ddea3a5669f6d389688155d44649fd3eafaaff8207e0f
+  data.tar.gz: 43ff09b3bab72b68bc7a6b3740902be64508496337a4cde61057d33b91d0f349
 SHA512:
-  metadata.gz: c504f010a70fc7a31afa4471096092adac9ff44de979d42d7277c63d737d517981e2ac6d7481ad42dc1c864f2a7756d39cd1d7697d2d4b1bca150d0a4eca3b8e
-  data.tar.gz: 8d89dc525ed37626a2d97e6fe3bebdacd4ec2945df285d8275cd0bd5df62c5ebc897dca67b91157df0bfc460fa987ed4098654ce6d164cb05ec46de4c6fe27af
+  metadata.gz: e1c1bed8269d3c768d75bd8a5e731b5d2da689ef7a235a70c5ea87090aac79889c9fe0a004eca73c3015aae42d068f44b2b1e3a61a03b641607b2909441513b6
+  data.tar.gz: 80a18ca4ec7eb2740148829024f0625c835f24b771bb321168d0cc3233d8e152257b5515355d99a968dc25a670f9a69f3e30b42bf190757206a64bbcd2babcd6

data/.gitignore

@@ -10,3 +10,7 @@
 
 # rspec failure tracking
 .rspec_status
+
+*.swp
+.DS_Store
+.ruby-version

data/.rubocop.yml

@@ -9,7 +9,7 @@ Documentation:
   Enabled: false
 
 Metrics/LineLength:
-  Max: 120
+  Max: 140
   IgnoredPatterns: ['(\A|\s)#']
 
 Metrics/ModuleLength:
@@ -21,6 +21,9 @@ Metrics/ClassLength:
 Metrics/MethodLength:
   Max: 40
 
+Metrics/AbcSize:
+  Max: 60
+
 Metrics/BlockLength:
   Exclude:
     - 'spec/**/*'
@@ -30,3 +33,9 @@ ParameterLists:
 
 Security/MarshalLoad:
   Enabled: false
+
+Naming/UncommunicativeMethodParamName:
+  Enabled: false
+
+Style/FormatStringToken:
+  Enabled: false

data/.rubocop_todo.yml

@@ -1,18 +1,59 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2018-02-04 11:34:28 +0900 using RuboCop version 0.52.1.
+# on 2018-04-14 20:44:19 +0900 using RuboCop version 0.54.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 5
-Metrics/AbcSize:
-  Max: 70
+# Offense count: 1
+# Configuration parameters: Include.
+# Include: **/*.gemspec
+Gemspec/RequiredRubyVersion:
+  Exclude:
+    - 'svmkit.gemspec'
 
-# Offense count: 4
-# Configuration parameters: .
-# SupportedStyles: annotated, template, unannotated
-Style/FormatStringToken:
-  EnforcedStyle: unannotated
-  Enabled: false
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: auto_detection, squiggly, active_support, powerpack, unindent
+Layout/IndentHeredoc:
+  Exclude:
+    - 'svmkit.gemspec'
+
+# Offense count: 1
+# Configuration parameters: CountComments, ExcludedMethods.
+Metrics/BlockLength:
+  Max: 30
+
+# Offense count: 1
+Metrics/CyclomaticComplexity:
+  Max: 12
+
+# Offense count: 1
+Metrics/PerceivedComplexity:
+  Max: 12
+
+# Offense count: 1
+# Cop supports --auto-correct.
+Style/Encoding:
+  Exclude:
+    - 'svmkit.gemspec'
+
+# Offense count: 1
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, UseHashRocketsWithSymbolValues, PreferHashRocketsForNonAlnumEndingSymbols.
+# SupportedStyles: ruby19, hash_rockets, no_mixed_keys, ruby19_no_mixed_keys
+Style/HashSyntax:
+  Exclude:
+    - 'Rakefile'
+
+# Offense count: 6
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
+# SupportedStyles: single_quotes, double_quotes
+Style/StringLiterals:
+  Exclude:
+    - 'Gemfile'
+    - 'Rakefile'
+    - 'bin/console'

data/Gemfile

@@ -1,6 +1,6 @@
 source "https://rubygems.org"
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
 # Specify your gem's dependencies in svmkit.gemspec
 gemspec

data/HISTORY.md

@@ -1,59 +1,69 @@
+# 0.2.9
+- Add predict_proba method to SVC and KernelSVC.
+- Add class for evaluating logarithmic loss.
+- Add classes for Label- and One-Hot- encoding.
+- Add some validator.
+- Fix bug on training data score calculation of cross validation.
+- Fix fit method of SVC for performance.
+- Fix criterion calculation on Decision Tree for performance.
+- Fix data structure of Decision Tree for performance.
+
 # 0.2.8
-- Fixed bug on gradient calculation of Logistic Regression.
-- Fixed to change accessor of params of estimators to read only.
-- Added parameter validation.
+- Fix bug on gradient calculation of Logistic Regression.
+- Fix to change accessor of params of estimators to read only.
+- Add parameter validation.
 
 # 0.2.7
-- Fixed to support multiclass classifiction into LinearSVC, LogisticRegression, KernelSVC, and FactorizationMachineClassifier
+- Fix to support multiclass classifiction into LinearSVC, LogisticRegression, KernelSVC, and FactorizationMachineClassifier
 
 # 0.2.6
-- Added class for Decision Tree classifier.
-- Added class for Random Forest classifier.
-- Fixed to use frozen string literal.
-- Refactored marshal dump method on some classes.
-- Introduced Coveralls to confirm test coverage.
+- Add class for Decision Tree classifier.
+- Add class for Random Forest classifier.
+- Fix to use frozen string literal.
+- Refactor marshal dump method on some classes.
+- Introduce Coveralls to confirm test coverage.
 
 # 0.2.5
-- Added classes for Naive Bayes classifier.
-- Fixed decision function method on Logistic Regression class.
-- Fixed method visibility on RBF kernel approximation class.
+- Add classes for Naive Bayes classifier.
+- Fix decision function method on Logistic Regression class.
+- Fix method visibility on RBF kernel approximation class.
 
 # 0.2.4
-- Added class for Factorization Machine classifier.
-- Added classes for evaluation measures.
-- Fixed the method for prediction of class probability in Logistic Regression.
+- Add class for Factorization Machine classifier.
+- Add classes for evaluation measures.
+- Fix the method for prediction of class probability in Logistic Regression.
 
 # 0.2.3
-- Added class for cross validation.
-- Added specs for base modules.
-- Fixed validation of the number of splits when a negative label is given.
+- Add class for cross validation.
+- Add specs for base modules.
+- Fix validation of the number of splits when a negative label is given.
 
 # 0.2.2
-- Added data splitter classes for K-fold cross validation.
+- Add data splitter classes for K-fold cross validation.
 
 # 0.2.1
-- Added class for K-nearest neighbors classifier.
+- Add class for K-nearest neighbors classifier.
 
 # 0.2.0
 - Migrated the linear algebra library to Numo::NArray.
-- Added module for loading and saving libsvm format file.
+- Add module for loading and saving libsvm format file.
 
 # 0.1.3
-- Added class for Kernel Support Vector Machine with Pegasos algorithm.
-- Added module for calculating pairwise kernel fuctions and euclidean distances.
+- Add class for Kernel Support Vector Machine with Pegasos algorithm.
+- Add module for calculating pairwise kernel fuctions and euclidean distances.
 
 # 0.1.2
-- Added the function learning a model with bias term to the PegasosSVC and LogisticRegression classes.
-- Rewrited the document with yard notation.
+- Add the function learning a model with bias term to the PegasosSVC and LogisticRegression classes.
+- Rewrite the document with yard notation.
 
 # 0.1.1
-- Added class for Logistic Regression with SGD optimization.
-- Fixed some mistakes on the document.
+- Add class for Logistic Regression with SGD optimization.
+- Fix some mistakes on the document.
 
 # 0.1.0
-- Added basic classes.
-- Added an utility module.
-- Added class for RBF kernel approximation.
-- Added class for Support Vector Machine with Pegasos alogrithm.
-- Added class that performs mutlclass classification with one-vs.-rest strategy.
-- Added classes for preprocessing such as min-max scaling, standardization, and L2 normalization.
+- Add basic classes.
+- Add an utility module.
+- Add class for RBF kernel approximation.
+- Add class for Support Vector Machine with Pegasos alogrithm.
+- Add class that performs mutlclass classification with one-vs.-rest strategy.
+- Add classes for preprocessing such as min-max scaling, standardization, and L2 normalization.

data/lib/svmkit.rb

@@ -6,6 +6,7 @@ require 'svmkit/version'
 require 'svmkit/validation'
 require 'svmkit/pairwise_metric'
 require 'svmkit/dataset'
+require 'svmkit/probabilistic_output'
 require 'svmkit/base/base_estimator'
 require 'svmkit/base/classifier'
 require 'svmkit/base/transformer'
@@ -24,6 +25,8 @@ require 'svmkit/ensemble/random_forest_classifier'
 require 'svmkit/preprocessing/l2_normalizer'
 require 'svmkit/preprocessing/min_max_scaler'
 require 'svmkit/preprocessing/standard_scaler'
+require 'svmkit/preprocessing/label_encoder'
+require 'svmkit/preprocessing/one_hot_encoder'
 require 'svmkit/model_selection/k_fold'
 require 'svmkit/model_selection/stratified_k_fold'
 require 'svmkit/model_selection/cross_validation'
@@ -31,3 +34,4 @@ require 'svmkit/evaluation_measure/accuracy'
 require 'svmkit/evaluation_measure/precision'
 require 'svmkit/evaluation_measure/recall'
 require 'svmkit/evaluation_measure/f_score'
+require 'svmkit/evaluation_measure/log_loss'

data/lib/svmkit/base/classifier.rb

@@ -22,6 +22,7 @@ module SVMKit
       def score(x, y)
         SVMKit::Validation.check_sample_array(x)
         SVMKit::Validation.check_label_array(y)
+        SVMKit::Validation.check_sample_label_size(x, y)
         evaluator = SVMKit::EvaluationMeasure::Accuracy.new
         evaluator.score(y, predict(x))
       end

data/lib/svmkit/ensemble/random_forest_classifier.rb

@@ -51,10 +51,12 @@ module SVMKit
       def initialize(n_estimators: 10, criterion: 'gini', max_depth: nil, max_leaf_nodes: nil, min_samples_leaf: 1,
                      max_features: nil, random_seed: nil)
         SVMKit::Validation.check_params_type_or_nil(Integer, max_depth: max_depth, max_leaf_nodes: max_leaf_nodes,
-                                                    max_features: max_features, random_seed: random_seed)
+                                                             max_features: max_features, random_seed: random_seed)
         SVMKit::Validation.check_params_integer(n_estimators: n_estimators, min_samples_leaf: min_samples_leaf)
         SVMKit::Validation.check_params_string(criterion: criterion)
-
+        SVMKit::Validation.check_params_positive(n_estimators: n_estimators, max_depth: max_depth,
+                                                 max_leaf_nodes: max_leaf_nodes, min_samples_leaf: min_samples_leaf,
+                                                 max_features: max_features)
         @params = {}
         @params[:n_estimators] = n_estimators
         @params[:criterion] = criterion
@@ -78,6 +80,7 @@ module SVMKit
       def fit(x, y)
         SVMKit::Validation.check_sample_array(x)
         SVMKit::Validation.check_label_array(y)
+        SVMKit::Validation.check_sample_label_size(x, y)
         # Initialize some variables.
         n_samples, n_features = x.shape
         @params[:max_features] = n_features unless @params[:max_features].is_a?(Integer)

data/lib/svmkit/evaluation_measure/log_loss.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'svmkit/base/evaluator'
+
+module SVMKit
+  module EvaluationMeasure
+    # LogLoss is a class that calculates the logarithmic loss of predicted class probability.
+    #
+    # @example
+    #   evaluator = SVMKit::EvaluationMeasure::LogLoss.new
+    #   puts evaluator.score(ground_truth, predicted)
+    class LogLoss
+      include Base::Evaluator
+
+      # Claculate mean logarithmic loss.
+      # If both y_true and y_pred are array (both shapes are [n_samples]), this method calculates
+      # mean logarithmic loss for binary classification.
+      #
+      # @param y_true [Numo::Int32] (shape: [n_samples]) Ground truth labels.
+      # @param y_pred [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted class probability.
+      # @param eps [Float] A small value close to zero to avoid outputting infinity in logarithmic calcuation.
+      # @return [Float] mean logarithmic loss
+      def score(y_true, y_pred, eps = 1e-15)
+        SVMKit::Validation.check_params_type(Numo::Int32, y_true: y_true)
+        SVMKit::Validation.check_params_type(Numo::DFloat, y_pred: y_pred)
+
+        n_samples, n_classes = y_pred.shape
+        clipped_p = y_pred.clip(eps, 1 - eps)
+
+        log_loss = if n_classes.nil?
+                     negative_label = y_true.to_a.uniq.sort.first
+                     bin_y_true = Numo::DFloat.cast(y_true.ne(negative_label))
+                     -(bin_y_true * Numo::NMath.log(clipped_p) + (1 - bin_y_true) * Numo::NMath.log(1 - clipped_p))
+                   else
+                     encoder = SVMKit::Preprocessing::OneHotEncoder.new
+                     encoded_y_true = encoder.fit_transform(y_true)
+                     clipped_p /= clipped_p.sum(1).expand_dims(1)
+                     -(encoded_y_true * Numo::NMath.log(clipped_p)).sum(1)
+                   end
+        log_loss.sum / n_samples
+      end
+    end
+  end
+end

data/lib/svmkit/kernel_approximation/rbf.rb

@@ -40,7 +40,7 @@ module SVMKit
         SVMKit::Validation.check_params_float(gamma: gamma)
         SVMKit::Validation.check_params_integer(n_components: n_components)
         SVMKit::Validation.check_params_type_or_nil(Integer, random_seed: random_seed)
-
+        SVMKit::Validation.check_params_positive(gamma: gamma, n_components: n_components)
         @params = {}
         @params[:gamma] = gamma
         @params[:n_components] = n_components

data/lib/svmkit/kernel_machine/kernel_svc.rb

@@ -40,18 +40,22 @@ module SVMKit
       #
       # @param reg_param [Float] The regularization parameter.
       # @param max_iter [Integer] The maximum number of iterations.
+      # @param probability [Boolean] The flag indicating whether to perform probability estimation.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(reg_param: 1.0, max_iter: 1000, random_seed: nil)
+      def initialize(reg_param: 1.0, max_iter: 1000, probability: false, random_seed: nil)
         SVMKit::Validation.check_params_float(reg_param: reg_param)
         SVMKit::Validation.check_params_integer(max_iter: max_iter)
+        SVMKit::Validation.check_params_boolean(probability: probability)
         SVMKit::Validation.check_params_type_or_nil(Integer, random_seed: random_seed)
-
+        SVMKit::Validation.check_params_positive(reg_param: reg_param, max_iter: max_iter)
         @params = {}
         @params[:reg_param] = reg_param
         @params[:max_iter] = max_iter
+        @params[:probability] = probability
         @params[:random_seed] = random_seed
         @params[:random_seed] ||= srand
         @weight_vec = nil
+        @prob_param = nil
         @classes = nil
         @rng = Random.new(@params[:random_seed])
       end
@@ -65,6 +69,7 @@ module SVMKit
       def fit(x, y)
         SVMKit::Validation.check_sample_array(x)
         SVMKit::Validation.check_label_array(y)
+        SVMKit::Validation.check_sample_label_size(x, y)
 
         @classes = Numo::Int32[*y.to_a.uniq.sort]
         n_classes = @classes.size
@@ -72,14 +77,25 @@ module SVMKit
 
         if n_classes > 2
           @weight_vec = Numo::DFloat.zeros(n_classes, n_features)
+          @prob_param = Numo::DFloat.zeros(n_classes, 2)
           n_classes.times do |n|
             bin_y = Numo::Int32.cast(y.eq(@classes[n])) * 2 - 1
             @weight_vec[n, true] = binary_fit(x, bin_y)
+            @prob_param[n, true] = if @params[:probability]
+                                     SVMKit::ProbabilisticOutput.fit_sigmoid(x.dot(@weight_vec[n, true].transpose), bin_y)
+                                   else
+                                     Numo::DFloat[1, 0]
+                                   end
           end
         else
           negative_label = y.to_a.uniq.sort.first
           bin_y = Numo::Int32.cast(y.ne(negative_label)) * 2 - 1
           @weight_vec = binary_fit(x, bin_y)
+          @prob_param = if @params[:probability]
+                          SVMKit::ProbabilisticOutput.fit_sigmoid(x.dot(@weight_vec.transpose), bin_y)
+                        else
+                          Numo::DFloat[1, 0]
+                        end
         end
 
         self
@@ -111,11 +127,32 @@ module SVMKit
         Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })
       end
 
+      # Predict probability for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_testing_samples, n_training_samples])
+      #     The kernel matrix between testing samples and training samples to predict the labels.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
+      def predict_proba(x)
+        SVMKit::Validation.check_sample_array(x)
+
+        if @classes.size > 2
+          probs = 1.0 / (Numo::NMath.exp(@prob_param[true, 0] * decision_function(x) + @prob_param[true, 1]) + 1.0)
+          return (probs.transpose / probs.sum(axis: 1)).transpose
+        end
+
+        n_samples, = x.shape
+        probs = Numo::DFloat.zeros(n_samples, 2)
+        probs[true, 1] = 1.0 / (Numo::NMath.exp(@prob_param[0] * decision_function(x) + @prob_param[1]) + 1.0)
+        probs[true, 0] = 1.0 - probs[true, 1]
+        probs
+      end
+
       # Dump marshal data.
       # @return [Hash] The marshal data about KernelSVC.
       def marshal_dump
         { params: @params,
           weight_vec: @weight_vec,
+          prob_param: @prob_param,
           classes: @classes,
           rng: @rng }
       end
@@ -125,6 +162,7 @@ module SVMKit
       def marshal_load(obj)
         @params = obj[:params]
         @weight_vec = obj[:weight_vec]
+        @prob_param = obj[:prob_param]
         @classes = obj[:classes]
         @rng = obj[:rng]
         nil

data/lib/svmkit/linear_model/logistic_regression.rb

@@ -54,7 +54,8 @@ module SVMKit
         SVMKit::Validation.check_params_integer(max_iter: max_iter, batch_size: batch_size)
         SVMKit::Validation.check_params_boolean(fit_bias: fit_bias, normalize: normalize)
         SVMKit::Validation.check_params_type_or_nil(Integer, random_seed: random_seed)
-
+        SVMKit::Validation.check_params_positive(reg_param: reg_param, bias_scale: bias_scale, max_iter: max_iter,
+                                                 batch_size: batch_size)
         @params = {}
         @params[:reg_param] = reg_param
         @params[:fit_bias] = fit_bias
@@ -78,6 +79,7 @@ module SVMKit
       def fit(x, y)
         SVMKit::Validation.check_sample_array(x)
         SVMKit::Validation.check_label_array(y)
+        SVMKit::Validation.check_sample_label_size(x, y)
 
         @classes = Numo::Int32[*y.to_a.uniq.sort]
         n_classes = @classes.size