rumale 0.22.0 → 0.22.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e2f68b3182ada73537901e7bc74bddd100aff75264f9147c88d8240fb624e29
-  data.tar.gz: e2639a55fc84d1399b925f65b3a56b38f2ae3150dd15ab8556120af28d408cae
+  metadata.gz: 475798da3815141b5337bc7803eca26978bbc98c36a2be4d681bc63f778f5840
+  data.tar.gz: 71841127edccbeea2b30c4bd8a744735933c1fcf8c7d6afa507cd2d361c8b5c8
 SHA512:
-  metadata.gz: 91ffcbade578bbb9c6a5d87a54ebd89a2b5990eb70835e7a5549afe78541dbfeafe3af50833725bee751fa89c059484970e5add7ebf8adee3e25bc000fbe3778
-  data.tar.gz: 2ee2b1448a486581ef98561f65bc3446b2e161c89a3a12bd6cd78867350e26151bc0b350bd431902d21f6979493ab2d01a6ee81b55c1099f631aa84c84a704e6
+  metadata.gz: 8c90eaffa3847e3cc4f31c58e3d74f66a86e2cf0bd1c6e5aa386f9519de3984ac1c605187119e0ec01585c82cf8c06e4f4aa2f19c7b40883b784e834e2e801d8
+  data.tar.gz: 336b1afcc35e52e1c13ced74f527d54c994ff66509cf9b8b2f81dce62692078964a453df790a5bcaf8ea2bd156277d719a33958de2f57aa64d97beeafdc48d01

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 0.22.1
+- Add transfomer class for MLKR, that implements Metric Learning for Kernel Regression.
+- Refactor NeighbourhoodComponentAnalysis.
+- Update API documentation.
+
 # 0.22.0
 ## Breaking change
 - Add lbfgsb.rb gem to runtime dependencies. Rumale uses lbfgsb gem for optimization.

data/README.md

@@ -113,10 +113,10 @@ require 'rumale'
 samples, labels = Rumale::Dataset.load_libsvm_file('pendigits')
 
 # Define the estimator to be evaluated.
-lr = Rumale::LinearModel::LogisticRegression.new(learning_rate: 0.00001, reg_param: 0.0001, random_seed: 1)
+lr = Rumale::LinearModel::LogisticRegression.new
 
 # Define the evaluation measure, splitting strategy, and cross validation.
-ev = Rumale::EvaluationMeasure::LogLoss.new
+ev = Rumale::EvaluationMeasure::Accuracy.new
 kf = Rumale::ModelSelection::StratifiedKFold.new(n_splits: 5, shuffle: true, random_seed: 1)
 cv = Rumale::ModelSelection::CrossValidation.new(estimator: lr, splitter: kf, evaluator: ev)
 
@@ -124,15 +124,15 @@ cv = Rumale::ModelSelection::CrossValidation.new(estimator: lr, splitter: kf, ev
 report = cv.perform(samples, labels)
 
 # Output result.
-mean_logloss = report[:test_score].inject(:+) / kf.n_splits
-puts("5-CV mean log-loss: %.3f" % mean_logloss)
+mean_accuracy = report[:test_score].sum / kf.n_splits
+puts "5-CV mean accuracy: %.1f%%" % (100.0 * mean_accuracy)
 ```
 
 Execution of the above scripts result in the following.
 
 ```bash
 $ ruby cross_validation.rb
-5-CV mean log-loss: 0.355
+5-CV mean accuracy: 95.4%
 ```
 
 ### Example 3. Pipeline
@@ -143,10 +143,10 @@ require 'rumale'
 # Load dataset.
 samples, labels = Rumale::Dataset.load_libsvm_file('pendigits')
 
-# Construct pipeline with kernel approximation and SVC.
-rbf = Rumale::KernelApproximation::RBF.new(gamma: 0.0001, n_components: 800, random_seed: 1)
-svc = Rumale::LinearModel::SVC.new(reg_param: 0.0001, random_seed: 1)
-pipeline = Rumale::Pipeline::Pipeline.new(steps: { trns: rbf, clsf: svc })
+# Construct pipeline with kernel approximation and LogisticRegression.
+rbf = Rumale::KernelApproximation::RBF.new(gamma: 1e-4, n_components: 800, random_seed: 1)
+lr = Rumale::LinearModel::LogisticRegression.new(reg_param: 1e-3)
+pipeline = Rumale::Pipeline::Pipeline.new(steps: { trns: rbf, clsf: lr })
 
 # Define the splitting strategy and cross validation.
 kf = Rumale::ModelSelection::StratifiedKFold.new(n_splits: 5, shuffle: true, random_seed: 1)
@@ -156,7 +156,7 @@ cv = Rumale::ModelSelection::CrossValidation.new(estimator: pipeline, splitter:
 report = cv.perform(samples, labels)
 
 # Output result.
-mean_accuracy = report[:test_score].inject(:+) / kf.n_splits
+mean_accuracy = report[:test_score].sum / kf.n_splits
 puts("5-CV mean accuracy: %.1f %%" % (mean_accuracy * 100.0))
 ```
 

data/lib/rumale.rb

@@ -77,6 +77,7 @@ require 'rumale/manifold/tsne'
 require 'rumale/manifold/mds'
 require 'rumale/metric_learning/fisher_discriminant_analysis'
 require 'rumale/metric_learning/neighbourhood_component_analysis'
+require 'rumale/metric_learning/mlkr'
 require 'rumale/neural_network/adam'
 require 'rumale/neural_network/base_mlp'
 require 'rumale/neural_network/mlp_regressor'

data/lib/rumale/kernel_machine/kernel_svc.rb

@@ -11,9 +11,10 @@ module Rumale
     # with stochastic gradient descent (SGD) optimization.
     # For multiclass classification problem, it uses one-vs-the-rest strategy.
     #
-    # Rumale::SVM provides kernel support vector classifier based on LIBSVM.
-    # If you prefer execution speed, you should use Rumale::SVM::SVC.
-    # https://github.com/yoshoku/rumale-svm
+    # @note
+    #   Rumale::SVM provides kernel support vector classifier based on LIBSVM.
+    #   If you prefer execution speed, you should use Rumale::SVM::SVC.
+    #   https://github.com/yoshoku/rumale-svm
     #
     # @example
     #   training_kernel_matrix = Rumale::PairwiseMetric::rbf_kernel(training_samples)

data/lib/rumale/linear_model/svc.rb

@@ -11,9 +11,10 @@ module Rumale
     # with stochastic gradient descent optimization.
     # For multiclass classification problem, it uses one-vs-the-rest strategy.
     #
-    # Rumale::SVM provides linear support vector classifier based on LIBLINEAR.
-    # If you prefer execution speed, you should use Rumale::SVM::LinearSVC.
-    # https://github.com/yoshoku/rumale-svm
+    # @note
+    #   Rumale::SVM provides linear support vector classifier based on LIBLINEAR.
+    #   If you prefer execution speed, you should use Rumale::SVM::LinearSVC.
+    #   https://github.com/yoshoku/rumale-svm
     #
     # @example
     #   estimator =

data/lib/rumale/linear_model/svr.rb

@@ -8,9 +8,10 @@ module Rumale
     # SVR is a class that implements Support Vector Regressor
     # with stochastic gradient descent optimization.
     #
-    # Rumale::SVM provides linear and kernel support vector regressor based on LIBLINEAR and LIBSVM.
-    # If you prefer execution speed, you should use Rumale::SVM::LinearSVR.
-    # https://github.com/yoshoku/rumale-svm
+    # @note
+    #   Rumale::SVM provides linear and kernel support vector regressor based on LIBLINEAR and LIBSVM.
+    #   If you prefer execution speed, you should use Rumale::SVM::LinearSVR.
+    #   https://github.com/yoshoku/rumale-svm
     #
     # @example
     #   estimator =

data/lib/rumale/metric_learning/mlkr.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator'
+require 'rumale/base/transformer'
+require 'rumale/decomposition/pca'
+require 'rumale/pairwise_metric'
+require 'rumale/utils'
+require 'lbfgsb'
+
+module Rumale
+  module MetricLearning
+    # MLKR is a class that implements Metric Learning for Kernel Regression.
+    #
+    # @example
+    #   transformer = Rumale::MetricLearning::MLKR.new
+    #   transformer.fit(training_samples, traininig_target_values)
+    #   low_samples = transformer.transform(testing_samples)
+    #
+    # *Reference*
+    # - Weinberger, K. Q. and Tesauro, G., "Metric Learning for Kernel Regression," Proc. AISTATS'07, pp. 612--629, 2007.
+    class MLKR
+      include Base::BaseEstimator
+      include Base::Transformer
+
+      # Returns the metric components.
+      # @return [Numo::DFloat] (shape: [n_components, n_features])
+      attr_reader :components
+
+      # Return the number of iterations run for optimization
+      # @return [Integer]
+      attr_reader :n_iter
+
+      # Return the random generator.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new transformer with MLKR.
+      #
+      # @param n_components [Integer] The number of components.
+      # @param init [String] The initialization method for components ('random' or 'pca').
+      # @param max_iter [Integer] The maximum number of iterations.
+      # @param tol [Float] The tolerance of termination criterion.
+      #   This value is given as tol / Lbfgsb::DBL_EPSILON to the factr argument of Lbfgsb.minimize method.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
+      #   If true is given, 'iterate.dat' file is generated by lbfgsb.rb.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(n_components: nil, init: 'random', max_iter: 100, tol: 1e-6, verbose: false, random_seed: nil)
+        check_params_numeric_or_nil(n_components: n_components, random_seed: random_seed)
+        check_params_numeric(max_iter: max_iter, tol: tol)
+        check_params_string(init: init)
+        check_params_boolean(verbose: verbose)
+        @params = {}
+        @params[:n_components] = n_components
+        @params[:init] = init
+        @params[:max_iter] = max_iter
+        @params[:tol] = tol
+        @params[:verbose] = verbose
+        @params[:random_seed] = random_seed
+        @params[:random_seed] ||= srand
+        @components = nil
+        @n_iter = nil
+        @rng = Random.new(@params[:random_seed])
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::DFloat] (shape: [n_samples]) The target values to be used for fitting the model.
+      # @return [MLKR] The learned classifier itself.
+      def fit(x, y)
+        x = check_convert_sample_array(x)
+        y = check_convert_tvalue_array(y)
+        check_sample_tvalue_size(x, y)
+        n_features = x.shape[1]
+        n_components = if @params[:n_components].nil?
+                         n_features
+                       else
+                         [n_features, @params[:n_components]].min
+                       end
+        @components, @n_iter = optimize_components(x, y, n_features, n_components)
+        @prototypes = x.dot(@components.transpose)
+        @values = y
+        self
+      end
+
+      # Fit the model with training data, and then transform them with the learned model.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::DFloat] (shape: [n_samples]) The target values to be used for fitting the model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The transformed data
+      def fit_transform(x, y)
+        x = check_convert_sample_array(x)
+        y = check_convert_tvalue_array(y)
+        check_sample_tvalue_size(x, y)
+        fit(x, y).transform(x)
+      end
+
+      # Transform the given data with the learned model.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The data to be transformed with the learned model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The transformed data.
+      def transform(x)
+        x = check_convert_sample_array(x)
+        x.dot(@components.transpose)
+      end
+
+      private
+
+      def init_components(x, n_features, n_components)
+        if @params[:init] == 'pca'
+          pca = Rumale::Decomposition::PCA.new(n_components: n_components)
+          pca.fit(x).components.flatten.dup
+        else
+          Rumale::Utils.rand_normal([n_features, n_components], @rng.dup).flatten.dup
+        end
+      end
+
+      def optimize_components(x, y, n_features, n_components)
+        # initialize components.
+        comp_init = init_components(x, n_features, n_components)
+        # initialize optimization results.
+        res = {}
+        res[:x] = comp_init
+        res[:n_iter] = 0
+        # perform optimization.
+        verbose = @params[:verbose] ? 1 : -1
+        res = Lbfgsb.minimize(
+          fnc: method(:mlkr_fnc), jcb: true, x_init: comp_init, args: [x, y],
+          maxiter: @params[:max_iter], factr: @params[:tol] / Lbfgsb::DBL_EPSILON, verbose: verbose
+        )
+        # return the results.
+        n_iter = res[:n_iter]
+        comps = n_components == 1 ? res[:x].dup : res[:x].reshape(n_components, n_features)
+        [comps, n_iter]
+      end
+
+      def mlkr_fnc(w, x, y)
+        # initialize some variables.
+        n_features = x.shape[1]
+        n_components = w.size / n_features
+        # projection.
+        w = w.reshape(n_components, n_features)
+        z = x.dot(w.transpose)
+        # predict values.
+        kernel_mat = Numo::NMath.exp(-Rumale::PairwiseMetric.squared_error(z))
+        kernel_mat[kernel_mat.diag_indices] = 0.0
+        norm = kernel_mat.sum(1)
+        norm[norm.eq(0)] = 1
+        y_pred = kernel_mat.dot(y) / norm
+        # calculate loss.
+        y_diff = y_pred - y
+        loss = (y_diff**2).sum
+        # calculate gradient.
+        weight_mat = y_diff * y_diff.expand_dims(1) * kernel_mat
+        weight_mat = weight_mat.sum(0).diag - weight_mat
+        gradient = 8 * z.transpose.dot(weight_mat).dot(x)
+        [loss, gradient.flatten.dup]
+      end
+    end
+  end
+end

data/lib/rumale/metric_learning/neighbourhood_component_analysis.rb

@@ -2,6 +2,8 @@
 
 require 'rumale/base/base_estimator'
 require 'rumale/base/transformer'
+require 'rumale/utils'
+require 'rumale/pairwise_metric'
 require 'lbfgsb'
 
 module Rumale
@@ -146,10 +148,11 @@ module Rumale
         mask_mat = y.expand_dims(1).eq(y)
         masked_prob_mat = prob_mat * mask_mat
         loss = n_samples - masked_prob_mat.sum
-        weighted_prob_mat = masked_prob_mat - prob_mat * masked_prob_mat.sum(1).expand_dims(1)
-        weighted_prob_mat += weighted_prob_mat.transpose
-        weighted_prob_mat[weighted_prob_mat.diag_indices] = -weighted_prob_mat.sum(0)
-        gradient = -2 * z.transpose.dot(weighted_prob_mat).dot(x)
+        sum_probs = masked_prob_mat.sum(1)
+        weight_mat = (sum_probs.expand_dims(1) * prob_mat - masked_prob_mat)
+        weight_mat += weight_mat.transpose
+        weight_mat = weight_mat.sum(0).diag - weight_mat
+        gradient = -2 * z.transpose.dot(weight_mat).dot(x)
         [loss, gradient.flatten.dup]
       end
 

data/lib/rumale/version.rb

@@ -3,5 +3,5 @@
 # Rumale is a machine learning library in Ruby.
 module Rumale
   # The version of Rumale you are using.
-  VERSION = '0.22.0'
+  VERSION = '0.22.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rumale
 version: !ruby/object:Gem::Version
-  version: 0.22.0
+  version: 0.22.1
 platform: ruby
 authors:
 - yoshoku
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-11-22 00:00:00.000000000 Z
+date: 2020-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -145,6 +145,7 @@ files:
 - lib/rumale/manifold/mds.rb
 - lib/rumale/manifold/tsne.rb
 - lib/rumale/metric_learning/fisher_discriminant_analysis.rb
+- lib/rumale/metric_learning/mlkr.rb
 - lib/rumale/metric_learning/neighbourhood_component_analysis.rb
 - lib/rumale/model_selection/cross_validation.rb
 - lib/rumale/model_selection/function.rb