rumale 0.17.0 → 0.17.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d1071dfdccfc177ea5902e5e1b09fce084fd4b6ce403fae6797e6b93c3f826ad
-  data.tar.gz: 30768881f5c826f59dbcca0b17a1192dbdc17ca835c8bcc626e874391131bf92
+  metadata.gz: 54719fad09a9036dbdc1430323272af7f3c3e746aecac9b6199eef70bfe07856
+  data.tar.gz: b9df7b5ff0cc4feebb053955aea4d022e72263be1288578906b67f2100b97160
 SHA512:
-  metadata.gz: e748eedf78b040a7dbe1a1b744f87f1c1e9c3ae751417711eccd5f69dca68335f1a206b9258503da70a8486c3d8588d61ae78081ebdecc6a8ee40f85383a319f
-  data.tar.gz: 2abae603660179e05f8341ab5351fb9e028549674bb13901e4cae4dfd13c99995de0de3c63f8c75182acd155a1d2171b02e4db74fcaa08c1108a1a0e92ad3eee
+  metadata.gz: 42aedd744761fb61e6fcfa32643ee56b267ea163a2376f3eb308b181cd5981ceef4270e88ed1fe322e5f198d94b3b34af277f246e99f0439cfc3699c3838d76e
+  data.tar.gz: '082dda62d97b2655413ef599185b38c88cceacdec3ceecb99aab67b222306b5478963dfa7e216daa56838c6c3a9ac65ced53e51d513ae23ddf0954fb2c5982e5'

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.17.1
+- Add transformer class for [PolynomialFeatures](https://yoshoku.github.io/rumale/doc/Rumale/Preprocessing/PolynomialFeatures.html)
+- Add verbose and tol parameter to [FactorizationMachineClassifier](https://yoshoku.github.io/rumale/doc/Rumale/PolynomialModel/FactorizationMachineClassifier.html)
+  and [FactorizationMachineRegressor](https://yoshoku.github.io/rumale/doc/Rumale/PolynomialModel/FactorizationMachineRegressor.html)
+- Fix bug that factor elements of Factorization Machines estimators are not learned caused by initializing factors to zero.
+
 # 0.17.0
 ## Breaking changes
 - Fix all linear model estimators to use the new abstract class ([BaseSGD](https://yoshoku.github.io/rumale/doc/Rumale/LinearModel/BaseSGD.html)) introduced in version 0.16.1.

data/lib/rumale.rb

@@ -91,6 +91,7 @@ require 'rumale/preprocessing/label_binarizer'
 require 'rumale/preprocessing/label_encoder'
 require 'rumale/preprocessing/one_hot_encoder'
 require 'rumale/preprocessing/ordinal_encoder'
+require 'rumale/preprocessing/polynomial_features'
 require 'rumale/model_selection/k_fold'
 require 'rumale/model_selection/stratified_k_fold'
 require 'rumale/model_selection/shuffle_split'

data/lib/rumale/optimizer/ada_grad.rb

@@ -7,11 +7,6 @@ module Rumale
   module Optimizer
     # AdaGrad is a class that implements AdaGrad optimizer.
     #
-    # @example
-    #   optimizer = Rumale::Optimizer::AdaGrad.new(learning_rate: 0.01, momentum: 0.9)
-    #   estimator = Rumale::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
-    #   estimator.fit(samples, values)
-    #
     # *Reference*
     # - J. Duchi, E Hazan, and Y. Singer, "Adaptive Subgradient Methods for Online Learning and Stochastic Optimization," J. Machine Learning Research, vol. 12, pp. 2121--2159, 2011.
     class AdaGrad

data/lib/rumale/optimizer/adam.rb

@@ -7,11 +7,6 @@ module Rumale
   module Optimizer
     # Adam is a class that implements Adam optimizer.
     #
-    # @example
-    #   optimizer = Rumale::Optimizer::Adam.new(learning_rate: 0.01, momentum: 0.9, decay1: 0.9, decay2: 0.999)
-    #   estimator = Rumale::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
-    #   estimator.fit(samples, values)
-    #
     # *Reference*
     # - D P. Kingma and J. Ba, "Adam: A Method for Stochastic Optimization," Proc. ICLR'15, 2015.
     class Adam

data/lib/rumale/optimizer/nadam.rb

@@ -8,11 +8,6 @@ module Rumale
   module Optimizer
     # Nadam is a class that implements Nadam optimizer.
     #
-    # @example
-    #   optimizer = Rumale::Optimizer::Nadam.new(learning_rate: 0.01, decay1: 0.9, decay2: 0.999)
-    #   estimator = Rumale::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
-    #   estimator.fit(samples, values)
-    #
     # *Reference*
     # - T. Dozat, "Incorporating Nesterov Momentum into Adam," Tech. Repo. Stanford University, 2015.
     class Nadam

data/lib/rumale/optimizer/rmsprop.rb

@@ -7,11 +7,6 @@ module Rumale
   module Optimizer
     # RMSProp is a class that implements RMSProp optimizer.
     #
-    # @example
-    #   optimizer = Rumale::Optimizer::RMSProp.new(learning_rate: 0.01, momentum: 0.9, decay: 0.9)
-    #   estimator = Rumale::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
-    #   estimator.fit(samples, values)
-    #
     # *Reference*
     # - I. Sutskever, J. Martens, G. Dahl, and G. Hinton, "On the importance of initialization and momentum in deep learning," Proc. ICML' 13, pp. 1139--1147, 2013.
     # - G. Hinton, N. Srivastava, and K. Swersky, "Lecture 6e rmsprop," Neural Networks for Machine Learning, 2012.

data/lib/rumale/optimizer/sgd.rb

@@ -6,11 +6,6 @@ require 'rumale/base/base_estimator'
 module Rumale
   module Optimizer
     # SGD is a class that implements SGD optimizer.
-    #
-    # @example
-    #   optimizer = Rumale::Optimizer::SGD.new(learning_rate: 0.01, momentum: 0.9, decay: 0.9)
-    #   estimator = Rumale::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
-    #   estimator.fit(samples, values)
     class SGD
       include Base::BaseEstimator
       include Validation

data/lib/rumale/optimizer/yellow_fin.rb

@@ -7,11 +7,6 @@ module Rumale
   module Optimizer
     # YellowFin is a class that implements YellowFin optimizer.
     #
-    # @example
-    #   optimizer = Rumale::Optimizer::YellowFin.new(learning_rate: 0.01, momentum: 0.9, decay: 0.999, window_width: 20)
-    #   estimator = Rumale::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
-    #   estimator.fit(samples, values)
-    #
     # *Reference*
     # - J. Zhang and I. Mitliagkas, "YellowFin and the Art of Momentum Tuning," CoRR abs/1706.03471, 2017.
     class YellowFin

data/lib/rumale/polynomial_model/base_factorization_machine.rb

@@ -20,15 +20,18 @@ module Rumale
       # @param max_iter [Integer] The maximum number of epochs that indicates
       #   how many times the whole data is given to the training process.
       # @param batch_size [Integer] The size of the mini batches.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
       # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
       #   If nil is given, Nadam is used.
       # @param n_jobs [Integer] The number of jobs for running the fit and predict methods in parallel.
       #   If nil is given, the methods do not execute in parallel.
       #   If zero or less is given, it becomes equal to the number of processors.
       #   This parameter is ignored if the Parallel gem is not loaded.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(n_factors: 2, loss: nil, reg_param_linear: 1.0, reg_param_factor: 1.0,
-                     max_iter: 200, batch_size: 50, optimizer: nil, n_jobs: nil, random_seed: nil)
+                     max_iter: 200, batch_size: 50, tol: 1e-4,
+                     optimizer: nil, n_jobs: nil, verbose: false, random_seed: nil)
         @params = {}
         @params[:n_factors] = n_factors
         @params[:loss] = loss unless loss.nil?
@@ -36,9 +39,11 @@ module Rumale
         @params[:reg_param_factor] = reg_param_factor
         @params[:max_iter] = max_iter
         @params[:batch_size] = batch_size
+        @params[:tol] = tol
         @params[:optimizer] = optimizer
         @params[:optimizer] ||= Optimizer::Nadam.new
         @params[:n_jobs] = n_jobs
+        @params[:verbose] = verbose
         @params[:random_seed] = random_seed
         @params[:random_seed] ||= srand
         @factor_mat = nil
@@ -51,14 +56,15 @@ module Rumale
 
       def partial_fit(x, y)
         # Initialize some variables.
+        class_name = self.class.to_s.split('::').last if @params[:verbose]
         n_samples, n_features = x.shape
         sub_rng = @rng.dup
         weight_vec = Numo::DFloat.zeros(n_features + 1)
-        factor_mat = Numo::DFloat.zeros(@params[:n_factors], n_features)
+        factor_mat = Rumale::Utils.rand_normal([@params[:n_factors], n_features], sub_rng)
         weight_optimizer = @params[:optimizer].dup
         factor_optimizers = Array.new(@params[:n_factors]) { @params[:optimizer].dup }
         # Start optimization.
-        @params[:max_iter].times do |_t|
+        @params[:max_iter].times do |t|
           sample_ids = [*0...n_samples]
           sample_ids.shuffle!(random: sub_rng)
           until (subset_ids = sample_ids.shift(@params[:batch_size])).empty?
@@ -76,10 +82,17 @@ module Rumale
                                                               factor_gradient(loss_grad, sub_x, factor_mat[n, true]))
             end
           end
+          loss = loss_func(x, expand_feature(x), y, factor_mat, weight_vec)
+          puts "[#{class_name}] Loss after #{t + 1} epochs: #{loss}" if @params[:verbose]
+          break if loss < @params[:tol]
         end
         [factor_mat, *split_weight_vec_bias(weight_vec)]
       end
 
+      def loss_func(_x, _expanded_x, _y, _factor, _weight)
+        raise NotImplementedError, "#{__method__} has to be implemented in #{self.class}."
+      end
+
       def loss_gradient(_x, _expanded_x, _y, _factor, _weight)
         raise NotImplementedError, "#{__method__} has to be implemented in #{self.class}."
       end

data/lib/rumale/polynomial_model/factorization_machine_classifier.rb

@@ -53,18 +53,22 @@ module Rumale
       # @param max_iter [Integer] The maximum number of epochs that indicates
       #   how many times the whole data is given to the training process.
       # @param batch_size [Integer] The size of the mini batches.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
       # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
       #   If nil is given, Nadam is used.
       # @param n_jobs [Integer] The number of jobs for running the fit and predict methods in parallel.
       #   If nil is given, the methods do not execute in parallel.
       #   If zero or less is given, it becomes equal to the number of processors.
       #   This parameter is ignored if the Parallel gem is not loaded.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(n_factors: 2, loss: 'hinge', reg_param_linear: 1.0, reg_param_factor: 1.0,
-                     max_iter: 200, batch_size: 50, optimizer: nil, n_jobs: nil, random_seed: nil)
+                     max_iter: 200, batch_size: 50, tol: 1e-4,
+                     optimizer: nil, n_jobs: nil, verbose: false, random_seed: nil)
         check_params_numeric(reg_param_linear: reg_param_linear, reg_param_factor: reg_param_factor,
-                             n_factors: n_factors, max_iter: max_iter, batch_size: batch_size)
+                             n_factors: n_factors, max_iter: max_iter, batch_size: batch_size, tol: tol)
         check_params_string(loss: loss)
+        check_params_boolean(verbose: verbose)
         check_params_numeric_or_nil(n_jobs: n_jobs, random_seed: random_seed)
         check_params_positive(n_factors: n_factors,
                               reg_param_linear: reg_param_linear, reg_param_factor: reg_param_factor,
@@ -196,6 +200,15 @@ module Rumale
         ex_x.dot(weight) + 0.5 * (factor.dot(x.transpose)**2 - (factor**2).dot(x.transpose**2)).sum(0)
       end
 
+      def loss_func(x, ex_x, y, factor, weight)
+        z = bin_decision_function(x, ex_x, factor, weight)
+        if @params[:loss] == 'hinge'
+          z.class.maximum(0.0, 1 - y * z).sum.fdiv(y.shape[0])
+        else
+          Numo::NMath.log(1 + Numo::NMath.exp(-y * z)).sum.fdiv(y.shape[0])
+        end
+      end
+
       def hinge_loss_gradient(x, ex_x, y, factor, weight)
         evaluated = y * bin_decision_function(x, ex_x, factor, weight)
         gradient = Numo::DFloat.zeros(evaluated.size)

data/lib/rumale/polynomial_model/factorization_machine_regressor.rb

@@ -46,17 +46,21 @@ module Rumale
       # @param max_iter [Integer] The maximum number of epochs that indicates
       #   how many times the whole data is given to the training process.
       # @param batch_size [Integer] The size of the mini batches.
+      # @param tol [Float] The tolerance of loss for terminating optimization.
       # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
       #   If nil is given, Nadam is used.
       # @param n_jobs [Integer] The number of jobs for running the fit method in parallel.
       #   If nil is given, the method does not execute in parallel.
       #   If zero or less is given, it becomes equal to the number of processors.
       #   This parameter is ignored if the Parallel gem is not loaded.
+      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(n_factors: 2, reg_param_linear: 1.0, reg_param_factor: 1.0,
-                     max_iter: 200, batch_size: 50, optimizer: nil, n_jobs: nil, random_seed: nil)
+                     max_iter: 200, batch_size: 50, tol: 1e-4,
+                     optimizer: nil, n_jobs: nil, verbose: false, random_seed: nil)
         check_params_numeric(reg_param_linear: reg_param_linear, reg_param_factor: reg_param_factor,
-                             n_factors: n_factors, max_iter: max_iter, batch_size: batch_size)
+                             n_factors: n_factors, max_iter: max_iter, batch_size: batch_size, tol: tol)
+        check_params_boolean(verbose: verbose)
         check_params_numeric_or_nil(n_jobs: n_jobs, random_seed: random_seed)
         check_params_positive(n_factors: n_factors, reg_param_linear: reg_param_linear, reg_param_factor: reg_param_factor,
                               max_iter: max_iter, batch_size: batch_size)
@@ -132,6 +136,11 @@ module Rumale
 
       private
 
+      def loss_func(x, ex_x, y, factor, weight)
+        z = ex_x.dot(weight) + 0.5 * (factor.dot(x.transpose)**2 - (factor**2).dot(x.transpose**2)).sum(0)
+        ((z - y)**2).sum.fdiv(y.shape[0])
+      end
+
       def loss_gradient(x, ex_x, y, factor, weight)
         z = ex_x.dot(weight) + 0.5 * (factor.dot(x.transpose)**2 - (factor**2).dot(x.transpose**2)).sum(0)
         2.0 * (z - y)

data/lib/rumale/preprocessing/polynomial_features.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator'
+require 'rumale/base/transformer'
+
+module Rumale
+  module Preprocessing
+    # Generating polynomial features from the given samples.
+    #
+    # @example
+    #   require 'rumale'
+    #
+    #   transformer = Rumale::Preprocessing::PolynomialFeatures.new(degree: 2)
+    #   x = Numo::DFloat[[0, 1], [2, 3], [4, 5]]
+    #   z = transformer.fit_transform(x)
+    #   p z
+    #
+    #   # Numo::DFloat#shape=[3,6]
+    #   # [[1, 0, 1, 0, 0, 1],
+    #   #  [1, 2, 3, 4, 6, 9],
+    #   #  [1, 4, 5, 16, 20, 25]]
+    #
+    #   # If you want to perform polynomial regression, combine it with LinearRegression as follows:
+    #   ply = Rumale::Preprocessing::PolynomialFeatures.new(degree: 2)
+    #   reg = Rumale::LinearModel::LinearRegression.new(fit_bias: false, random_seed: 1)
+    #   pipeline = Rumale::Pipeline::Pipeline.new(steps: { trs: ply, est: reg })
+    #   pipeline.fit(training_samples, training_values)
+    #   results = pipeline.predict(testing_samples)
+    #
+    class PolynomialFeatures
+      include Base::BaseEstimator
+      include Base::Transformer
+
+      # Return the number of polynomial features.
+      # @return [Integer]
+      attr_reader :n_output_features
+
+      # Create a transformer for generating polynomial features.
+      #
+      # @param degree [Integer] The degree of polynomial features.
+      def initialize(degree: 2)
+        check_params_numeric(degree: degree)
+        raise ArgumentError, 'Expect the value of degree parameter greater than or eqaul to 1.' if degree < 1
+        @params = {}
+        @params[:degree] = degree
+        @n_output_features = nil
+      end
+
+      # Calculate the number of output polynomial fetures.
+      #
+      # @overload fit(x) -> PolynomialFeatures
+      #   @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to calculate the number of output polynomial fetures.
+      # @return [PolynomialFeatures]
+      def fit(x, _y = nil)
+        x = check_convert_sample_array(x)
+        n_features = x.shape[1]
+        @n_output_features = 1
+        @params[:degree].times do |t|
+          @n_output_features += Array.new(n_features) { |n| n }.repeated_combination(t + 1).size
+        end
+        self
+      end
+
+      # Calculate the number of polynomial features, and then transform samples to polynomial features.
+      #
+      # @overload fit_transform(x) -> Numo::DFloat
+      #   @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to calculate the number of polynomial features
+      #     and be transformed.
+      # @return [Numo::DFloat] (shape: [n_samples, n_output_features]) The transformed samples.
+      def fit_transform(x, _y = nil)
+        x = check_convert_sample_array(x)
+        fit(x).transform(x)
+      end
+
+      # Transform the given samples to polynomial features.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to be transformed.
+      # @return [Numo::DFloat] (shape: [n_samples, n_output_features]) The transformed samples.
+      def transform(x)
+        x = check_convert_sample_array(x)
+        # initialize transformed features
+        n_samples, n_features = x.shape
+        z = Numo::DFloat.zeros(n_samples, n_output_features)
+        # bias
+        z[true, 0] = 1
+        curr_col = 1
+        # itself
+        z[true, 1..n_features] = x
+        curr_col += n_features
+        # high degree features
+        curr_feat_ids = Array.new(n_features + 1) { |n| n + 1 }
+        (1...@params[:degree]).each do
+          next_feat_ids = []
+          n_features.times do |d|
+            f_range = curr_feat_ids[d]...curr_feat_ids.last
+            next_col = curr_col + f_range.size
+            z[true, curr_col...next_col] = z[true, f_range] * x[true, d..d]
+            next_feat_ids.push(curr_col)
+            curr_col = next_col
+          end
+          next_feat_ids.push(curr_col)
+          curr_feat_ids = next_feat_ids
+        end
+        z
+      end
+    end
+  end
+end

data/lib/rumale/version.rb

@@ -3,5 +3,5 @@
 # Rumale is a machine learning library in Ruby.
 module Rumale
   # The version of Rumale you are using.
-  VERSION = '0.17.0'
+  VERSION = '0.17.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rumale
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.17.1
 platform: ruby
 authors:
 - yoshoku
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-18 00:00:00.000000000 Z
+date: 2020-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -258,6 +258,7 @@ files:
 - lib/rumale/preprocessing/min_max_scaler.rb
 - lib/rumale/preprocessing/one_hot_encoder.rb
 - lib/rumale/preprocessing/ordinal_encoder.rb
+- lib/rumale/preprocessing/polynomial_features.rb
 - lib/rumale/preprocessing/standard_scaler.rb
 - lib/rumale/probabilistic_output.rb
 - lib/rumale/tree/base_decision_tree.rb