rumale 0.15.0 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d8823e97350be198c39b1896dc88978a47b526f1
-  data.tar.gz: d65a2d3274d104eae9aa20dd97078159614783e1
+  metadata.gz: e97522eaf22db6c80513f8d69513b6aef7dc1a9f
+  data.tar.gz: a2c2d51a15465f5bac67bc0b60aef1c70067bba4
 SHA512:
-  metadata.gz: 5f06921658636e7765edc7a71aa9df28bf6a5cd4b36706671b2bf3a75c55755d190e8ab4dc9801cf94a4034b04429a6f14d8978c0323a9a2a6bf5d34456aa2e5
-  data.tar.gz: 74dfe7a75358e9e26da392dae39e71c1441a56c63ad51236db3bedfda4156f8cd86a1b69c40574e34b2db46fb8bdb9f22da60bde0a9423afa6849b11551a9494
+  metadata.gz: b04158c2f4247c58593dad54e1d8c79030aff2a3826032610b9d7afee73490843caa3c5636c86fc3fc3196ed45affdb2da17c7d926091f77c784f1970464c2a4
+  data.tar.gz: e75814bcbe3aa087f2a45a15790a180016147d9197442d5d329a6484d1baee8b4d2726265d640a3d47ada45189d6dfb3157f2fa5eec7e334824cf96bd8fba493

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+# 0.16.0
+## Breaking changes
+- The meaning of the `max_iter` parameter of the multi-layer perceptron estimators
+has been changed from the maximum number of iterations to the number of epochs.
+The number of epochs is how many times the whole data is given to the training process.
+As a future plan, similar changes will be applied to other estimators used stochastic gradient descent such as SVC and Lasso.
+  - [MLPClassifier](https://yoshoku.github.io/rumale/doc/Rumale/NeuralNetwork/MLPClassifier.html)
+  - [MLPRegressor](https://yoshoku.github.io/rumale/doc/Rumale/NeuralNetwork/MLPRegressor.html)
+
 # 0.15.0
 - Add feature extractor classes:
   - [HashVectorizer](https://yoshoku.github.io/rumale/doc/Rumale/FeatureExtraction/HashVectorizer.html)

data/lib/rumale.rb

@@ -74,6 +74,7 @@ require 'rumale/decomposition/factor_analysis'
 require 'rumale/decomposition/fast_ica'
 require 'rumale/manifold/tsne'
 require 'rumale/manifold/mds'
+require 'rumale/neural_network/adam'
 require 'rumale/neural_network/base_mlp'
 require 'rumale/neural_network/mlp_regressor'
 require 'rumale/neural_network/mlp_classifier'

data/lib/rumale/neural_network/adam.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator'
+
+module Rumale
+  module NeuralNetwork
+    # @!visibility private
+    # This module consists of the classes that implement optimizers adaptively tuning learning rate.
+    module Optimizer
+      # @!visibility private
+      # Adam is a class that implements Adam optimizer.
+      #
+      # *Reference*
+      # - D P. Kingma and J. Ba, "Adam: A Method for Stochastic Optimization," Proc. ICLR'15, 2015.
+      class Adam
+        include Base::BaseEstimator
+
+        # @!visibility private
+        # Create a new optimizer with Adam
+        #
+        # @param learning_rate [Float] The initial value of learning rate.
+        # @param decay1 [Float] The smoothing parameter for the first moment.
+        # @param decay2 [Float] The smoothing parameter for the second moment.
+        def initialize(learning_rate: 0.001, decay1: 0.9, decay2: 0.999)
+          @params = {}
+          @params[:learning_rate] = learning_rate
+          @params[:decay1] = decay1
+          @params[:decay2] = decay2
+          @fst_moment = nil
+          @sec_moment = nil
+          @iter = 0
+        end
+
+        # @!visibility private
+        # Calculate the updated weight with Nadam adaptive learning rate.
+        #
+        # @param weight [Numo::DFloat] (shape: [n_features]) The weight to be updated.
+        # @param gradient [Numo::DFloat] (shape: [n_features]) The gradient for updating the weight.
+        # @return [Numo::DFloat] (shape: [n_feautres]) The updated weight.
+        def call(weight, gradient)
+          @fst_moment ||= Numo::DFloat.zeros(weight.shape)
+          @sec_moment ||= Numo::DFloat.zeros(weight.shape)
+
+          @iter += 1
+
+          @fst_moment = @params[:decay1] * @fst_moment + (1.0 - @params[:decay1]) * gradient
+          @sec_moment = @params[:decay2] * @sec_moment + (1.0 - @params[:decay2]) * gradient**2
+          nm_fst_moment = @fst_moment / (1.0 - @params[:decay1]**@iter)
+          nm_sec_moment = @sec_moment / (1.0 - @params[:decay2]**@iter)
+
+          weight - @params[:learning_rate] * nm_fst_moment / (nm_sec_moment**0.5 + 1e-8)
+        end
+
+        # Dump marshal data.
+        # @return [Hash] The marshal data.
+        # def marshal_dump
+        #  { params: @params,
+        #    fst_moment: @fst_moment,
+        #    sec_moment: @sec_moment,
+        #    iter: @iter }
+        # end
+
+        # Load marshal data.
+        # @return [nil]
+        # def marshal_load(obj)
+        #  @params = obj[:params]
+        #  @fst_moment = obj[:fst_moment]
+        #  @sec_moment = obj[:sec_moment]
+        #  @iter = obj[:iter]
+        #  nil
+        # end
+      end
+    end
+  end
+end

data/lib/rumale/neural_network/base_mlp.rb

@@ -177,13 +177,14 @@ module Rumale
       # @param learning_rate [Float] The initial value of learning rate in Adam optimizer.
       # @param decay1 [Float] The smoothing parameter for the first moment in Adam optimizer.
       # @param decay2 [Float] The smoothing parameter for the second moment in Adam optimizer.
-      # @param max_iter [Integer] The maximum number of iterations.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
       # @param batch_size [Intger] The size of the mini batches.
       # @param tol [Float] The tolerance of loss for terminating optimization.
       # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(hidden_units: [128, 128], dropout_rate: 0.4, learning_rate: 0.001, decay1: 0.9, decay2: 0.999,
-                     max_iter: 10000, batch_size: 50, tol: 1e-4, verbose: false, random_seed: nil)
+                     max_iter: 200, batch_size: 50, tol: 1e-4, verbose: false, random_seed: nil)
         @params = {}
         @params[:hidden_units] = hidden_units
         @params[:dropout_rate] = dropout_rate
@@ -203,7 +204,9 @@ module Rumale
       private
 
       def buld_network(n_inputs, n_outputs, srng = nil)
-        adam = Rumale::Optimizer::Adam.new(learning_rate: @params[:learning_rate], decay1: @params[:decay1], decay2: @params[:decay2])
+        adam = Rumale::NeuralNetwork::Optimizer::Adam.new(
+          learning_rate: @params[:learning_rate], decay1: @params[:decay1], decay2: @params[:decay2]
+        )
         model = Model::Sequential.new
         n_units = [n_inputs, *@params[:hidden_units]]
         n_units.each_cons(2) do |n_in, n_out|
@@ -216,25 +219,25 @@ module Rumale
 
       def train(x, y, network, loss_func, srng = nil)
         class_name = self.class.to_s.split('::').last
-
         n_samples = x.shape[0]
-        rand_ids = [*0...n_samples].shuffle(random: srng)
 
         @params[:max_iter].times do |t|
-          # random sampling
-          subset_ids = rand_ids.shift(@params[:batch_size])
-          rand_ids.concat(subset_ids)
-          sub_x = x[subset_ids, true].dup
-          sub_y = y[subset_ids, true].dup
-          # forward
-          out, backward = network.forward(sub_x)
-          # calc loss function
-          loss, dout = loss_func.call(out, sub_y)
+          sample_ids = [*0...n_samples]
+          sample_ids.shuffle!(random: srng)
+          until (subset_ids = sample_ids.shift(@params[:batch_size])).empty?
+            # random sampling
+            sub_x = x[subset_ids, true].dup
+            sub_y = y[subset_ids, true].dup
+            # forward
+            out, backward = network.forward(sub_x)
+            # calc loss function
+            loss, dout = loss_func.call(out, sub_y)
+            break if loss < @params[:tol]
+            # backward
+            backward.call(dout)
+          end
           @n_iter = t + 1
-          puts "[#{class_name}] Loss after #{@n_iter} iterations: #{loss}" if @params[:verbose] && (@n_iter % 10).zero?
-          break if loss < @params[:tol]
-          # backward
-          backward.call(dout)
+          puts "[#{class_name}] Loss after #{@n_iter} epochs: #{loss}" if @params[:verbose]
         end
 
         network

data/lib/rumale/neural_network/mlp_classifier.rb

@@ -40,13 +40,14 @@ module Rumale
       # @param learning_rate [Float] The initial value of learning rate in Adam optimizer.
       # @param decay1 [Float] The smoothing parameter for the first moment in Adam optimizer.
       # @param decay2 [Float] The smoothing parameter for the second moment in Adam optimizer.
-      # @param max_iter [Integer] The maximum number of iterations.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
       # @param batch_size [Intger] The size of the mini batches.
       # @param tol [Float] The tolerance of loss for terminating optimization.
       # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(hidden_units: [128, 128], dropout_rate: 0.4, learning_rate: 0.001, decay1: 0.9, decay2: 0.999,
-                     max_iter: 10000, batch_size: 50, tol: 1e-4, verbose: false, random_seed: nil)
+                     max_iter: 200, batch_size: 50, tol: 1e-4, verbose: false, random_seed: nil)
         check_params_type(Array, hidden_units: hidden_units)
         check_params_numeric(dropout_rate: dropout_rate, learning_rate: learning_rate, decay1: decay1, decay2: decay2,
                              max_iter: max_iter, batch_size: batch_size, tol: tol)

data/lib/rumale/neural_network/mlp_regressor.rb

@@ -35,13 +35,14 @@ module Rumale
       # @param learning_rate [Float] The initial value of learning rate in Adam optimizer.
       # @param decay1 [Float] The smoothing parameter for the first moment in Adam optimizer.
       # @param decay2 [Float] The smoothing parameter for the second moment in Adam optimizer.
-      # @param max_iter [Integer] The maximum number of iterations.
+      # @param max_iter [Integer] The maximum number of epochs that indicates
+      #   how many times the whole data is given to the training process.
       # @param batch_size [Intger] The size of the mini batches.
       # @param tol [Float] The tolerance of loss for terminating optimization.
       # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
       # @param random_seed [Integer] The seed value using to initialize the random generator.
       def initialize(hidden_units: [128, 128], dropout_rate: 0.4, learning_rate: 0.001, decay1: 0.9, decay2: 0.999,
-                     max_iter: 10000, batch_size: 50, tol: 1e-4, verbose: false, random_seed: nil)
+                     max_iter: 200, batch_size: 50, tol: 1e-4, verbose: false, random_seed: nil)
         check_params_type(Array, hidden_units: hidden_units)
         check_params_numeric(dropout_rate: dropout_rate, learning_rate: learning_rate, decay1: decay1, decay2: decay2,
                              max_iter: max_iter, batch_size: batch_size, tol: tol)

data/lib/rumale/version.rb

@@ -3,5 +3,5 @@
 # Rumale is a machine learning library in Ruby.
 module Rumale
   # The version of Rumale you are using.
-  VERSION = '0.15.0'
+  VERSION = '0.16.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rumale
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.16.0
 platform: ruby
 authors:
 - yoshoku
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-01 00:00:00.000000000 Z
+date: 2020-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -233,6 +233,7 @@ files:
 - lib/rumale/nearest_neighbors/k_neighbors_classifier.rb
 - lib/rumale/nearest_neighbors/k_neighbors_regressor.rb
 - lib/rumale/nearest_neighbors/vp_tree.rb
+- lib/rumale/neural_network/adam.rb
 - lib/rumale/neural_network/base_mlp.rb
 - lib/rumale/neural_network/mlp_classifier.rb
 - lib/rumale/neural_network/mlp_regressor.rb