ruby-dnn 0.9.4 → 0.10.0

data/lib/dnn/core/model.rb

@@ -22,14 +22,14 @@ module DNN
     # @return [DNN::Model]
     def self.load_json(json_str)
       hash = JSON.parse(json_str, symbolize_names: true)
-      model = self.load_hash(hash)
-      model.compile(Utils.load_hash(hash[:optimizer]), Utils.load_hash(hash[:loss]))
+      model = self.from_hash(hash)
+      model.compile(Utils.from_hash(hash[:optimizer]), Utils.from_hash(hash[:loss]))
       model
     end
 
-    def self.load_hash(hash)
+    def self.from_hash(hash)
       model = self.new
-      model.layers = hash[:layers].map { |hash_layer| Utils.load_hash(hash_layer) }
+      model.layers = hash[:layers].map { |hash_layer| Utils.from_hash(hash_layer) }
       model
     end
   
@@ -97,63 +97,61 @@ module DNN
     # @param [DNN::Layers::Layer] layer Layer to add to the model.
     # @return [DNN::Model] return self.
     def <<(layer)
-      # Due to a bug in saving nested models, temporarily prohibit model nesting.
-      # if !layer.is_a?(Layers::Layer) && !layer.is_a?(Model)
-      #   raise TypeError.new("layer is not an instance of the DNN::Layers::Layer class or DNN::Model class.")
-      # end
-      unless layer.is_a?(Layers::Layer)
-        raise TypeError.new("layer:#{layer.class.name} is not an instance of the DNN::Layers::Layer class.")
+      if !layer.is_a?(Layers::Layer) && !layer.is_a?(Model)
+        raise TypeError.new("layer is not an instance of the DNN::Layers::Layer class or DNN::Model class.")
       end
       @layers << layer
       self
     end
 
-    # Set optimizer and loss to model and build all layers.
+    # Set optimizer and loss_func to model and build all layers.
     # @param [DNN::Optimizers::Optimizer] optimizer Optimizer to use for learning.
-    # @param [DNN::Losses::Loss] loss Lptimizer to use for learning.
-    def compile(optimizer, loss)
+    # @param [DNN::Losses::Loss] loss_func Loss function to use for learning.
+    def compile(optimizer, loss_func)
       raise DNN_Error.new("The model is already compiled.") if compiled?
       unless optimizer.is_a?(Optimizers::Optimizer)
         raise TypeError.new("optimizer:#{optimizer.class} is not an instance of DNN::Optimizers::Optimizer class.")
       end
-      unless loss.is_a?(Losses::Loss)
-        raise TypeError.new("loss:#{loss.class} is not an instance of DNN::Losses::Loss class.")
+      unless loss_func.is_a?(Losses::Loss)
+        raise TypeError.new("loss_func:#{loss_func.class} is not an instance of DNN::Losses::Loss class.")
       end
       @compiled = true
       layers_check
       @optimizer = optimizer
-      @loss = loss
+      @loss_func = loss_func
       build
       layers_shape_check
     end
 
-    # Set optimizer and loss to model and recompile. But does not build layers.
+    # Set optimizer and loss_func to model and recompile. But does not build layers.
     # @param [DNN::Optimizers::Optimizer] optimizer Optimizer to use for learning.
-    # @param [DNN::Losses::Loss] loss Lptimizer to use for learning.
-    def recompile(optimizer, loss)
+    # @param [DNN::Losses::Loss] loss_func Loss function to use for learning.
+    def recompile(optimizer, loss_func)
       unless optimizer.is_a?(Optimizers::Optimizer)
         raise TypeError.new("optimizer:#{optimizer.class} is not an instance of DNN::Optimizers::Optimizer class.")
       end
-      unless loss.is_a?(Losses::Loss)
-        raise TypeError.new("loss:#{loss.class} is not an instance of DNN::Losses::Loss class.")
+      unless loss_func.is_a?(Losses::Loss)
+        raise TypeError.new("loss_func:#{loss_func.class} is not an instance of DNN::Losses::Loss class.")
       end
       @compiled = true
       layers_check
       @optimizer = optimizer
-      @loss = loss
+      @loss_func = loss_func
       layers_shape_check
     end
 
     def build(super_model = nil)
       @super_model = super_model
       shape = if super_model
-        super_model.output_shape
+        super_model.get_prev_layer(self).output_shape
       else
         @layers.first.build
       end
-      @layers[1..-1].each do |layer|
+      layers = super_model ? @layers : @layers[1..-1]
+      layers.each do |layer|
         if layer.is_a?(Model)
           layer.build(self)
+          layer.recompile(@optimizer, @loss_func)
         else
           layer.build(shape)
         end
@@ -174,13 +172,13 @@ module DNN
     # @return [DNN::Optimizers::Optimizer] optimizer Return the optimizer to use for learning.
     def optimizer
       raise DNN_Error.new("The model is not compiled.") unless compiled?
-      @optimizer ? @optimizer : @super_model.optimizer
+      @optimizer
     end
 
     # @return [DNN::Losses::Loss] loss Return the loss to use for learning.
-    def loss
+    def loss_func
       raise DNN_Error.new("The model is not compiled.") unless compiled?
-      @loss ? @loss : @super_model.loss
+      @loss_func
     end
 
     # @return [Bool] Returns whether the model is learning.
@@ -195,25 +193,31 @@ module DNN
     # @param [Integer] epochs Number of training.
     # @param [Integer] batch_size Batch size used for one training.
     # @param [Array or NilClass] test If you to test the model for every 1 epoch,
-    #     specify [x_test, y_test]. Don't test to the model, specify nil.                     
+    #                            specify [x_test, y_test]. Don't test to the model, specify nil.
     # @param [Bool] verbose Set true to display the log. If false is set, the log is not displayed.
-    # @param [Proc] batch_proc Set proc to process per batch.
-    # @yield [epoch] Process performed before one training.
+    # @param [Lambda] before_epoch_cbk Process performed before one training.
+    # @param [Lambda] after_epoch_cbk Process performed after one training.
+    # @param [Lambda] before_batch_cbk Set the proc to be performed before batch processing.
+    # @param [Lambda] after_batch_cbk Set the proc to be performed after batch processing.
     def train(x, y, epochs,
               batch_size: 1,
               test: nil,
               verbose: true,
-              batch_proc: nil,
-              &epoch_proc)
+              before_epoch_cbk: nil,
+              after_epoch_cbk: nil,
+              before_batch_cbk: nil,
+              after_batch_cbk: nil)
       raise DNN_Error.new("The model is not compiled.") unless compiled?
       check_xy_type(x, y)
       dataset = Dataset.new(x, y)
       num_train_datas = x.shape[0]
       (1..epochs).each do |epoch|
+        before_epoch_cbk.call(epoch) if before_epoch_cbk
         puts "【 epoch #{epoch}/#{epochs} 】" if verbose
         (num_train_datas.to_f / batch_size).ceil.times do |index|
-          x_batch, y_batch = dataset.get_batch(batch_size)
-          loss_value = train_on_batch(x_batch, y_batch, &batch_proc)
+          x_batch, y_batch = dataset.next_batch(batch_size)
+          loss_value = train_on_batch(x_batch, y_batch,
+                                      before_batch_cbk: before_batch_cbk, after_batch_cbk: after_batch_cbk)
           if loss_value.is_a?(Numo::SFloat)
             loss_value = loss_value.mean
           elsif loss_value.nan?
@@ -236,11 +240,12 @@ module DNN
           print log if verbose
         end
         if verbose && test
-          acc = accurate(test[0], test[1], batch_size, &batch_proc)
-          print "  accurate: #{acc}"
+          acc, test_loss = accurate(test[0], test[1], batch_size,
+                                    before_batch_cbk: before_batch_cbk, after_batch_cbk: after_batch_cbk)
+          print "  accurate: #{acc}, test loss: #{sprintf('%.8f', test_loss)}"
         end
         puts "" if verbose
-        epoch_proc.call(epoch) if epoch_proc
+        after_epoch_cbk.call(epoch) if after_epoch_cbk
       end
     end
   
@@ -248,51 +253,58 @@ module DNN
     # Compile the model before use this method.
     # @param [Numo::SFloat] x Input training data.
     # @param [Numo::SFloat] y Output training data.
+    # @param [Lambda] before_batch_cbk Set the proc to be performed before batch processing.
+    # @param [Lambda] after_batch_cbk Set the proc to be performed after batch processing.
     # @return [Float | Numo::SFloat] Return loss value in the form of Float or Numo::SFloat.
-    # @yield [x, y] batch_proc Set proc to process per batch.
-    def train_on_batch(x, y, &batch_proc)
+    def train_on_batch(x, y, before_batch_cbk: nil, after_batch_cbk: nil)
       raise DNN_Error.new("The model is not compiled.") unless compiled?
       check_xy_type(x, y)
       input_data_shape_check(x, y)
-      x, y = batch_proc.call(x, y) if batch_proc
-      out = forward(x, true)
-      loss_value = @loss.forward(out, y, get_all_layers)
-      dout = @loss.backward(y)
-      backward(dout)
-      @loss.regularizes_backward(get_all_layers)
+      x, y = before_batch_cbk.call(x, y, true) if before_batch_cbk
+      x = forward(x, true)
+      loss_value = @loss_func.forward(x, y, get_all_layers)
+      dy = @loss_func.backward(y, get_all_layers)
+      backward(dy)
       update
+      after_batch_cbk.call(loss_value, true) if after_batch_cbk
       loss_value
     end
   
     # Evaluate model and get accurate of test data.
     # @param [Numo::SFloat] x Input test data.
     # @param [Numo::SFloat] y Output test data.
-    # @yield [x, y] batch_proc Set proc to process per batch.
-    def accurate(x, y, batch_size = 100, &batch_proc)
+    # @param [Lambda] before_batch_cbk Set the proc to be performed before batch processing.
+    # @param [Lambda] after_batch_cbk Set the proc to be performed after batch processing.
+    # @return [Array] Returns the test data accurate and mean loss in the form [accurate, mean_loss].
+    def accurate(x, y, batch_size = 100, before_batch_cbk: nil, after_batch_cbk: nil)
       check_xy_type(x, y)
       input_data_shape_check(x, y)
       batch_size = batch_size >= x.shape[0] ? x.shape[0] : batch_size
+      dataset = Dataset.new(x, y, false)
       correct = 0
+      sum_loss = 0
       (x.shape[0].to_f / batch_size).ceil.times do |i|
-        x_batch = Xumo::SFloat.zeros(batch_size, *x.shape[1..-1])
-        y_batch = Xumo::SFloat.zeros(batch_size, *y.shape[1..-1])
-        batch_size.times do |j|
-          k = i * batch_size + j
-          break if k >= x.shape[0]
-          x_batch[j, false] = x[k, false]
-          y_batch[j, false] = y[k, false]
-        end
-        x_batch, y_batch = batch_proc.call(x_batch, y_batch) if batch_proc
-        out = forward(x_batch, false)
+        x_batch, y_batch = dataset.next_batch(batch_size)
+        x_batch, y_batch = before_batch_cbk.call(x_batch, y_batch, true) if before_batch_cbk
+        x_batch = forward(x_batch, false)
+        sigmoid = Sigmoid.new
         batch_size.times do |j|
           if @layers.last.output_shape == [1]
-            correct += 1 if out[j, 0].round == y_batch[j, 0].round
+            if @loss_func.is_a?(SigmoidCrossEntropy)
+              correct += 1 if sigmoid.forward(x_batch[j, 0]).round == y_batch[j, 0].round
+            else
+              correct += 1 if x_batch[j, 0].round == y_batch[j, 0].round
+            end
           else
-            correct += 1 if out[j, true].max_index == y_batch[j, true].max_index
+            correct += 1 if x_batch[j, true].max_index == y_batch[j, true].max_index
           end
         end
+        loss_value = @loss_func.forward(x_batch, y_batch, get_all_layers)
+        after_batch_cbk.call(loss_value, false) if after_batch_cbk
+        sum_loss += loss_value.is_a?(Numo::SFloat) ? loss_value.mean : loss_value
       end
-      correct.to_f / x.shape[0]
+      mean_loss = sum_loss / batch_size
+      [correct.to_f / x.shape[0], mean_loss]
     end
 
     # Predict data.
@@ -310,6 +322,17 @@ module DNN
       predict(x.reshape(1, *x.shape))[0, false]
     end
 
+    # Get loss value.
+    # @param [Numo::SFloat] x Input data.
+    # @param [Numo::SFloat] y Output data.
+    # @return [Float | Numo::SFloat] Return loss value in the form of Float or Numo::SFloat.
+    def loss(x, y)
+      check_xy_type(x, y)
+      input_data_shape_check(x, y)
+      x = forward(x, false)
+      @loss_func.forward(x, y, get_all_layers)
+    end
+
     # @return [DNN::Model] Copy this model.
     def copy
       Marshal.load(Marshal.dump(self))
@@ -334,35 +357,35 @@ module DNN
       }.flatten
     end
   
-    # TODO
-    # It is not good to write the Layer class name directly in the Model class. I will fix it later.
     def forward(x, learning_phase)
       @layers.each do |layer|
-        x = if layer.is_a?(Layers::Dropout) || layer.is_a?(Layers::BatchNormalization) || layer.is_a?(Model)
+        x = if layer.is_a?(Model)
           layer.forward(x, learning_phase)
         else
+          layer.learning_phase = learning_phase
           layer.forward(x)
         end
       end
       x
     end
   
-    def backward(dout)
+    def backward(dy)
       @layers.reverse.each do |layer|
-        dout = layer.backward(dout)
+        dy = layer.backward(dy)
       end
-      dout
+      dy
     end
 
     def update
       return unless @trainable
-      @layers.each do |layer|
-        if layer.is_a?(Layers::HasParamLayer)
-          layer.update(@optimizer)
-        elsif layer.is_a?(Model)
-          layer.update
+      all_trainable_layers = @layers.map { |layer|
+        if layer.is_a?(Model)
+          layer.trainable ? layer.get_all_layers : nil
+        else
+          layer
         end
-      end
+      }.flatten.compact.uniq
+      @optimizer.update(all_trainable_layers)
     end
 
     def get_prev_layer(layer)
@@ -385,13 +408,13 @@ module DNN
 
     def to_hash
       hash_layers = @layers.map { |layer| layer.to_hash }
-      {class: Model.name, layers: hash_layers, optimizer: @optimizer.to_hash, loss: @loss.to_hash}
+      {class: Model.name, layers: hash_layers, optimizer: @optimizer.to_hash, loss: @loss_func.to_hash}
     end
 
     private
 
     def layers_check
-      unless @layers.first.is_a?(Layers::InputLayer)
+      if !@layers.first.is_a?(Layers::InputLayer) && !@super_model
         raise TypeError.new("The first layer is not an InputLayer.")
       end
     end

data/lib/dnn/core/normalizations.rb

@@ -0,0 +1,72 @@
+module DNN
+  module Layers
+
+    class BatchNormalization < HasParamLayer
+      # @return [Integer] The axis to normalization.
+      attr_reader :axis
+      # @return [Float] Exponential moving average of mean and variance.
+      attr_accessor :momentum
+      # @return [Float] Value to avoid division by zero.
+      attr_accessor :eps
+
+      def self.from_hash(hash)
+        self.new(axis: hash[:axis], momentum: hash[:momentum])
+      end
+
+      # @param [integer] axis The axis to normalization.
+      # @param [Float] momentum Exponential moving average of mean and variance.
+      # @param [Float] eps Value to avoid division by zero.
+      def initialize(axis: 0, momentum: 0.9, eps: 1e-7)
+        super()
+        @axis = axis
+        @momentum = momentum
+        @eps = eps
+      end
+
+      def build(input_shape)
+        super
+        @params[:gamma] = @gamma = Param.new(Xumo::SFloat.ones(*output_shape), 0)
+        @params[:beta] = @beta = Param.new(Xumo::SFloat.zeros(*output_shape), 0)
+        @params[:running_mean] = @running_mean = Param.new(Xumo::SFloat.zeros(*output_shape))
+        @params[:running_var] = @running_var = Param.new(Xumo::SFloat.zeros(*output_shape))
+      end
+
+      def forward(x)
+        if learning_phase
+          mean = x.mean(axis: @axis, keepdims: true)
+          @xc = x - mean
+          var = (@xc**2).mean(axis: @axis, keepdims: true)
+          @std = NMath.sqrt(var + @eps)
+          xn = @xc / @std
+          @xn = xn
+          @running_mean.data = @momentum * @running_mean.data + (1 - @momentum) * mean
+          @running_var.data = @momentum * @running_var.data + (1 - @momentum) * var
+        else
+          xc = x - @running_mean.data
+          xn = xc / NMath.sqrt(@running_var.data + @eps)
+        end
+        @gamma.data * xn + @beta.data
+      end
+
+      def backward(dy)
+        batch_size = dy.shape[@axis]
+        if @trainable
+          @beta.grad = dy.sum(axis: @axis, keepdims: true)
+          @gamma.grad = (@xn * dy).sum(axis: @axis, keepdims: true)
+        end
+        dxn = @gamma.data * dy
+        dxc = dxn / @std
+        dstd = -((dxn * @xc) / (@std**2)).sum(axis: @axis, keepdims: true)
+        dvar = 0.5 * dstd / @std
+        dxc += (2.0 / batch_size) * @xc * dvar
+        dmean = dxc.sum(axis: @axis, keepdims: true)
+        dxc - dmean / batch_size
+      end
+
+      def to_hash
+        super({axis: @axis, momentum: @momentum, eps: @eps})
+      end
+    end
+
+  end
+end

data/lib/dnn/core/optimizers.rb

@@ -3,16 +3,22 @@ module DNN
 
     # Super class of all optimizer classes.
     class Optimizer
+      # @return [Float] Return the Learning rate.
       attr_accessor :learning_rate
 
       def initialize(learning_rate)
         @learning_rate = learning_rate
       end
 
-      # Update params.
-      # Classes that inherit from this class must implement this method.
-      def update(params)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'update'")
+      # Update layers has param.
+      def update(layers)
+        target_params = layers.select { |layer| layer.is_a?(HasParamLayer) && layer.trainable }
+                              .map { |layer| layer.params.values }.flatten
+                              .select { |param| param.grad }
+        target_params.each do |param|
+          update_param(param)
+          param.grad = 0
+        end
       end
 
       def to_hash(merge_hash = nil)
@@ -20,169 +26,256 @@ module DNN
         hash.merge!(merge_hash) if merge_hash
         hash
       end
+
+      # Update param.
+      # Classes that inherit from this class must implement this method.
+      private def update_param(param)
+        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'update_param'")
+      end
     end
 
 
     class SGD < Optimizer
+      # @return [Float] Return the momentum coefficient.
       attr_accessor :momentum
 
-      def self.load_hash(hash)
+      def self.from_hash(hash)
         self.new(hash[:learning_rate], momentum: hash[:momentum])
       end
 
+      # @param [Float] learning_rate Learning rate.
+      # @param [Float] momentum momentum coefficient.
       def initialize(learning_rate = 0.01, momentum: 0)
         super(learning_rate)
         @momentum = momentum
         @v = {}
       end
-    
-      def update(params)
-        params.select { |key, param| param.grad }.each_value do |param|
-          amount = param.grad * @learning_rate
-          if @momentum > 0
-            @v[param] ||= 0
-            amount += @momentum * @v[param]
-            @v[param] = amount
-          end
-          param.data -= amount
-        end
-      end
 
       def to_hash
-        super({momentum: @momentum})
+        super(momentum: @momentum)
+      end
+
+      private def update_param(param)
+        amount = param.grad * @learning_rate
+        if @momentum > 0
+          @v[param] ||= 0
+          amount += @momentum * @v[param]
+          @v[param] = amount
+        end
+        param.data -= amount
       end
     end
 
 
-    class Nesterov < SGD
-      def self.load_hash(hash)
+    class Nesterov < Optimizer
+      attr_accessor :momentum
+      
+      def self.from_hash(hash)
         self.new(hash[:learning_rate], momentum: hash[:momentum])
       end
 
+      # @param [Float] learning_rate Learning rate.
+      # @param [Float] momentum momentum coefficient.
       def initialize(learning_rate = 0.01, momentum: 0.9)
-        super(learning_rate, momentum: momentum)
+        super(learning_rate)
+        @momentum = momentum
+        @v = {}
+      end
+
+      def to_hash
+        super(momentum: @momentum)
       end
     
-      def update(params)
-        params.select { |key, param| param.grad }.each_value do |param|
-          @v[param] ||= 0
-          amount = param.grad * @learning_rate
-          @v[param] = @v[param] * @momentum - amount
-          param.data = (param.data + @momentum**2 * @v[param]) - (1 + @momentum) * amount
-        end
+      private def update_param(param)
+        @v[param] ||= 0
+        amount = param.grad * @learning_rate
+        @v[param] = @v[param] * @momentum - amount
+        param.data = (param.data + @momentum**2 * @v[param]) - (1 + @momentum) * amount
       end
     end
     
     
     class AdaGrad < Optimizer
-      def initialize(learning_rate = 0.01)
+      # @return [Float] Return the eps value.
+      attr_accessor :eps
+
+      # @param [Float] learning_rate Learning rate.
+      # @param [Float] eps Value to avoid division by zero.
+      def initialize(learning_rate = 0.01, eps: 1e-7)
         super(learning_rate)
+        @eps = eps
         @g = {}
       end
 
-      def self.load_hash(hash)
-        self.new(hash[:learning_rate])
+      def self.from_hash(hash)
+        self.new(hash[:learning_rate], eps: hash[:eps])
       end
     
-      def update(params)
-        params.select { |key, param| param.grad }.each_value do |param|
-          @g[param] ||= 0
-          @g[param] += param.grad**2
-          param.data -= (@learning_rate / NMath.sqrt(@g[param] + 1e-7)) * param.grad
-        end
+      private def update_param(param)
+        @g[param] ||= 0
+        @g[param] += param.grad**2
+        param.data -= (@learning_rate / NMath.sqrt(@g[param] + @eps)) * param.grad
+      end
+
+      def to_hash
+        super(eps: @eps)
       end
     end
     
-    
+
     class RMSProp < Optimizer
+      # @return [Float] Return the alpha value.
       attr_accessor :alpha
+      # @return [Float] Return the eps value.
+      attr_accessor :eps
 
-      def self.load_hash(hash)
-        self.new(hash[:learning_rate], alpha: hash[:alpha])
+      def self.from_hash(hash)
+        self.new(hash[:learning_rate], alpha: hash[:alpha], eps: hash[:eps])
       end
-    
-      def initialize(learning_rate = 0.001, alpha: 0.9)
+
+      # @param [Float] learning_rate Learning rate.
+      # @param [Float] alpha Moving average index of past slopes.
+      # @param [Float] eps Value to avoid division by zero.
+      def initialize(learning_rate = 0.001, alpha: 0.9, eps: 1e-7)
         super(learning_rate)
         @alpha = alpha
+        @eps = eps
         @g = {}
       end
-    
-      def update(params)
-        params.select { |key, param| param.grad }.each_value do |param|
-          @g[param] ||= 0
-          @g[param] = @alpha * @g[param] + (1 - @alpha) * param.grad**2
-          param.data -= (@learning_rate / NMath.sqrt(@g[param] + 1e-7)) * param.grad
-        end
-      end
 
       def to_hash
-        super({alpha: @alpha})
+        super(alpha: @alpha, eps: @eps)
+      end
+
+      private def update_param(param)
+        @g[param] ||= 0
+        @g[param] = @alpha * @g[param] + (1 - @alpha) * param.grad**2
+        param.data -= (@learning_rate / NMath.sqrt(@g[param] + @eps)) * param.grad
       end
     end
 
 
     class AdaDelta < Optimizer
+      # @return [Float] Return the rho value.
       attr_accessor :rho
+      # @return [Float] Return the eps value.
+      attr_accessor :eps
 
-      def self.load_hash(hash)
-        self.new(rho: hash[:rho])
+      def self.from_hash(hash)
+        self.new(rho: hash[:rho], eps: hash[:eps])
       end
 
-      def initialize(rho: 0.95)
+      # @param [Float] rho Moving average index of past slopes.
+      # @param [Float] eps Value to avoid division by zero.
+      def initialize(rho: 0.95, eps: 1e-6)
         super(nil)
         @rho = rho
+        @eps = eps
         @h = {}
         @s = {}
       end
 
-      def update(params)
-        params.select { |key, param| param.grad }.each_value do |param|
-          @h[param] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @s[param] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @h[param] = @rho * @h[param] + (1 - @rho) * param.grad**2
-          v = (NMath.sqrt(@s[param] + 1e-6) / NMath.sqrt(@h[param] + 1e-6)) * param.grad
-          @s[param] = @rho * @s[param] + (1 - @rho) * v**2
-          param.data -= v
-        end
+      def to_hash
+        super(rho: @rho, eps: @eps)
       end
 
-      def to_hash
-        super({rho: @rho})
+      private def update_param(param)
+        @h[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+        @s[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+        @h[param] = @rho * @h[param] + (1 - @rho) * param.grad**2
+        v = (NMath.sqrt(@s[param] + @eps) / NMath.sqrt(@h[param] + @eps)) * param.grad
+        @s[param] = @rho * @s[param] + (1 - @rho) * v**2
+        param.data -= v
       end
     end
 
 
     class Adam < Optimizer
+      # @return [Float] Return the alpha value.
+      attr_accessor :alpha
+      # @return [Float] Return the beta1 value.
       attr_accessor :beta1
+      # @return [Float] Return the beta2 value.
       attr_accessor :beta2
+      # @return [Float] Return the eps value.
+      attr_accessor :eps
       
-      def self.load_hash(hash)
-        self.new(hash[:learning_rate], beta1: hash[:beta1], beta2: hash[:beta2])
+      def self.from_hash(hash)
+        self.new(alpha: hash[:alpha], beta1: hash[:beta1], beta2: hash[:beta2], eps: hash[:eps])
       end
 
-      def initialize(learning_rate = 0.001, beta1: 0.9, beta2: 0.999)
-        super(learning_rate)
+      # @param [Float] alpha Value used to calculate learning rate.
+      # @param [Float] beta1 Moving average index of beta1.
+      # @param [Float] beta2 Moving average index of beta2.
+      # @param [Float] eps Value to avoid division by zero.
+      def initialize(alpha: 0.001, beta1: 0.9, beta2: 0.999, eps: 1e-7)
+        super(nil)
+        @alpha = alpha
         @beta1 = beta1
         @beta2 = beta2
+        @eps = eps
         @iter = 0
         @m = {}
         @v = {}
       end
 
-      def update(params)
+      def update(layers)
         @iter += 1
-        lr = @learning_rate * Math.sqrt(1 - @beta2**@iter) / (1 - @beta1**@iter) 
-        params.select { |key, param| param.grad }.each_value do |param|
-          @m[param] ||= 0
-          @v[param] ||= 0
-          @m[param] += (1 - @beta1) * (param.grad - @m[param])
-          @v[param] += (1 - @beta2) * (param.grad**2 - @v[param])
-          param.data -= lr * @m[param] / NMath.sqrt(@v[param] + 1e-7)
+        learning_rate = @alpha * Math.sqrt(1 - @beta2**@iter) / (1 - @beta1**@iter) 
+        target_params = layers.select { |layer| layer.is_a?(HasParamLayer) && layer.trainable }
+                              .map { |layer| layer.params.values }.flatten
+                              .select { |param| param.grad }
+        target_params.each do |param|
+          update_param(param, learning_rate)
+          param.grad = 0
         end
       end
 
       def to_hash
-        super({beta1: @beta1, beta2: @beta2})
+        super(alpha: @alpha, beta1: @beta1, beta2: @beta2, eps: @eps)
+      end
+
+      private def update_param(param, learning_rate)
+        @m[param] ||= 0
+        @v[param] ||= 0
+        @m[param] += (1 - @beta1) * (param.grad - @m[param])
+        @v[param] += (1 - @beta2) * (param.grad**2 - @v[param])
+        param.data -= learning_rate * @m[param] / NMath.sqrt(@v[param] + @eps)
+      end
+    end
+
+
+    class RMSPropGraves < Optimizer
+      # @return [Float] Return the alpha value.
+      attr_accessor :alpha
+      # @return [Float] Return the eps value.
+      attr_accessor :eps
+      
+      def self.from_hash(hash)
+        self.new(hash[:learning_rate], alpha: hash[:alpha], eps: hash[:eps])
+      end
+
+      # @param [Float] learning_rate Learning rate.
+      # @param [Float] alpha Moving average index of past slopes.
+      # @param [Float] eps Value to avoid division by zero.
+      def initialize(learning_rate = 0.0001, alpha: 0.95, eps: 0.0001)
+        super(learning_rate)
+        @alpha = alpha
+        @eps = eps
+        @m = {}
+        @v = {}
+      end
+
+      def to_hash
+        super(alpha: @alpha, eps: @eps)
+      end
+
+      private def update_param(param)
+        @m[param] ||= 0
+        @v[param] ||= 0
+        @m[param] = @alpha * @m[param] + (1 - @alpha) * param.grad
+        @v[param] = @alpha * @v[param] + (1 - @alpha) * param.grad**2
+        param.data -= (@learning_rate / NMath.sqrt(@v[param] - @m[param]**2 + @eps)) * param.grad
       end
     end