ruby-dnn 0.14.3 → 0.15.0

data/lib/dnn/core/losses.rb

@@ -53,6 +53,14 @@ module DNN
       def load_hash(hash)
         initialize
       end
+
+      def clean
+        hash = to_hash
+        instance_variables.each do |ivar|
+          instance_variable_set(ivar, nil)
+        end
+        load_hash(hash)
+      end
     end
 
     class MeanSquaredError < Loss

data/lib/dnn/core/models.rb

@@ -1,7 +1,97 @@
 module DNN
   module Models
+
+    class LayersList < Array
+      def self.from_hash_list(hash_list)
+        layers_list = new
+        hash_list.each do |hash|
+          obj_class = DNN.const_get(hash[:class])
+          obj = obj_class.allocate
+          if obj.is_a?(Chain)
+            obj = obj_class.new
+            obj.load_hash(hash)
+          else
+            obj = Layers::Layer.from_hash(hash)
+          end
+          layers_list << obj
+        end
+        layers_list
+      end
+
+      def to_hash_list
+        map { |layer| layer.to_hash }
+      end
+
+      # Get the all layers.
+      # @return [Array] All layers array.
+      def layers
+        layers_array = []
+        each do |layer|
+          if layer.is_a?(Layers::Layer)
+            layers_array << layer
+          elsif layer.is_a?(Chain) || layer.is_a?(LayersList)
+            layers_array.concat(layer.layers)
+          end
+        end
+        layers_array
+      end
+    end
+
+    class Chain
+      def call(x)
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'call'"
+      end
+
+      # Get the all layers.
+      # @return [Array] All layers array.
+      def layers
+        layers_array = []
+        instance_variables.sort.each do |ivar|
+          obj = instance_variable_get(ivar)
+          if obj.is_a?(Layers::Layer)
+            layers_array << obj
+          elsif obj.is_a?(Chain) || obj.is_a?(LayersList)
+            layers_array.concat(obj.layers)
+          end
+        end
+        layers_array
+      end
+
+      def to_hash
+        layers_hash = { class: self.class.name }
+        instance_variables.sort.each do |ivar|
+          obj = instance_variable_get(ivar)
+          if obj.is_a?(Layers::Layer) || obj.is_a?(Chain)
+            layers_hash[ivar] = obj.to_hash
+          elsif obj.is_a?(LayersList)
+            layers_hash[ivar] = obj.to_hash_list
+          end
+        end
+        layers_hash
+      end
+
+      def load_hash(layers_hash)
+        instance_variables.sort.each do |ivar|
+          hash_or_array = layers_hash[ivar]
+          if hash_or_array.is_a?(Array)
+            instance_variable_set(ivar, LayersList.from_hash_list(hash_or_array))
+          elsif hash_or_array.is_a?(Hash)
+            obj_class = DNN.const_get(hash_or_array[:class])
+            obj = obj_class.allocate
+            if obj.is_a?(Chain)
+              obj = obj_class.new
+              obj.load_hash(hash_or_array)
+              instance_variable_set(ivar, obj)
+            else
+              instance_variable_set(ivar, Layers::Layer.from_hash(hash_or_array))
+            end
+          end
+        end
+      end
+    end
+
     # This class deals with the model of the network.
-    class Model
+    class Model < Chain
       attr_accessor :optimizer
       attr_accessor :loss_func
       attr_reader :last_log
@@ -10,7 +100,7 @@ module DNN
       # @param [String] file_name File name of marshal model to load.
       # @return [DNN::Models::Model] Return the loaded model.
       def self.load(file_name)
-        model = new
+        model = self.allocate
         loader = Loaders::MarshalLoader.new(model)
         loader.load(file_name)
         model
@@ -113,8 +203,12 @@ module DNN
             end
 
             if test
-              test_met = test(test[0], test[1], batch_size: batch_size)
-              print "  " + metrics_to_str(test_met) if verbose
+              acc, loss = if test.is_a?(Array)
+                evaluate(test[0], test[1], batch_size: batch_size)
+              else
+                evaluate_by_iterator(test, batch_size: batch_size)
+              end
+              print "  " + metrics_to_str({ accuracy: acc, test_loss: loss }) if verbose
             end
             puts "" if verbose
             call_callbacks(:after_epoch)
@@ -138,16 +232,6 @@ module DNN
         { loss: loss_value }
       end
 
-      # Implement the test process to be performed.
-      # @param [Numo::SFloat] x Input training data.
-      # @param [Numo::SFloat] y Output training data.
-      # @param [Integer] batch_size Batch size used for one test.
-      # @return [Hash] Hash of contents to be output to log.
-      private def test(x, y, batch_size: 100)
-        acc, test_loss = accuracy(x, y, batch_size: batch_size)
-        { accuracy: acc, test_loss: test_loss }
-      end
-
       # Training once.
       # Setup the model before use this method.
       # @param [Numo::SFloat] x Input training data.
@@ -169,20 +253,24 @@ module DNN
         loss_value
       end
 
-      # Evaluate model and get accuracy of test data.
+      # Evaluate model and get accuracy and loss of test data.
       # @param [Numo::SFloat] x Input test data.
       # @param [Numo::SFloat] y Output test data.
       # @param [Integer] batch_size Batch size used for one test.
       # @return [Array] Returns the test data accuracy and mean loss in the form [accuracy, mean_loss].
-      def accuracy(x, y, batch_size: 100)
+      def evaluate(x, y, batch_size: 100)
         check_xy_type(x, y)
-        num_test_datas = x.is_a?(Array) ? x[0].shape[0] : x.shape[0]
+        evaluate_by_iterator(Iterator.new(x, y, random: false))
+      end
+
+      # Evaluate model by iterator
+      def evaluate_by_iterator(test_iterator, batch_size: 100)
+        num_test_datas = test_iterator.num_datas
         batch_size = batch_size >= num_test_datas[0] ? num_test_datas : batch_size
-        iter = Iterator.new(x, y, random: false)
         total_correct = 0
         sum_loss = 0
         max_steps = (num_test_datas.to_f / batch_size).ceil
-        iter.foreach(batch_size) do |x_batch, y_batch|
+        test_iterator.foreach(batch_size) do |x_batch, y_batch|
           correct, loss_value = test_on_batch(x_batch, y_batch)
           total_correct += correct
           sum_loss += loss_value
@@ -201,16 +289,16 @@ module DNN
       def test_on_batch(x, y)
         call_callbacks(:before_test_on_batch)
         x = forward(x, false)
-        correct = evaluate(x, y)
+        correct = accuracy(x, y)
         loss_value = @loss_func.loss(x, y)
         call_callbacks(:after_test_on_batch)
         [correct, loss_value]
       end
 
-      # Implement the process to evaluate this model.
+      # Implement the process to accuracy this model.
       # @param [Numo::SFloat] x Input test data.
       # @param [Numo::SFloat] y Output test data.
-      private def evaluate(x, y)
+      private def accuracy(x, y)
         if x.shape[1..-1] == [1]
           correct = 0
           x.shape[0].times do |i|
@@ -257,11 +345,24 @@ module DNN
         @callbacks = []
       end
 
+      # Load marshal params.
+      # @param [String] file_name File name of marshal model to load.
+      def load_params(file_name)
+        loader = Loaders::MarshalLoader.new(self)
+        loader.load(file_name)
+      end
+
       # Save the model in marshal format.
       # @param [String] file_name Name to save model.
-      # @param [Boolean] include_optimizer Set true to save data included optimizer status.
-      def save(file_name, include_optimizer: true)
-        saver = Savers::MarshalSaver.new(self, include_optimizer: include_optimizer)
+      def save(file_name)
+        saver = Savers::MarshalSaver.new(self, include_model: true)
+        saver.save(file_name)
+      end
+
+      # Save the params in marshal format.
+      # @param [String] file_name Name to save model.
+      def save_params(file_name)
+        saver = Savers::MarshalSaver.new(self, include_model: false)
         saver.save(file_name)
       end
 
@@ -270,37 +371,21 @@ module DNN
         Marshal.load(Marshal.dump(self))
       end
 
-      # Get the all layers.
-      # @return [Array] All layers array.
-      def layers
-        raise DNN_Error, "This model is not built. You need build this model using predict or train." unless built?
-        return @layers_cache if @layers_cache
-        layers = []
-        get_layers = -> link do
-          return unless link
-          layers.unshift(link.layer)
-          if link.is_a?(TwoInputLink)
-            get_layers.(link.prev1)
-            get_layers.(link.prev2)
-          else
-            get_layers.(link.prev)
-          end
-        end
-        get_layers.(@last_link)
-        @layers_cache = layers.uniq
-      end
-
-      # Get the all has param layers.
+      # Get the all trainable layers.
       # @return [Array] All has param layers array.
-      def has_param_layers
-        layers.select { |layer| layer.is_a?(Layers::HasParamLayer) }
+      def trainable_layers
+        layers.select { |layer| layer.is_a?(Layers::TrainableLayer) }
       end
 
       # Get the layer that the model has.
       # @param [Symbol] name The name of the layer to get.
       # @return [DNN::Layers::Layer] Return the layer.
       def get_layer(name)
-        layers.find { |layer| layer.name == name }
+        layer = instance_variable_get("@#{name}")
+        if layer.is_a?(Layers::Layer) || layer.is_a?(Chain) || layer.is_a?(LayersList)
+          return layer
+        end
+        nil
       end
 
       # @return [Boolean] If model have already been built then return true.
@@ -308,6 +393,31 @@ module DNN
         @built
       end
 
+      def clean_layers
+        layers.each do |layer|
+          layer.clean
+        end
+        @loss_func.clean
+        @last_link = nil
+        @layers_cache = nil
+      end
+
+      def get_all_params_data
+        trainable_layers.map do |layer|
+          layer.get_params.to_h do |key, param|
+            [key, param.data]
+          end
+        end
+      end
+
+      def set_all_params_data(params_data)
+        trainable_layers.each.with_index do |layer, i|
+          params_data[i].each do |(key, data)|
+            layer.get_params[key].data = data
+          end
+        end
+      end
+
       private
 
       def forward(x, learning_phase)
@@ -322,7 +432,6 @@ module DNN
         @last_link = output_tensor.link
         unless @built
           @built = true
-          naming
         end
         output_tensor.data
       end
@@ -337,19 +446,6 @@ module DNN
         end
       end
 
-      def naming
-        layers.each do |layer|
-          id = layers.select { |l| l.is_a?(layer.class) }.index(layer)
-          class_name = layer.class.name.split("::").last
-          layer.name = "#{class_name}_#{id}".to_sym unless layer.name
-          if layer.is_a?(Layers::HasParamLayer)
-            layer.get_params.each do |param_key, param|
-              param.name = "#{layer.name}__#{param_key}".to_sym unless param.name
-            end
-          end
-        end
-      end
-
       def metrics_to_str(mertics)
         mertics.map { |key, num| "#{key}: #{sprintf('%.4f', num)}" }.join(", ")
       end
@@ -370,7 +466,7 @@ module DNN
       # @param [Array] stack All layers possessed by the model.
       def initialize(stack = [])
         super()
-        @stack = []
+        @stack = LayersList.new
         stack.each do |layer|
           add(layer)
         end
@@ -380,8 +476,8 @@ module DNN
       # @param [DNN::Layers::Layer] layer Layer to add to the model.
       # @return [DNN::Models::Model] Return self.
       def add(layer)
-        if layer.is_a?(MergeLayers::MergeLayer)
-          raise TypeError, "layer: #{layer.class.name} should not be a DNN::MergeLayers::MergeLayer class."
+        if layer.is_a?(Layers::MergeLayer)
+          raise TypeError, "layer: #{layer.class.name} should not be a DNN::Layers::MergeLayer class."
         end
         unless layer.is_a?(Layers::Layer) || layer.is_a?(Model)
           raise TypeError, "layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Model class."
@@ -396,8 +492,8 @@ module DNN
       # @param [DNN::Layers::Layer] layer Layer to add to the model.
       # @return [DNN::Models::Model] Return self.
       def insert(index, layer)
-        if layer.is_a?(MergeLayers::MergeLayer)
-          raise TypeError, "layer: #{layer.class.name} should not be a DNN::MergeLayers::MergeLayer class."
+        if layer.is_a?(Layers::MergeLayer)
+          raise TypeError, "layer: #{layer.class.name} should not be a DNN::Layers::MergeLayer class."
         end
         unless layer.is_a?(Layers::Layer) || layer.is_a?(Model)
           raise TypeError, "layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Model class."

data/lib/dnn/core/optimizers.rb

@@ -3,7 +3,6 @@ module DNN
 
     # Super class of all optimizer classes.
     class Optimizer
-      attr_reader :status
       attr_accessor :clip_norm
 
       def self.from_hash(hash)
@@ -15,17 +14,6 @@ module DNN
         optimizer
       end
 
-      def self.load(dumped)
-        opt = from_hash(dumped[:hash])
-        return opt unless dumped[:status]
-        dumped[:status].each do |key, state|
-          state = state.clone
-          opt.status[key] = state
-          opt.instance_variable_set("@#{key}", state)
-        end
-        opt
-      end
-
       # @param [Float | NilClass] clip_norm Gradient clip norm.
       def initialize(clip_norm: nil)
         @clip_norm = clip_norm
@@ -33,7 +21,7 @@ module DNN
 
       # Update layers has params.
       def update(layers)
-        target_params = layers.select { |layer| layer.is_a?(Layers::HasParamLayer) && layer.trainable }
+        target_params = layers.select { |layer| layer.is_a?(Layers::TrainableLayer) && layer.trainable }
                               .map { |layer| layer.get_params.values }.flatten.compact
                               .select(&:grad)
         clip_grads(target_params) if @clip_norm
@@ -43,11 +31,6 @@ module DNN
         end
       end
 
-      def dump(require_status = true)
-        status = require_status ? @status : nil
-        { hash: to_hash, status: status }
-      end
-
       def to_hash(merge_hash = nil)
         hash = { class: self.class.name, clip_norm: @clip_norm }
         hash.merge!(merge_hash) if merge_hash
@@ -80,12 +63,11 @@ module DNN
 
       # @param [Float] lr Learning rate.
       # @param [Float] momentum Momentum coefficient.
-      def initialize(lr = 0.01, momentum: 0, clip_norm: nil)
+      def initialize(lr: 0.01, momentum: 0, clip_norm: nil)
         super(clip_norm: clip_norm)
         @lr = lr
         @momentum = momentum
         @v = {}
-        @status = { v: @v }
       end
 
       def to_hash
@@ -96,30 +78,30 @@ module DNN
         params.each do |param|
           amount = param.grad * @lr
           if @momentum > 0
-            @v[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-            amount += @momentum * @v[param.name]
-            @v[param.name] = amount
+            @v[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+            amount += @momentum * @v[param]
+            @v[param] = amount
           end
           param.data -= amount
         end
       end
 
       def load_hash(hash)
-        initialize(hash[:lr], momentum: hash[:momentum], clip_norm: hash[:clip_norm])
+        initialize(lr: hash[:lr], momentum: hash[:momentum], clip_norm: hash[:clip_norm])
       end
     end
 
     class Nesterov < SGD
-      def initialize(lr = 0.01, momentum: 0.9, clip_norm: nil)
-        super(lr, momentum: momentum, clip_norm: clip_norm)
+      def initialize(lr: 0.01, momentum: 0.9, clip_norm: nil)
+        super(lr: lr, momentum: momentum, clip_norm: clip_norm)
       end
 
       private def update_params(params)
         params.each do |param|
-          @v[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @v[param] ||= Xumo::SFloat.zeros(*param.data.shape)
           amount = param.grad * @lr
-          @v[param.name] = @v[param.name] * @momentum - amount
-          param.data = (param.data + @momentum**2 * @v[param.name]) - (1 + @momentum) * amount
+          @v[param] = @v[param] * @momentum - amount
+          param.data = (param.data + @momentum**2 * @v[param]) - (1 + @momentum) * amount
         end
       end
     end
@@ -130,19 +112,18 @@ module DNN
 
       # @param [Float] lr Learning rate.
       # @param [Float] eps Value to avoid division by zero.
-      def initialize(lr = 0.01, eps: 1e-7, clip_norm: nil)
+      def initialize(lr: 0.01, eps: 1e-7, clip_norm: nil)
         super(clip_norm: clip_norm)
         @lr = lr
         @eps = eps
         @g = {}
-        @status = { g: @g }
       end
 
       private def update_params(params)
         params.each do |param|
-          @g[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @g[param.name] += param.grad**2
-          param.data -= (@lr / Xumo::NMath.sqrt(@g[param.name] + @eps)) * param.grad
+          @g[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @g[param] += param.grad**2
+          param.data -= (@lr / Xumo::NMath.sqrt(@g[param] + @eps)) * param.grad
         end
       end
 
@@ -151,7 +132,7 @@ module DNN
       end
 
       def load_hash(hash)
-        initialize(hash[:lr], eps: hash[:eps], clip_norm: hash[:clip_norm])
+        initialize(lr: hash[:lr], eps: hash[:eps], clip_norm: hash[:clip_norm])
       end
     end
 
@@ -163,13 +144,12 @@ module DNN
       # @param [Float] lr Learning rate.
       # @param [Float] alpha Moving average index of past slopes.
       # @param [Float] eps Value to avoid division by zero.
-      def initialize(lr = 0.001, alpha: 0.9, eps: 1e-7, clip_norm: nil)
+      def initialize(lr: 0.001, alpha: 0.9, eps: 1e-7, clip_norm: nil)
         super(clip_norm: clip_norm)
         @lr = lr
         @alpha = alpha
         @eps = eps
         @g = {}
-        @status = { g: @g }
       end
 
       def to_hash
@@ -178,14 +158,14 @@ module DNN
 
       private def update_params(params)
         params.each do |param|
-          @g[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @g[param.name] = @alpha * @g[param.name] + (1 - @alpha) * param.grad**2
-          param.data -= (@lr / Xumo::NMath.sqrt(@g[param.name] + @eps)) * param.grad
+          @g[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @g[param] = @alpha * @g[param] + (1 - @alpha) * param.grad**2
+          param.data -= (@lr / Xumo::NMath.sqrt(@g[param] + @eps)) * param.grad
         end
       end
 
       def load_hash(hash)
-        initialize(hash[:lr], alpha: hash[:alpha], eps: hash[:eps], clip_norm: hash[:clip_norm])
+        initialize(lr: hash[:lr], alpha: hash[:alpha], eps: hash[:eps], clip_norm: hash[:clip_norm])
       end
     end
 
@@ -201,7 +181,6 @@ module DNN
         @eps = eps
         @h = {}
         @s = {}
-        @status = { h: @h, s: @s }
       end
 
       def to_hash
@@ -210,11 +189,11 @@ module DNN
 
       private def update_params(params)
         params.each do |param|
-          @h[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @s[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @h[param.name] = @rho * @h[param.name] + (1 - @rho) * param.grad**2
-          v = (Xumo::NMath.sqrt(@s[param.name] + @eps) / Xumo::NMath.sqrt(@h[param.name] + @eps)) * param.grad
-          @s[param.name] = @rho * @s[param.name] + (1 - @rho) * v**2
+          @h[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @s[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @h[param] = @rho * @h[param] + (1 - @rho) * param.grad**2
+          v = (Xumo::NMath.sqrt(@s[param] + @eps) / Xumo::NMath.sqrt(@h[param] + @eps)) * param.grad
+          @s[param] = @rho * @s[param] + (1 - @rho) * v**2
           param.data -= v
         end
       end
@@ -232,14 +211,13 @@ module DNN
       # @param [Float] lr Learning rate.
       # @param [Float] alpha Moving average index of past slopes.
       # @param [Float] eps Value to avoid division by zero.
-      def initialize(lr = 0.0001, alpha: 0.95, eps: 0.0001, clip_norm: nil)
+      def initialize(lr: 0.0001, alpha: 0.95, eps: 0.0001, clip_norm: nil)
         super(clip_norm: clip_norm)
         @lr = lr
         @alpha = alpha
         @eps = eps
         @m = {}
         @v = {}
-        @status = { m: @m, v: @v }
       end
 
       def to_hash
@@ -248,16 +226,16 @@ module DNN
 
       private def update_params(params)
         params.each do |param|
-          @m[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @v[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @m[param.name] = @alpha * @m[param.name] + (1 - @alpha) * param.grad
-          @v[param.name] = @alpha * @v[param.name] + (1 - @alpha) * param.grad**2
-          param.data -= (@lr / Xumo::NMath.sqrt(@v[param.name] - @m[param.name]**2 + @eps)) * param.grad
+          @m[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @v[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @m[param] = @alpha * @m[param] + (1 - @alpha) * param.grad
+          @v[param] = @alpha * @v[param] + (1 - @alpha) * param.grad**2
+          param.data -= (@lr / Xumo::NMath.sqrt(@v[param] - @m[param]**2 + @eps)) * param.grad
         end
       end
 
       def load_hash(hash)
-        initialize(hash[:lr], alpha: hash[:alpha], eps: hash[:eps], clip_norm: hash[:clip_norm])
+        initialize(lr: hash[:lr], alpha: hash[:alpha], eps: hash[:eps], clip_norm: hash[:clip_norm])
       end
     end
 
@@ -284,7 +262,6 @@ module DNN
         @m = {}
         @v = {}
         @s = amsgrad ? {} : nil
-        @status = { t: @t, m: @m, v: @v, s: @s }
       end
 
       def to_hash
@@ -298,16 +275,16 @@ module DNN
         @t += 1
         lr = @alpha * Math.sqrt(1 - @beta2**@t) / (1 - @beta1**@t)
         params.each do |param|
-          @m[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @v[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @m[param.name] += (1 - @beta1) * (param.grad - @m[param.name])
-          @v[param.name] += (1 - @beta2) * (param.grad**2 - @v[param.name])
+          @m[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @v[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @m[param] += (1 - @beta1) * (param.grad - @m[param])
+          @v[param] += (1 - @beta2) * (param.grad**2 - @v[param])
           if @amsgrad
-            @s[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-            @s[param.name] = Xumo::SFloat.maximum(@s[param.name], @v[param.name])
-            param.data -= lr * @m[param.name] / Xumo::NMath.sqrt(@s[param.name] + @eps)
+            @s[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+            @s[param] = Xumo::SFloat.maximum(@s[param], @v[param])
+            param.data -= lr * @m[param] / Xumo::NMath.sqrt(@s[param] + @eps)
           else
-            param.data -= lr * @m[param.name] / Xumo::NMath.sqrt(@v[param.name] + @eps)
+            param.data -= lr * @m[param] / Xumo::NMath.sqrt(@v[param] + @eps)
           end
         end
       end
@@ -344,16 +321,16 @@ module DNN
         lower_bound = final_lr * (1 - 1 / (@gamma * @t + 1))
         upper_bound = final_lr * (1 + 1 / (@gamma * @t))
         params.each do |param|
-          @m[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @v[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @m[param.name] += (1 - @beta1) * (param.grad - @m[param.name])
-          @v[param.name] += (1 - @beta2) * (param.grad**2 - @v[param.name])
+          @m[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @v[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @m[param] += (1 - @beta1) * (param.grad - @m[param])
+          @v[param] += (1 - @beta2) * (param.grad**2 - @v[param])
           if @amsgrad
-            @s[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-            @s[param.name] = Xumo::SFloat.maximum(@s[param.name], @v[param.name])
-            param.data -= clip_lr(lr / (Xumo::NMath.sqrt(@s[param.name]) + @eps), lower_bound, upper_bound) * @m[param.name]
+            @s[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+            @s[param] = Xumo::SFloat.maximum(@s[param], @v[param])
+            param.data -= clip_lr(lr / (Xumo::NMath.sqrt(@s[param]) + @eps), lower_bound, upper_bound) * @m[param]
           else
-            param.data -= clip_lr(lr / (Xumo::NMath.sqrt(@v[param.name]) + @eps), lower_bound, upper_bound) * @m[param.name]
+            param.data -= clip_lr(lr / (Xumo::NMath.sqrt(@v[param]) + @eps), lower_bound, upper_bound) * @m[param]
           end
         end
       end