ruby-dnn 0.10.4 → 0.12.4

data/lib/dnn/core/normalizations.rb

@@ -2,18 +2,19 @@ module DNN
   module Layers
 
     class BatchNormalization < HasParamLayer
-      # @return [Integer] The axis to normalization.
+      attr_reader :gamma
+      attr_reader :beta
+      attr_reader :running_mean
+      attr_reader :running_var
       attr_reader :axis
-      # @return [Float] Exponential moving average of mean and variance.
       attr_accessor :momentum
-      # @return [Float] Value to avoid division by zero.
       attr_accessor :eps
 
       def self.from_hash(hash)
         self.new(axis: hash[:axis], momentum: hash[:momentum])
       end
 
-      # @param [integer] axis The axis to normalization.
+      # @param [Integer] axis The axis to normalization.
       # @param [Float] momentum Exponential moving average of mean and variance.
       # @param [Float] eps Value to avoid division by zero.
       def initialize(axis: 0, momentum: 0.9, eps: 1e-7)
@@ -23,27 +24,35 @@ module DNN
         @eps = eps
       end
 
+      def call(input)
+        x, prev_link, learning_phase = *input
+        build(x.shape[1..-1]) unless built?
+        y = forward(x, learning_phase)
+        link = Link.new(prev_link, self)
+        [y, link, learning_phase]
+      end
+
       def build(input_shape)
         super
-        @params[:gamma] = @gamma = Param.new(Xumo::SFloat.ones(*output_shape), 0)
-        @params[:beta] = @beta = Param.new(Xumo::SFloat.zeros(*output_shape), 0)
-        @params[:running_mean] = @running_mean = Param.new(Xumo::SFloat.zeros(*output_shape))
-        @params[:running_var] = @running_var = Param.new(Xumo::SFloat.zeros(*output_shape))
+        @gamma = Param.new(Xumo::SFloat.ones(*output_shape), 0)
+        @beta = Param.new(Xumo::SFloat.zeros(*output_shape), 0)
+        @running_mean = Param.new(Xumo::SFloat.zeros(*output_shape))
+        @running_var = Param.new(Xumo::SFloat.zeros(*output_shape))
       end
 
-      def forward(x)
+      def forward(x, learning_phase)
         if learning_phase
           mean = x.mean(axis: @axis, keepdims: true)
           @xc = x - mean
-          var = (@xc**2).mean(axis: @axis, keepdims: true)
-          @std = NMath.sqrt(var + @eps)
+          var = (@xc ** 2).mean(axis: @axis, keepdims: true)
+          @std = Xumo::NMath.sqrt(var + @eps)
           xn = @xc / @std
           @xn = xn
           @running_mean.data = @momentum * @running_mean.data + (1 - @momentum) * mean
           @running_var.data = @momentum * @running_var.data + (1 - @momentum) * var
         else
           xc = x - @running_mean.data
-          xn = xc / NMath.sqrt(@running_var.data + @eps)
+          xn = xc / Xumo::NMath.sqrt(@running_var.data + @eps)
         end
         @gamma.data * xn + @beta.data
       end
@@ -56,7 +65,7 @@ module DNN
         end
         dxn = @gamma.data * dy
         dxc = dxn / @std
-        dstd = -((dxn * @xc) / (@std**2)).sum(axis: @axis, keepdims: true)
+        dstd = -((dxn * @xc) / (@std ** 2)).sum(axis: @axis, keepdims: true)
         dvar = 0.5 * dstd / @std
         dxc += (2.0 / batch_size) * @xc * dvar
         dmean = dxc.sum(axis: @axis, keepdims: true)
@@ -64,7 +73,11 @@ module DNN
       end
 
       def to_hash
-        super({axis: @axis, momentum: @momentum, eps: @eps})
+        super(axis: @axis, momentum: @momentum, eps: @eps)
+      end
+
+      def get_params
+        { gamma: @gamma, beta: @beta, running_mean: @running_mean, running_var: @running_var }
       end
     end
 

data/lib/dnn/core/optimizers.rb

@@ -3,172 +3,191 @@ module DNN
 
     # Super class of all optimizer classes.
     class Optimizer
-      # @return [Float] Return the Learning rate.
-      attr_accessor :learning_rate
+      attr_accessor :clip_norm
 
-      def initialize(learning_rate)
-        @learning_rate = learning_rate
+      # @param [Float | NilClass] clip_norm Gradient clip norm.
+      def initialize(clip_norm: nil)
+        @clip_norm = clip_norm
       end
 
-      # Update layers has param.
+      # Update layers has params.
       def update(layers)
-        target_params = layers.select { |layer| layer.is_a?(HasParamLayer) && layer.trainable }
-                              .map { |layer| layer.params.values }.flatten
+        target_params = layers.select { |layer| layer.is_a?(Layers::HasParamLayer) && layer.trainable }
+                              .map { |layer| layer.get_params.values }.flatten.compact
                               .select { |param| param.grad }
+        clip_grads(target_params) if @clip_norm
+        update_params(target_params)
         target_params.each do |param|
-          update_param(param)
-          param.grad = 0
+          param.grad = Xumo::SFloat.zeros(*param.data.shape)
         end
       end
 
       def to_hash(merge_hash = nil)
-        hash = {class: self.class.name, learning_rate: @learning_rate}
+        hash = { class: self.class.name, clip_norm: @clip_norm }
         hash.merge!(merge_hash) if merge_hash
         hash
       end
 
-      # Update param.
-      # Classes that inherit from this class must implement this method.
-      private def update_param(param)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'update_param'")
+      # Update params.
+      private def update_params(params)
+        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'update_params'")
+      end
+
+      private def clip_grads(params)
+        norm = Math.sqrt(params.reduce(0) { |sum, param| sum + (param.grad == 0 ? 0 : (param.grad ** 2).sum) })
+        return if norm <= @clip_norm
+        rate = @clip_norm / (norm + 1e-7)
+        params.each do |param|
+          param.grad *= rate
+        end
       end
     end
 
 
     class SGD < Optimizer
-      # @return [Float] Return the momentum coefficient.
+      attr_accessor :lr
       attr_accessor :momentum
 
       def self.from_hash(hash)
-        self.new(hash[:learning_rate], momentum: hash[:momentum])
+        self.new(hash[:lr], momentum: hash[:momentum], clip_norm: hash[:clip_norm])
       end
 
-      # @param [Float] learning_rate Learning rate.
-      # @param [Float] momentum momentum coefficient.
-      def initialize(learning_rate = 0.01, momentum: 0)
-        super(learning_rate)
+      # @param [Float] lr Learning rate.
+      # @param [Float] momentum Momentum coefficient.
+      def initialize(lr = 0.01, momentum: 0, clip_norm: nil)
+        super(clip_norm: clip_norm)
+        @lr = lr
         @momentum = momentum
         @v = {}
       end
 
       def to_hash
-        super(momentum: @momentum)
-      end
-
-      private def update_param(param)
-        amount = param.grad * @learning_rate
-        if @momentum > 0
-          @v[param] ||= 0
-          amount += @momentum * @v[param]
-          @v[param] = amount
+        super(lr: @lr, momentum: @momentum)
+      end
+
+      private def update_params(params)
+        params.each do |param|
+          amount = param.grad * @lr
+          if @momentum > 0
+            @v[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+            amount += @momentum * @v[param]
+            @v[param] = amount
+          end
+          param.data -= amount
         end
-        param.data -= amount
       end
     end
 
 
     class Nesterov < Optimizer
+      attr_accessor :lr
       attr_accessor :momentum
-      
+
       def self.from_hash(hash)
-        self.new(hash[:learning_rate], momentum: hash[:momentum])
+        self.new(hash[:lr], momentum: hash[:momentum], clip_norm: hash[:clip_norm])
       end
 
-      # @param [Float] learning_rate Learning rate.
-      # @param [Float] momentum momentum coefficient.
-      def initialize(learning_rate = 0.01, momentum: 0.9)
-        super(learning_rate)
+      # @param [Float] lr Learning rate.
+      # @param [Float] momentum Momentum coefficient.
+      def initialize(lr = 0.01, momentum: 0.9, clip_norm: nil)
+        super(clip_norm: clip_norm)
+        @lr = lr
         @momentum = momentum
         @v = {}
       end
 
       def to_hash
-        super(momentum: @momentum)
+        super(lr: @lr, momentum: @momentum)
       end
-    
-      private def update_param(param)
-        @v[param] ||= 0
-        amount = param.grad * @learning_rate
-        @v[param] = @v[param] * @momentum - amount
-        param.data = (param.data + @momentum**2 * @v[param]) - (1 + @momentum) * amount
+
+      private def update_params(params)
+        params.each do |param|
+          @v[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          amount = param.grad * @lr
+          @v[param] = @v[param] * @momentum - amount
+          param.data = (param.data + @momentum ** 2 * @v[param]) - (1 + @momentum) * amount
+        end
       end
     end
-    
-    
+
+
     class AdaGrad < Optimizer
-      # @return [Float] Return the eps value.
+      attr_accessor :lr
       attr_accessor :eps
 
-      # @param [Float] learning_rate Learning rate.
+      def self.from_hash(hash)
+        self.new(hash[:lr], eps: hash[:eps], clip_norm: hash[:clip_norm])
+      end
+
+      # @param [Float] lr Learning rate.
       # @param [Float] eps Value to avoid division by zero.
-      def initialize(learning_rate = 0.01, eps: 1e-7)
-        super(learning_rate)
+      def initialize(lr = 0.01, eps: 1e-7, clip_norm: nil)
+        super(clip_norm: clip_norm)
+        @lr = lr
         @eps = eps
         @g = {}
       end
 
-      def self.from_hash(hash)
-        self.new(hash[:learning_rate], eps: hash[:eps])
-      end
-    
-      private def update_param(param)
-        @g[param] ||= 0
-        @g[param] += param.grad**2
-        param.data -= (@learning_rate / NMath.sqrt(@g[param] + @eps)) * param.grad
+      private def update_params(params)
+        params.each do |param|
+          @g[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @g[param] += param.grad ** 2
+          param.data -= (@lr / Xumo::NMath.sqrt(@g[param] + @eps)) * param.grad
+        end
       end
 
       def to_hash
-        super(eps: @eps)
+        super(lr: @lr, eps: @eps)
       end
     end
-    
+
 
     class RMSProp < Optimizer
-      # @return [Float] Return the alpha value.
+      attr_accessor :lr
       attr_accessor :alpha
-      # @return [Float] Return the eps value.
       attr_accessor :eps
 
       def self.from_hash(hash)
-        self.new(hash[:learning_rate], alpha: hash[:alpha], eps: hash[:eps])
+        self.new(hash[:lr], alpha: hash[:alpha], eps: hash[:eps], clip_norm: hash[:clip_norm])
       end
 
-      # @param [Float] learning_rate Learning rate.
+      # @param [Float] lr Learning rate.
       # @param [Float] alpha Moving average index of past slopes.
       # @param [Float] eps Value to avoid division by zero.
-      def initialize(learning_rate = 0.001, alpha: 0.9, eps: 1e-7)
-        super(learning_rate)
+      def initialize(lr = 0.001, alpha: 0.9, eps: 1e-7, clip_norm: nil)
+        super(clip_norm: clip_norm)
+        @lr = lr
         @alpha = alpha
         @eps = eps
         @g = {}
       end
 
       def to_hash
-        super(alpha: @alpha, eps: @eps)
+        super(lr: @lr, alpha: @alpha, eps: @eps)
       end
 
-      private def update_param(param)
-        @g[param] ||= 0
-        @g[param] = @alpha * @g[param] + (1 - @alpha) * param.grad**2
-        param.data -= (@learning_rate / NMath.sqrt(@g[param] + @eps)) * param.grad
+      private def update_params(params)
+        params.each do |param|
+          @g[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @g[param] = @alpha * @g[param] + (1 - @alpha) * param.grad ** 2
+          param.data -= (@lr / Xumo::NMath.sqrt(@g[param] + @eps)) * param.grad
+        end
       end
     end
 
 
     class AdaDelta < Optimizer
-      # @return [Float] Return the rho value.
       attr_accessor :rho
-      # @return [Float] Return the eps value.
       attr_accessor :eps
 
       def self.from_hash(hash)
-        self.new(rho: hash[:rho], eps: hash[:eps])
+        self.new(rho: hash[:rho], eps: hash[:eps], clip_norm: hash[:clip_norm])
       end
 
       # @param [Float] rho Moving average index of past slopes.
       # @param [Float] eps Value to avoid division by zero.
-      def initialize(rho: 0.95, eps: 1e-6)
-        super(nil)
+      def initialize(rho: 0.95, eps: 1e-6, clip_norm: nil)
+        super(clip_norm: clip_norm)
         @rho = rho
         @eps = eps
         @h = {}
@@ -179,103 +198,162 @@ module DNN
         super(rho: @rho, eps: @eps)
       end
 
-      private def update_param(param)
-        @h[param] ||= Xumo::SFloat.zeros(*param.data.shape)
-        @s[param] ||= Xumo::SFloat.zeros(*param.data.shape)
-        @h[param] = @rho * @h[param] + (1 - @rho) * param.grad**2
-        v = (NMath.sqrt(@s[param] + @eps) / NMath.sqrt(@h[param] + @eps)) * param.grad
-        @s[param] = @rho * @s[param] + (1 - @rho) * v**2
-        param.data -= v
+      private def update_params(params)
+        params.each do |param|
+          @h[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @s[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @h[param] = @rho * @h[param] + (1 - @rho) * param.grad ** 2
+          v = (Xumo::NMath.sqrt(@s[param] + @eps) / Xumo::NMath.sqrt(@h[param] + @eps)) * param.grad
+          @s[param] = @rho * @s[param] + (1 - @rho) * v ** 2
+          param.data -= v
+        end
       end
     end
 
 
-    class Adam < Optimizer
-      # @return [Float] Return the alpha value.
+    class RMSPropGraves < Optimizer
+      attr_accessor :lr
       attr_accessor :alpha
-      # @return [Float] Return the beta1 value.
-      attr_accessor :beta1
-      # @return [Float] Return the beta2 value.
-      attr_accessor :beta2
-      # @return [Float] Return the eps value.
       attr_accessor :eps
-      
+
       def self.from_hash(hash)
-        self.new(alpha: hash[:alpha], beta1: hash[:beta1], beta2: hash[:beta2], eps: hash[:eps])
+        self.new(hash[:lr], alpha: hash[:alpha], eps: hash[:eps], clip_norm: hash[:clip_norm])
       end
 
-      # @param [Float] alpha Value used to calculate learning rate.
-      # @param [Float] beta1 Moving average index of beta1.
-      # @param [Float] beta2 Moving average index of beta2.
+      # @param [Float] lr Learning rate.
+      # @param [Float] alpha Moving average index of past slopes.
       # @param [Float] eps Value to avoid division by zero.
-      def initialize(alpha: 0.001, beta1: 0.9, beta2: 0.999, eps: 1e-7)
-        super(nil)
+      def initialize(lr = 0.0001, alpha: 0.95, eps: 0.0001, clip_norm: nil)
+        super(clip_norm: clip_norm)
+        @lr = lr
         @alpha = alpha
-        @beta1 = beta1
-        @beta2 = beta2
         @eps = eps
-        @iter = 0
         @m = {}
         @v = {}
       end
 
-      def update(layers)
-        @iter += 1
-        learning_rate = @alpha * Math.sqrt(1 - @beta2**@iter) / (1 - @beta1**@iter) 
-        target_params = layers.select { |layer| layer.is_a?(HasParamLayer) && layer.trainable }
-                              .map { |layer| layer.params.values }.flatten
-                              .select { |param| param.grad }
-        target_params.each do |param|
-          update_param(param, learning_rate)
-          param.grad = 0
-        end
-      end
-
       def to_hash
-        super(alpha: @alpha, beta1: @beta1, beta2: @beta2, eps: @eps)
+        super(lr: @lr, alpha: @alpha, eps: @eps)
       end
 
-      private def update_param(param, learning_rate)
-        @m[param] ||= 0
-        @v[param] ||= 0
-        @m[param] += (1 - @beta1) * (param.grad - @m[param])
-        @v[param] += (1 - @beta2) * (param.grad**2 - @v[param])
-        param.data -= learning_rate * @m[param] / NMath.sqrt(@v[param] + @eps)
+      private def update_params(params)
+        params.each do |param|
+          @m[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @v[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @m[param] = @alpha * @m[param] + (1 - @alpha) * param.grad
+          @v[param] = @alpha * @v[param] + (1 - @alpha) * param.grad ** 2
+          param.data -= (@lr / Xumo::NMath.sqrt(@v[param] - @m[param] ** 2 + @eps)) * param.grad
+        end
       end
     end
 
 
-    class RMSPropGraves < Optimizer
-      # @return [Float] Return the alpha value.
+    class Adam < Optimizer
       attr_accessor :alpha
-      # @return [Float] Return the eps value.
+      attr_accessor :beta1
+      attr_accessor :beta2
       attr_accessor :eps
-      
+      attr_reader :amsgrad
+
       def self.from_hash(hash)
-        self.new(hash[:learning_rate], alpha: hash[:alpha], eps: hash[:eps])
+        self.new(alpha: hash[:alpha], beta1: hash[:beta1], beta2: hash[:beta2],
+                 eps: hash[:eps], amsgrad: hash[:amsgrad], clip_norm: hash[:clip_norm])
       end
 
-      # @param [Float] learning_rate Learning rate.
-      # @param [Float] alpha Moving average index of past slopes.
+      # @param [Float] alpha Value used to calculate learning rate.
+      # @param [Float] beta1 Moving average index of beta1.
+      # @param [Float] beta2 Moving average index of beta2.
       # @param [Float] eps Value to avoid division by zero.
-      def initialize(learning_rate = 0.0001, alpha: 0.95, eps: 0.0001)
-        super(learning_rate)
+      # @param [Boolean] amsgrad Setting the true enable amsgrad.
+      def initialize(alpha: 0.001, beta1: 0.9, beta2: 0.999, eps: 1e-7, amsgrad: false, clip_norm: nil)
+        super(clip_norm: clip_norm)
         @alpha = alpha
+        @beta1 = beta1
+        @beta2 = beta2
         @eps = eps
+        @amsgrad = amsgrad
+        @t = 0
         @m = {}
         @v = {}
+        @s = {} if amsgrad
       end
 
       def to_hash
-        super(alpha: @alpha, eps: @eps)
+        {
+          class: self.class.name, alpha: @alpha, beta1: @beta1, beta2: @beta2,
+          eps: @eps, amsgrad: @amsgrad, clip_norm: @clip_norm
+        }
+      end
+
+      private def update_params(params)
+        @t += 1
+        lr = @alpha * Math.sqrt(1 - @beta2 ** @t) / (1 - @beta1 ** @t)
+        params.each do |param|
+          @m[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @v[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @m[param] += (1 - @beta1) * (param.grad - @m[param])
+          @v[param] += (1 - @beta2) * (param.grad ** 2 - @v[param])
+          if @amsgrad
+            @s[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+            @s[param] = Xumo::SFloat.maximum(@s[param], @v[param])
+            param.data -= lr * @m[param] / Xumo::NMath.sqrt(@s[param] + @eps)
+          else
+            param.data -= lr * @m[param] / Xumo::NMath.sqrt(@v[param] + @eps)
+          end
+        end
+      end
+    end
+
+
+    class AdaBound < Adam
+      attr_accessor :final_lr
+      attr_accessor :gamma
+
+      def self.from_hash(hash)
+        self.new(alpha: hash[:alpha], beta1: hash[:beta1], beta2: hash[:beta2],
+                 final_lr: hash[:final_lr], gamma: hash[:gamma], eps: hash[:eps], amsgrad: hash[:amsgrad], clip_norm: hash[:clip_norm])
+      end
+
+      # @param [Float] final_lr Final learning rate.
+      # @param [Float] gamma Lower and upper range value.
+      def initialize(alpha: 0.001, beta1: 0.9, beta2: 0.999, final_lr: 0.1, gamma: 0.001, eps: 1e-7, amsgrad: false, clip_norm: nil)
+        super(alpha: alpha, beta1: beta1, beta2: beta2, eps: eps, amsgrad: amsgrad, clip_norm: clip_norm)
+        @final_lr = final_lr
+        @gamma = gamma
+      end
+
+      def to_hash
+        {
+          class: self.class.name, alpha: @alpha, beta1: @beta1, beta2: @beta2,
+          final_lr: @final_lr, gamma: @gamma, eps: @eps, amsgrad: amsgrad, clip_norm: @clip_norm
+        }
+      end
+
+      private def update_params(params)
+        @t += 1
+        lr = @alpha * Math.sqrt(1 - @beta2 ** @t) / (1 - @beta1 ** @t)
+        final_lr = @final_lr * lr / @alpha
+        lower_bound = final_lr * (1 - 1 / (@gamma * @t + 1))
+        upper_bound = final_lr * (1 + 1 / (@gamma * @t))
+        params.each do |param|
+          @m[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @v[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @m[param] += (1 - @beta1) * (param.grad - @m[param])
+          @v[param] += (1 - @beta2) * (param.grad ** 2 - @v[param])
+          if @amsgrad
+            @s[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+            @s[param] = Xumo::SFloat.maximum(@s[param], @v[param])
+            param.data -= clip_lr(lr / (Xumo::NMath.sqrt(@s[param]) + @eps), lower_bound, upper_bound) * @m[param]
+          else
+            param.data -= clip_lr(lr / (Xumo::NMath.sqrt(@v[param]) + @eps), lower_bound, upper_bound) * @m[param]
+          end
+        end
       end
 
-      private def update_param(param)
-        @m[param] ||= 0
-        @v[param] ||= 0
-        @m[param] = @alpha * @m[param] + (1 - @alpha) * param.grad
-        @v[param] = @alpha * @v[param] + (1 - @alpha) * param.grad**2
-        param.data -= (@learning_rate / NMath.sqrt(@v[param] - @m[param]**2 + @eps)) * param.grad
+      private def clip_lr(lr, lower_bound, upper_bound)
+        lr[lr < lower_bound] = lower_bound
+        lr[lr > upper_bound] = upper_bound
+        lr
       end
     end