ruby-dnn 0.13.4 → 0.14.0

data/lib/dnn/core/normalizations.rb

@@ -32,7 +32,7 @@ module DNN
         if DNN.learning_phase
           mean = x.mean(axis: @axis, keepdims: true)
           @xc = x - mean
-          var = (@xc ** 2).mean(axis: @axis, keepdims: true)
+          var = (@xc**2).mean(axis: @axis, keepdims: true)
           @std = Xumo::NMath.sqrt(var + @eps)
           xn = @xc / @std
           @xn = xn
@@ -53,7 +53,7 @@ module DNN
         end
         dxn = @gamma.data * dy
         dxc = dxn / @std
-        dstd = -((dxn * @xc) / (@std ** 2)).sum(axis: @axis, keepdims: true)
+        dstd = -((dxn * @xc) / (@std**2)).sum(axis: @axis, keepdims: true)
         dvar = 0.5 * dstd / @std
         dxc += (2.0 / batch_size) * @xc * dvar
         dmean = dxc.sum(axis: @axis, keepdims: true)

data/lib/dnn/core/optimizers.rb

@@ -10,13 +10,14 @@ module DNN
         return nil unless hash
         optimizer_class = DNN.const_get(hash[:class])
         optimizer = optimizer_class.allocate
-        raise DNN_Error.new("#{optimizer.class} is not an instance of #{self} class.") unless optimizer.is_a?(self)
+        raise DNN_Error, "#{optimizer.class} is not an instance of #{self} class." unless optimizer.is_a?(self)
         optimizer.load_hash(hash)
         optimizer
       end
 
       def self.load(dumped)
         opt = from_hash(dumped[:hash])
+        return opt unless dumped[:status]
         dumped[:status].each do |key, state|
           state = state.clone
           opt.status[key] = state
@@ -34,7 +35,7 @@ module DNN
       def update(layers)
         target_params = layers.select { |layer| layer.is_a?(Layers::HasParamLayer) && layer.trainable }
                               .map { |layer| layer.get_params.values }.flatten.compact
-                              .select { |param| param.grad }
+                              .select(&:grad)
         clip_grads(target_params) if @clip_norm
         update_params(target_params)
         target_params.each do |param|
@@ -42,8 +43,9 @@ module DNN
         end
       end
 
-      def dump
-        { hash: to_hash, status: @status }
+      def dump(require_status = true)
+        status = require_status ? @status : nil
+        { hash: to_hash, status: status }
       end
 
       def to_hash(merge_hash = nil)
@@ -54,12 +56,13 @@ module DNN
 
       # Update params.
       private def update_params(params)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'update_params'")
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'update_params'"
       end
 
       private def clip_grads(params)
-        norm = Math.sqrt(params.reduce(0) { |total, param| total + (param.grad ** 2).sum })
+        norm = Math.sqrt(params.reduce(0) { |total, param| total + (param.grad**2).sum })
         return if norm <= @clip_norm
+
         rate = @clip_norm / (norm + 1e-7)
         params.each do |param|
           param.grad *= rate
@@ -71,7 +74,6 @@ module DNN
       end
     end
 
-
     class SGD < Optimizer
       attr_accessor :lr
       attr_accessor :momentum
@@ -107,7 +109,6 @@ module DNN
       end
     end
 
-
     class Nesterov < SGD
       def initialize(lr = 0.01, momentum: 0.9, clip_norm: nil)
         super(lr, momentum: momentum, clip_norm: clip_norm)
@@ -118,12 +119,11 @@ module DNN
           @v[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
           amount = param.grad * @lr
           @v[param.name] = @v[param.name] * @momentum - amount
-          param.data = (param.data + @momentum ** 2 * @v[param.name]) - (1 + @momentum) * amount
+          param.data = (param.data + @momentum**2 * @v[param.name]) - (1 + @momentum) * amount
         end
       end
     end
 
-
     class AdaGrad < Optimizer
       attr_accessor :lr
       attr_accessor :eps
@@ -141,7 +141,7 @@ module DNN
       private def update_params(params)
         params.each do |param|
           @g[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @g[param.name] += param.grad ** 2
+          @g[param.name] += param.grad**2
           param.data -= (@lr / Xumo::NMath.sqrt(@g[param.name] + @eps)) * param.grad
         end
       end
@@ -155,7 +155,6 @@ module DNN
       end
     end
 
-
     class RMSProp < Optimizer
       attr_accessor :lr
       attr_accessor :alpha
@@ -180,7 +179,7 @@ module DNN
       private def update_params(params)
         params.each do |param|
           @g[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @g[param.name] = @alpha * @g[param.name] + (1 - @alpha) * param.grad ** 2
+          @g[param.name] = @alpha * @g[param.name] + (1 - @alpha) * param.grad**2
           param.data -= (@lr / Xumo::NMath.sqrt(@g[param.name] + @eps)) * param.grad
         end
       end
@@ -190,7 +189,6 @@ module DNN
       end
     end
 
-
     class AdaDelta < Optimizer
       attr_accessor :rho
       attr_accessor :eps
@@ -214,9 +212,9 @@ module DNN
         params.each do |param|
           @h[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
           @s[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
-          @h[param.name] = @rho * @h[param.name] + (1 - @rho) * param.grad ** 2
+          @h[param.name] = @rho * @h[param.name] + (1 - @rho) * param.grad**2
           v = (Xumo::NMath.sqrt(@s[param.name] + @eps) / Xumo::NMath.sqrt(@h[param.name] + @eps)) * param.grad
-          @s[param.name] = @rho * @s[param.name] + (1 - @rho) * v ** 2
+          @s[param.name] = @rho * @s[param.name] + (1 - @rho) * v**2
           param.data -= v
         end
       end
@@ -226,7 +224,6 @@ module DNN
       end
     end
 
-
     class RMSPropGraves < Optimizer
       attr_accessor :lr
       attr_accessor :alpha
@@ -254,8 +251,8 @@ module DNN
           @m[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
           @v[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
           @m[param.name] = @alpha * @m[param.name] + (1 - @alpha) * param.grad
-          @v[param.name] = @alpha * @v[param.name] + (1 - @alpha) * param.grad ** 2
-          param.data -= (@lr / Xumo::NMath.sqrt(@v[param.name] - @m[param.name] ** 2 + @eps)) * param.grad
+          @v[param.name] = @alpha * @v[param.name] + (1 - @alpha) * param.grad**2
+          param.data -= (@lr / Xumo::NMath.sqrt(@v[param.name] - @m[param.name]**2 + @eps)) * param.grad
         end
       end
 
@@ -264,7 +261,6 @@ module DNN
       end
     end
 
-
     class Adam < Optimizer
       attr_accessor :alpha
       attr_accessor :beta1
@@ -300,12 +296,12 @@ module DNN
 
       private def update_params(params)
         @t += 1
-        lr = @alpha * Math.sqrt(1 - @beta2 ** @t) / (1 - @beta1 ** @t)
+        lr = @alpha * Math.sqrt(1 - @beta2**@t) / (1 - @beta1**@t)
         params.each do |param|
           @m[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
           @v[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
           @m[param.name] += (1 - @beta1) * (param.grad - @m[param.name])
-          @v[param.name] += (1 - @beta2) * (param.grad ** 2 - @v[param.name])
+          @v[param.name] += (1 - @beta2) * (param.grad**2 - @v[param.name])
           if @amsgrad
             @s[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
             @s[param.name] = Xumo::SFloat.maximum(@s[param.name], @v[param.name])
@@ -322,7 +318,6 @@ module DNN
       end
     end
 
-
     class AdaBound < Adam
       attr_accessor :final_lr
       attr_accessor :gamma
@@ -344,7 +339,7 @@ module DNN
 
       private def update_params(params)
         @t += 1
-        lr = @alpha * Math.sqrt(1 - @beta2 ** @t) / (1 - @beta1 ** @t)
+        lr = @alpha * Math.sqrt(1 - @beta2**@t) / (1 - @beta1**@t)
         final_lr = @final_lr * lr / @alpha
         lower_bound = final_lr * (1 - 1 / (@gamma * @t + 1))
         upper_bound = final_lr * (1 + 1 / (@gamma * @t))
@@ -352,7 +347,7 @@ module DNN
           @m[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
           @v[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
           @m[param.name] += (1 - @beta1) * (param.grad - @m[param.name])
-          @v[param.name] += (1 - @beta2) * (param.grad ** 2 - @v[param.name])
+          @v[param.name] += (1 - @beta2) * (param.grad**2 - @v[param.name])
           if @amsgrad
             @s[param.name] ||= Xumo::SFloat.zeros(*param.data.shape)
             @s[param.name] = Xumo::SFloat.maximum(@s[param.name], @v[param.name])

data/lib/dnn/core/regularizers.rb

@@ -8,17 +8,17 @@ module DNN
         return nil unless hash
         regularizer_class = DNN.const_get(hash[:class])
         regularizer = regularizer_class.allocate
-        raise DNN_Error.new("#{regularizer.class} is not an instance of #{self} class.") unless regularizer.is_a?(self)
+        raise DNN_Error, "#{regularizer.class} is not an instance of #{self} class." unless regularizer.is_a?(self)
         regularizer.load_hash(hash)
         regularizer
       end
 
       def forward(x)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'forward'"
       end
 
       def backward
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'backward'")
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'backward'"
       end
 
       def to_hash(merge_hash)
@@ -28,7 +28,7 @@ module DNN
       end
 
       def load_hash(hash)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'load_hash'")
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'load_hash'"
       end
     end
 
@@ -59,7 +59,6 @@ module DNN
       end
     end
 
-
     class L2 < Regularizer
       attr_accessor :l2_lambda
 
@@ -69,7 +68,7 @@ module DNN
       end
 
       def forward(x)
-        x + 0.5 * @l2_lambda * (@param.data ** 2).sum
+        x + 0.5 * @l2_lambda * (@param.data**2).sum
       end
 
       def backward
@@ -98,7 +97,7 @@ module DNN
 
       def forward(x)
         l1 = @l1_lambda * @param.data.abs.sum
-        l2 = 0.5 * @l2_lambda * (@param.data ** 2).sum
+        l2 = 0.5 * @l2_lambda * (@param.data**2).sum
         x + l1 + l2
       end
 

data/lib/dnn/core/rnn_layers.rb

@@ -40,7 +40,7 @@ module DNN
 
       def build(input_shape)
         unless input_shape.length == 2
-          raise DNN_ShapeError.new("Input shape is #{input_shape}. But input shape must be 2 dimensional.")
+          raise DNN_ShapeError, "Input shape is #{input_shape}. But input shape must be 2 dimensional."
         end
         super
         @time_length = @input_shape[0]
@@ -129,7 +129,6 @@ module DNN
       end
     end
 
-
     class SimpleRNNDense
       attr_accessor :trainable
 
@@ -162,7 +161,6 @@ module DNN
       end
     end
 
-
     class SimpleRNN < RNN
       attr_reader :activation
 
@@ -170,7 +168,7 @@ module DNN
       def initialize(num_nodes,
                      stateful: false,
                      return_sequences: true,
-                     activation: Activations::Tanh.new,
+                     activation: Layers::Tanh.new,
                      weight_initializer: Initializers::RandomNormal.new,
                      recurrent_weight_initializer: Initializers::RandomNormal.new,
                      bias_initializer: Initializers::Zeros.new,
@@ -222,7 +220,6 @@ module DNN
       end
     end
 
-
     class LSTMDense
       attr_accessor :trainable
 
@@ -230,11 +227,11 @@ module DNN
         @weight = weight
         @recurrent_weight = recurrent_weight
         @bias = bias
-        @tanh = Activations::Tanh.new
-        @g_tanh = Activations::Tanh.new
-        @forget_sigmoid = Activations::Sigmoid.new
-        @in_sigmoid = Activations::Sigmoid.new
-        @out_sigmoid = Activations::Sigmoid.new
+        @tanh = Layers::Tanh.new
+        @g_tanh = Layers::Tanh.new
+        @forget_sigmoid = Layers::Sigmoid.new
+        @in_sigmoid = Layers::Sigmoid.new
+        @out_sigmoid = Layers::Sigmoid.new
         @trainable = true
       end
 
@@ -280,7 +277,6 @@ module DNN
       end
     end
 
-
     class LSTM < RNN
       attr_reader :cell
 
@@ -359,7 +355,6 @@ module DNN
       end
     end
 
-
     class GRUDense
       attr_accessor :trainable
 
@@ -367,9 +362,9 @@ module DNN
         @weight = weight
         @recurrent_weight = recurrent_weight
         @bias = bias
-        @update_sigmoid = Activations::Sigmoid.new
-        @reset_sigmoid = Activations::Sigmoid.new
-        @tanh = Activations::Tanh.new
+        @update_sigmoid = Layers::Sigmoid.new
+        @reset_sigmoid = Layers::Sigmoid.new
+        @tanh = Layers::Tanh.new
         @trainable = true
       end
 
@@ -387,11 +382,11 @@ module DNN
         @weight_h = @weight.data[true, (num_nodes * 2)..-1]
         @weight2_h = @recurrent_weight.data[true, (num_nodes * 2)..-1]
         @tanh_h = if @bias
-          bias_h = @bias.data[(num_nodes * 2)..-1]
-          @tanh.forward(x.dot(@weight_h) + (h * @reset).dot(@weight2_h) + bias_h)
-        else
-          @tanh.forward(x.dot(@weight_h) + (h * @reset).dot(@weight2_h))
-        end
+                    bias_h = @bias.data[(num_nodes * 2)..-1]
+                    @tanh.forward(x.dot(@weight_h) + (h * @reset).dot(@weight2_h) + bias_h)
+                  else
+                    @tanh.forward(x.dot(@weight_h) + (h * @reset).dot(@weight2_h))
+                  end
         h2 = (1 - @update) * @tanh_h + @update * h
         h2
       end
@@ -428,7 +423,6 @@ module DNN
       end
     end
 
-
     class GRU < RNN
       def initialize(num_nodes,
                      stateful: false,

data/lib/dnn/core/savers.rb

@@ -17,7 +17,7 @@ module DNN
       private
 
       def load_bin(bin)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'load_bin'")
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'load_bin'"
       end
 
       def set_all_params_data(params_data)
@@ -30,13 +30,16 @@ module DNN
       end
     end
 
-
     class MarshalLoader < Loader
       private def load_bin(bin)
         data = Marshal.load(Zlib::Inflate.inflate(bin))
+        unless @model.class.name == data[:class]
+          raise DNN_Error, "Class name is not mismatch. Target model is #{@model.class.name}. But loading model is #{data[:class]}."
+        end
         opt = Optimizers::Optimizer.load(data[:optimizer])
         loss_func = Losses::Loss.from_hash(data[:loss_func])
         @model.setup(opt, loss_func)
+        @model.instance_variable_set(:@built, false)
         @model.predict1(Xumo::SFloat.zeros(*data[:input_shape]))
         set_all_params_data(data[:params])
       end
@@ -50,6 +53,7 @@ module DNN
         opt = Optimizers::Optimizer.from_hash(data[:optimizer])
         loss_func = Losses::Loss.from_hash(data[:loss_func])
         @model.setup(opt, loss_func)
+        @model.instance_variable_set(:@built, false)
         @model.predict1(Xumo::SFloat.zeros(*data[:input_shape]))
         base64_to_params_data(data[:params])
       end
@@ -65,7 +69,6 @@ module DNN
 
   end
 
-
   module Savers
 
     class Saver
@@ -87,7 +90,7 @@ module DNN
       private
 
       def dump_bin
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'dump_bin'")
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'dump_bin'"
       end
 
       def get_all_params_data
@@ -98,7 +101,6 @@ module DNN
       end
     end
 
-
     class MarshalSaver < Saver
       def initialize(model, include_optimizer: true)
         super(model)
@@ -106,10 +108,10 @@ module DNN
       end
 
       private def dump_bin
-        opt = @include_optimizer ? @model.optimizer.dump : @model.optimizer.class.new.dump
+        require_status = @include_optimizer ? true : false
         data = {
           version: VERSION, class: @model.class.name, input_shape: @model.layers.first.input_shape, params: get_all_params_data,
-          optimizer: opt, loss_func: @model.loss_func.to_hash
+          optimizer: @model.optimizer.dump(require_status), loss_func: @model.loss_func.to_hash
         }
         Zlib::Deflate.deflate(Marshal.dump(data))
       end

data/lib/dnn/core/tensor.rb

@@ -0,0 +1,11 @@
+module DNN
+  class Tensor
+    attr_reader :data
+    attr_accessor :link
+
+    def initialize(data, link = nil)
+      @data = data
+      @link = link
+    end
+  end
+end