ruby-dnn 0.9.1 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '02929b358bbd4ff8c3107c54be0ce37ae86c20a3d345c0090f4be5bcd2ad8b32'
-  data.tar.gz: 7641e5072f9bcdd4eb1bd93d173f78fe0fa11769ecca614ce44155dc7e310b96
+  metadata.gz: dc2897efaefa857bc21c4a4237c4c8d6fac3ec508708ba0935874050f8dab7f9
+  data.tar.gz: 2eaea58d620043e47c197f1ace3ec94f8108d84dbe3883a0455b64f39c8d6f82
 SHA512:
-  metadata.gz: df47d323eda15b0f11dcf2153083bf05a8bd6c158227c3c93c0c9e0ab1f4679769fa8ad031a8bcbb886e5610880b2c95e26ad425a49ccdfe1b79b0ee280628de
-  data.tar.gz: 86766ea8873229cb665d3e93e70142c246ad8b3e197543093d06ce674d66319a2f9a94a78cf089a0677df4a4966fdbfcfb0ffc95337e6465eabd2ab4d9ba7a1c
+  metadata.gz: e74b37d9d31af87cc833b4237b2f90051199a2020d645b733e5af2eb6f07b26b87debaac651818a95fca27ffeedb3c08a62c55191e3213b6b7a8c85762ba91fd
+  data.tar.gz: b4b25e79cd9ac57e464d70248178048473e4e6a26bcae34da9b5b63f109c336be5f3ccec700a23dc9dce427b7014be81b41ba547fa59ad3f5d9ebb5fa5d881c6

data/API-Reference.ja.md

@@ -2,7 +2,7 @@
 ruby-dnnのAPIリファレンスです。このリファレンスでは、APIを利用するうえで必要となるクラスとメソッドしか記載していません。
 そのため、プログラムの詳細が必要な場合は、ソースコードを参照してください。
 
-最終更新バージョン:0.9.0
+最終更新バージョン:0.9.2
 
 # module DNN
 ruby-dnnの名前空間をなすモジュールです。
@@ -119,6 +119,16 @@ Loss
 ### return
 なし。
 
+## def compile(optimizer, loss)
+モデルを再コンパイルします。ただし、レイヤーのビルドは行いません。
+### arguments
+* Optimizer optimizer
+モデルが学習に使用するオプティマイザー。
+* Loss loss
+モデルが学習に使用する損失関数。
+### return
+なし。
+
 ## def compiled?
 モデルがコンパイル済みであるか否かを取得します。
 ### arguments
@@ -485,6 +495,11 @@ Integer
 bool  
 レイヤーがステートフルであるか否かを返します。
 
+## attr_reader :return_sequences
+bool  
+trueを指定した場合、レイヤーのforward出力値において、時系列データ全てを返します。  
+falseを指定した場合、レイヤーのforward出力値において、時系列データの最後の値を返します。
+
 ## 【Instance methods】
 
 ## def initialize(num_nodes, stateful: false, return_sequences: true, weight_initializer: Initializers::RandomNormal.new, bias_initializer: Initializers::Zeros.new, l1_lamda: 0, l2_lambda: 0)

data/lib/dnn/core/cnn_layers.rb

@@ -4,33 +4,34 @@ module DNN
     module Conv2DModule
       private
 
+      # img[bsize, out_h, out_w, channel] to col[bsize * out_h * out_w, fil_h * fil_w * ch]
       def im2col(img, out_h, out_w, fil_h, fil_w, strides)
         bsize = img.shape[0]
         ch = img.shape[3]
-        col = Xumo::SFloat.zeros(bsize, ch, fil_h, fil_w, out_h, out_w)
-        img = img.transpose(0, 3, 1, 2)
+        col = Xumo::SFloat.zeros(bsize, out_h, out_w, fil_h, fil_w, ch)
         (0...fil_h).each do |i|
           i_range = (i...(i + strides[0] * out_h)).step(strides[0]).to_a
           (0...fil_w).each do |j|
             j_range = (j...(j + strides[1] * out_w)).step(strides[1]).to_a
-            col[true, true, i, j, true, true] = img[true, true, i_range, j_range]
+            col[true, true, true, i, j, true] = img[true, i_range, j_range, true]
           end
         end
-        col.transpose(0, 4, 5, 2, 3, 1).reshape(bsize * out_h * out_w, fil_h * fil_w * ch)
+        col.reshape(bsize * out_h * out_w, fil_h * fil_w * ch)
       end
 
+      # col[bsize * out_h * out_w, fil_h * fil_w * ch] to img[bsize, out_h, out_w, channel]
       def col2im(col, img_shape, out_h, out_w, fil_h, fil_w, strides)
         bsize, img_h, img_w, ch = img_shape
-        col = col.reshape(bsize, out_h, out_w, fil_h, fil_w, ch).transpose(0, 5, 3, 4, 1, 2)
-        img = Xumo::SFloat.zeros(bsize, ch, img_h, img_w)
+        col = col.reshape(bsize, out_h, out_w, fil_h, fil_w, ch)
+        img = Xumo::SFloat.zeros(bsize, img_h, img_w, ch)
         (0...fil_h).each do |i|
           i_range = (i...(i + strides[0] * out_h)).step(strides[0]).to_a
           (0...fil_w).each do |j|
             j_range = (j...(j + strides[1] * out_w)).step(strides[1]).to_a
-            img[true, true, i_range, j_range] += col[true, true, i, j, true, true]
+            img[true, i_range, j_range, true] += col[true, true, true, i, j, true]
           end
         end
-        img.transpose(0, 2, 3, 1)
+        img
       end
 
       def padding(img, pad)
@@ -57,16 +58,39 @@ module DNN
         out_w = (prev_w - fil_w) / strides[1] + 1
         [out_h, out_w]
       end
+
+      def padding_size(prev_h, prev_w, out_h, out_w, strides)
+        pad_h = (prev_h.to_f / strides[0]).ceil - out_h
+        pad_w = (prev_w.to_f / strides[1]).ceil - out_w
+        [pad_h, pad_w]
+      end
     end
     
     
     class Conv2D < Connection
       include Conv2DModule
 
+      # @return [Integer] number of filters.
       attr_reader :num_filters
+      # @return [Array] Return filter size. filter size is of the form [height, width].
       attr_reader :filter_size
+      # @return [Array] Return stride length. stride length is of the form [height, width].
       attr_reader :strides
-    
+
+      def self.load_hash(hash)
+        Conv2D.new(hash[:num_filters], hash[:filter_size],
+                   weight_initializer: Utils.load_hash(hash[:weight_initializer]),
+                   bias_initializer: Utils.load_hash(hash[:bias_initializer]),
+                   strides: hash[:strides],
+                   padding: hash[:padding],
+                   l1_lambda: hash[:l1_lambda],
+                   l2_lambda: hash[:l2_lambda])
+      end
+      
+      # @param [Integer] num_filters number of filters.
+      # @param [Array or Integer] filter_size filter size. filter size is of the form [height, width].
+      # @param [Array or Integer] strides stride length. stride length is of the form [height, width].
+      # @param [Bool] padding Whether to padding.
       def initialize(num_filters, filter_size,
                      weight_initializer: Initializers::RandomNormal.new,
                      bias_initializer: Initializers::RandomNormal.new,
@@ -82,29 +106,18 @@ module DNN
         @padding = padding
       end
 
-      def self.load_hash(hash)
-        Conv2D.new(hash[:num_filters], hash[:filter_size],
-                   weight_initializer: Utils.load_hash(hash[:weight_initializer]),
-                   bias_initializer: Utils.load_hash(hash[:bias_initializer]),
-                   strides: hash[:strides],
-                   padding: hash[:padding],
-                   l1_lambda: hash[:l1_lambda],
-                   l2_lambda: hash[:l2_lambda])
-      end
-
       def build(input_shape)
         super
         prev_h, prev_w = input_shape[0..1]
         @out_size = out_size(prev_h, prev_w, *@filter_size, @strides)
-        out_w, out_h = @out_size
         if @padding
-          @pad = [prev_h - out_h, prev_w - out_w]
-          @out_size = [prev_h, prev_w]
+          @pad_size = padding_size(prev_h, prev_w, *@out_size, @strides)
+          @out_size = [@out_size[0] + @pad_size[0], @out_size[1] + @pad_size[1]]
         end
       end
 
       def forward(x)
-        x = padding(x, @pad) if @padding
+        x = padding(x, @pad_size) if @padding
         @x_shape = x.shape
         @col = im2col(x, *@out_size, *@filter_size, @strides)
         out = @col.dot(@weight.data) + @bias.data
@@ -117,13 +130,30 @@ module DNN
         @bias.grad = dout.sum(0)
         dcol = dout.dot(@weight.data.transpose)
         dx = col2im(dcol, @x_shape, *@out_size, *@filter_size, @strides)
-        @padding ? back_padding(dx, @pad) : dx
+        @padding ? back_padding(dx, @pad_size) : dx
       end
 
       def output_shape
         [*@out_size, @num_filters]
       end
 
+      # @return [Bool] whether to padding.
+      def padding?
+        @padding
+      end
+
+      # @return [Numo::SFloat] Convert weight to filter and return.
+      def filters
+        num_prev_filter = @input_shape[2]
+        @weight.data.reshape(*@filter_size, num_prev_filter, @num_filters)
+      end
+
+      # @param [Numo::SFloat] filters Convert weight to filters and set.
+      def filters=(filters)
+        num_prev_filter = @input_shape[2]
+        @weight.data = filters.reshape(@filter_size.reduce(:*) * num_prev_filter, @num_filters)
+      end
+
       def to_hash
         super({num_filters: @num_filters,
                filter_size: @filter_size,
@@ -135,7 +165,7 @@ module DNN
     
       def init_params
         num_prev_filter = @input_shape[2]
-        @weight.data = Xumo::SFloat.new(num_prev_filter * @filter_size.reduce(:*), @num_filters)
+        @weight.data = Xumo::SFloat.new(@filter_size.reduce(:*) * num_prev_filter, @num_filters)
         @bias.data = Xumo::SFloat.new(@num_filters)
         super()
       end
@@ -146,13 +176,19 @@ module DNN
     class Pool2D < Layer
       include Conv2DModule
 
+      # @return [Array] Return pooling size. pooling size is of the form [height, width].
       attr_reader :pool_size
+      # @return [Array] Return stride length. stride length is of the form [height, width].
       attr_reader :strides
 
       def self.load_hash(pool2d_class, hash)
         pool2d_class.new(hash[:pool_size], strides: hash[:strides], padding: hash[:padding])
       end
 
+      # @param [Array or Integer] pool_size pooling size. pooling size is of the form [height, width].
+      # @param [Array or Integer or NilClass] strides stride length. stride length is of the form [height, width].
+      #   If you set nil, treat pool_size as strides.
+      # @param [Bool] padding Whether to padding.
       def initialize(pool_size, strides: nil, padding: false)
         super()
         @pool_size = pool_size.is_a?(Integer) ? [pool_size, pool_size] : pool_size
@@ -169,10 +205,9 @@ module DNN
         prev_h, prev_w = input_shape[0..1]
         @num_channel = input_shape[2]
         @out_size = out_size(prev_h, prev_w, *@pool_size, @strides)
-        out_w, out_h = @out_size
         if @padding
-          @pad = [prev_h - out_h, prev_w - out_w]
-          @out_size = [prev_h, prev_w]
+          @pad_size = padding_size(prev_h, prev_w, *@out_size, @strides)
+          @out_size = [@out_size[0] + @pad_size[0], @out_size[1] + @pad_size[1]]
         end
       end
 
@@ -180,6 +215,11 @@ module DNN
         [*@out_size, @num_channel]
       end
 
+      # @return [Bool] whether to padding.
+      def padding?
+        @padding
+      end
+
       def to_hash
         super({pool_size: @pool_size,
                strides: @strides,
@@ -194,10 +234,11 @@ module DNN
       end
 
       def forward(x)
-        x = padding(x, @pad) if @padding
+        x = padding(x, @pad_size) if @padding
         @x_shape = x.shape
         col = im2col(x, *@out_size, *@pool_size, @strides)
-        col = col.reshape(x.shape[0] * @out_size.reduce(:*) * x.shape[3], @pool_size.reduce(:*))
+        col = col.reshape(x.shape[0] * @out_size.reduce(:*), @pool_size.reduce(:*), x.shape[3]).transpose(0, 2, 1)
+                 .reshape(x.shape[0] * @out_size.reduce(:*) * x.shape[3], @pool_size.reduce(:*))
         @max_index = col.max_index(1)
         col.max(1).reshape(x.shape[0], *@out_size, x.shape[3])
       end
@@ -205,9 +246,9 @@ module DNN
       def backward(dout)
         dmax = Xumo::SFloat.zeros(dout.size * @pool_size.reduce(:*))
         dmax[@max_index] = dout.flatten
-        dcol = dmax.reshape(dout.shape[0..2].reduce(:*), dout.shape[3] * @pool_size.reduce(:*))
+        dcol = dmax.reshape(dout.shape[0..2].reduce(:*), @pool_size.reduce(:*) * dout.shape[3])
         dx = col2im(dcol, @x_shape, *@out_size, *@pool_size, @strides)
-        @padding ? back_padding(dx, @pad) : dx
+        @padding ? back_padding(dx, @pad_size) : dx
       end
     end
 
@@ -218,10 +259,11 @@ module DNN
       end
 
       def forward(x)
-        x = padding(x, @pad) if @padding
+        x = padding(x, @pad_size) if @padding
         @x_shape = x.shape
         col = im2col(x, *@out_size, *@pool_size, @strides)
-        col = col.reshape(x.shape[0] * @out_size.reduce(:*) * x.shape[3], @pool_size.reduce(:*))
+        col = col.reshape(x.shape[0] * @out_size.reduce(:*), @pool_size.reduce(:*), x.shape[3]).transpose(0, 2, 1)
+                 .reshape(x.shape[0] * @out_size.reduce(:*) * x.shape[3], @pool_size.reduce(:*))
         col.mean(1).reshape(x.shape[0], *@out_size, x.shape[3])
       end
 
@@ -234,14 +276,16 @@ module DNN
         end
         dcol = davg.reshape(dout.shape[0..2].reduce(:*), dout.shape[3] * @pool_size.reduce(:*))
         dx = col2im(dcol, @x_shape, *@out_size, *@pool_size, @strides)
-        @padding ? back_padding(dx, @pad) : dx
+        @padding ? back_padding(dx, @pad_size) : dx
       end
     end
 
 
     class UnPool2D < Layer
+      # @return [Array] Return unpooling size. unpooling size is of the form [height, width].
       attr_reader :unpool_size
 
+      # @param [Array or Integer] unpool_size Unpooling size. unpooling size is of the form [height, width].
       def initialize(unpool_size)
         super()
         @unpool_size = unpool_size.is_a?(Integer) ? [unpool_size, unpool_size] : unpool_size

data/lib/dnn/core/layers.rb

@@ -45,8 +45,10 @@ module DNN
     
     # This class is a superclass of all classes with learning parameters.
     class HasParamLayer < Layer
-      attr_accessor :trainable # Setting false prevents learning of parameters.
-      attr_reader :params      # The parameters of the layer.
+      # @return [Bool] trainable Setting false prevents learning of parameters.
+      attr_accessor :trainable
+      # @return [Array] The parameters of the layer.
+      attr_reader :params
     
       def initialize
         super()
@@ -107,11 +109,19 @@ module DNN
 
     # It is a superclass of all connection layers.
     class Connection < HasParamLayer
-      attr_reader :l1_lambda # L1 regularization
-      attr_reader :l2_lambda # L2 regularization
+      # @return [DNN::Initializers] weight initializer.
       attr_reader :weight_initializer
+      # @return [DNN::Initializers] bias initializer.
       attr_reader :bias_initializer
-
+      # @return [Float] L1 regularization
+      attr_reader :l1_lambda
+      # @return [Float] L2 regularization
+      attr_reader :l2_lambda
+
+      # @param [DNN::Initializers] weight_initializer weight initializer.
+      # @param [DNN::Initializers] bias_initializer bias initializer.
+      # @param [Float] l1_lambda L1 regularization
+      # @param [Float] l2_lambda L2 regularization
       def initialize(weight_initializer: Initializers::RandomNormal.new,
                      bias_initializer: Initializers::Zeros.new,
                      l1_lambda: 0,
@@ -171,7 +181,9 @@ module DNN
     end
     
     
+    # Full connnection layer.
     class Dense < Connection
+      # @return [Integer] number of nodes.
       attr_reader :num_nodes
 
       def self.load_hash(hash)
@@ -181,7 +193,8 @@ module DNN
                  l1_lambda: hash[:l1_lambda],
                  l2_lambda: hash[:l2_lambda])
       end
-    
+
+      # @param [Integer] num_nodes number of nodes.
       def initialize(num_nodes,
                      weight_initializer: Initializers::RandomNormal.new,
                      bias_initializer: Initializers::Zeros.new,
@@ -213,6 +226,8 @@ module DNN
     
       private
     
+      # TODO
+      # Change writing super() other than the first.
       def init_params
         num_prev_nodes = @input_shape[0]
         @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
@@ -266,16 +281,20 @@ module DNN
 
     
     class Dropout < Layer
+      # @return [Float] dropout ratio.
       attr_reader :dropout_ratio
+      # @return [Float] Use 'weight scaling inference rule'.
+      attr_reader :use_scale
 
       def self.load_hash(hash)
-        self.new(hash[:dropout_ratio], hash[:seed])
+        self.new(hash[:dropout_ratio], seed: hash[:seed], use_scale: hash[:use_scale])
       end
 
-      def initialize(dropout_ratio = 0.5, seed = rand(1 << 31))
+      def initialize(dropout_ratio = 0.5, seed: rand(1 << 31), use_scale: true)
         super()
         @dropout_ratio = dropout_ratio
         @seed = seed
+        @use_scale = use_scale
         @mask = nil
       end
 
@@ -285,29 +304,31 @@ module DNN
           @mask = Xumo::SFloat.ones(*x.shape).rand < @dropout_ratio
           x[@mask] = 0
         else
-          x *= (1 - @dropout_ratio)
+          x *= (1 - @dropout_ratio) if @use_scale
         end
         x
       end
     
-      def backward(dout, learning_phase)
-        dout[@mask] = 0 if learning_phase
+      def backward(dout)
+        dout[@mask] = 0
         dout
       end
 
       def to_hash
-        super({dropout_ratio: @dropout_ratio, seed: @seed})
+        super({dropout_ratio: @dropout_ratio, seed: @seed, use_scale: @use_scale})
       end
     end
     
     
     class BatchNormalization < HasParamLayer
+      # @return [Float] Exponential moving average of mean and variance.
       attr_reader :momentum
 
       def self.load_hash(hash)
         self.new(momentum: hash[:momentum])
       end
 
+      # @param [Float] momentum Exponential moving average of mean and variance.
       def initialize(momentum: 0.9)
         super()
         @momentum = momentum
@@ -330,7 +351,7 @@ module DNN
         @gamma.data * xn + @beta.data
       end
     
-      def backward(dout, learning_phase)
+      def backward(dout)
         batch_size = dout.shape[0]
         @beta.grad = dout.sum(0)
         @gamma.grad = (@xn * dout).sum(0)

data/lib/dnn/core/losses.rb

@@ -2,19 +2,16 @@ module DNN
   module Losses
 
     class Loss
-      def forward(out, y)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
+      def forward(out, y, layers)
+        regularize = layers.select { |layer| layer.is_a?(Connection) }
+                           .reduce(0) { |sum, layer| sum + layer.lasso + layer.ridge }
+        loss(out, y) + regularize
       end
 
       def backward(y)
         raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'backward'")
       end
 
-      def regularize(layers)
-        layers.select { |layer| layer.is_a?(Connection) }
-              .reduce(0) { |sum, layer| sum + layer.lasso + layer.ridge }
-      end
-
       def d_regularize(layers)
         layers.select { |layer| layer.is_a?(Connection) }.each do |layer|
           layer.d_lasso
@@ -25,10 +22,16 @@ module DNN
       def to_hash
         {class: self.class.name}
       end
+
+      private
+
+      def loss(out, y)
+        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'loss'")
+      end
     end
 
     class MeanSquaredError < Loss
-      def forward(out, y)
+      def loss(out, y)
         @out = out
         batch_size = y.shape[0]
         0.5 * ((out - y)**2).sum / batch_size
@@ -41,7 +44,7 @@ module DNN
 
 
     class MeanAbsoluteError < Loss
-      def forward(out, y)
+      def loss(out, y)
         @out = out
         batch_size = y.shape[0]
         (out - y).abs.sum / batch_size
@@ -58,16 +61,18 @@ module DNN
 
     class HuberLoss < Loss
       def forward(out, y, layers)
+        @loss_value = super(out, y, layers)
+      end
+
+      def loss(out, y)
         @out = out
-        loss = loss_l1(y)
-        loss = loss > 1 ? loss : loss_l2(y)
-        #@loss = loss + regularize(layers)
-        @loss = loss
+        loss_value = loss_l1(y)
+        loss_value > 1 ? loss_value : loss_l2(y)
       end
 
       def backward(y)
         dout = @out - y
-        if @loss > 1
+        if @loss_value > 1
           dout[dout >= 0] = 1
           dout[dout < 0] = -1
         end
@@ -89,7 +94,7 @@ module DNN
 
 
     class SoftmaxCrossEntropy < Loss
-      def forward(x, y)
+      def loss(x, y)
         @out = Utils.softmax(x)
         batch_size = y.shape[0]
         -(y * NMath.log(@out + 1e-7)).sum / batch_size
@@ -102,7 +107,7 @@ module DNN
 
 
     class SigmoidCrossEntropy < Loss
-      def forward(x, y)
+      def loss(x, y)
         @out = Utils.sigmoid(x)
         batch_size = y.shape[0]
         -(y * NMath.log(@out + 1e-7) + (1 - y) * NMath.log(1 - @out + 1e-7)).sum / batch_size

data/lib/dnn/core/model.rb

@@ -6,13 +6,20 @@ module DNN
 
   # This class deals with the model of the network.
   class Model
-    attr_accessor :layers    # All layers possessed by the model
-    attr_accessor :trainable # Setting false prevents learning of parameters.
+    # @return [Array] All layers possessed by the model.
+    attr_accessor :layers
+    # @return [Bool] Setting false prevents learning of parameters.
+    attr_accessor :trainable
 
+    # Load marshal model.
+    # @param [String] file_name File name of marshal model to load.
     def self.load(file_name)
       Marshal.load(Zlib::Inflate.inflate(File.binread(file_name)))
     end
 
+    # Load json model.
+    # @param [String] json_str json string to load model.
+    # @return [DNN::Model]
     def self.load_json(json_str)
       hash = JSON.parse(json_str, symbolize_names: true)
       model = self.load_hash(hash)
@@ -33,6 +40,8 @@ module DNN
       @compiled = false
     end
 
+    # Load json model parameters.
+    # @param [String] json_str json string to load model parameters.
     def load_json_params(json_str)
       hash = JSON.parse(json_str, symbolize_names: true)
       has_param_layers_params = hash[:params]
@@ -49,6 +58,8 @@ module DNN
       end
     end
 
+    # Save the model in marshal format.
+    # @param [String] file_name name to save model.
     def save(file_name)
       bin = Zlib::Deflate.deflate(Marshal.dump(self))
       begin
@@ -60,12 +71,16 @@ module DNN
       end
     end
 
+    # Convert model to json string.
+    # @return [String] json string.
     def to_json
       hash = self.to_hash
       hash[:version] = VERSION
       JSON.pretty_generate(hash)
     end
     
+    # Convert model parameters to json string.
+    # @return [String] json string.
     def params_to_json
       has_param_layers = get_all_layers.select { |layer| layer.is_a?(Layers::HasParamLayer) }
       has_param_layers_params = has_param_layers.map do |layer|
@@ -78,6 +93,9 @@ module DNN
       JSON.dump(hash)
     end
 
+    # Add layer to the model.
+    # @param [DNN::Layers::Layer] layer Layer to add to the model.
+    # @return [DNN::Model] return self.
     def <<(layer)
       # Due to a bug in saving nested models, temporarily prohibit model nesting.
       # if !layer.is_a?(Layers::Layer) && !layer.is_a?(Model)
@@ -90,7 +108,11 @@ module DNN
       self
     end
 
+    # Set optimizer and loss to model and build all layers.
+    # @param [DNN::Optimizers::Optimizer] optimizer Optimizer to use for learning.
+    # @param [DNN::Losses::Loss] loss Lptimizer to use for learning.
     def compile(optimizer, loss)
+      raise DNN_Error.new("The model is already compiled.") if compiled?
       unless optimizer.is_a?(Optimizers::Optimizer)
         raise TypeError.new("optimizer:#{optimizer.class} is not an instance of DNN::Optimizers::Optimizer class.")
       end
@@ -105,6 +127,23 @@ module DNN
       layers_shape_check
     end
 
+    # Set optimizer and loss to model and recompile. But does not build layers.
+    # @param [DNN::Optimizers::Optimizer] optimizer Optimizer to use for learning.
+    # @param [DNN::Losses::Loss] loss Lptimizer to use for learning.
+    def recompile(optimizer, loss)
+      unless optimizer.is_a?(Optimizers::Optimizer)
+        raise TypeError.new("optimizer:#{optimizer.class} is not an instance of DNN::Optimizers::Optimizer class.")
+      end
+      unless loss.is_a?(Losses::Loss)
+        raise TypeError.new("loss:#{loss.class} is not an instance of DNN::Losses::Loss class.")
+      end
+      @compiled = true
+      layers_check
+      @optimizer = optimizer
+      @loss = loss
+      layers_shape_check
+    end
+
     def build(super_model = nil)
       @super_model = super_model
       shape = if super_model
@@ -122,28 +161,44 @@ module DNN
       end
     end
 
+    # @return [Array] Return the input shape of the model.
     def input_shape
       @layers.first.input_shape
     end
 
+    # @return [Array] Return the output shape of the model.
     def output_shape
       @layers.last.output_shape
     end
 
+    # @return [DNN::Optimizers::Optimizer] optimizer Return the optimizer to use for learning.
     def optimizer
       raise DNN_Error.new("The model is not compiled.") unless compiled?
       @optimizer ? @optimizer : @super_model.optimizer
     end
 
+    # @return [DNN::Losses::Loss] loss Return the loss to use for learning.
     def loss
       raise DNN_Error.new("The model is not compiled.") unless compiled?
       @loss ? @loss : @super_model.loss
     end
 
+    # @return [Bool] Returns whether the model is learning.
     def compiled?
       @compiled
     end
 
+    # Start training.
+    # Compile the model before use this method.
+    # @param [Numo::SFloat] x Input training data.
+    # @param [Numo::SFloat] y Output training data.
+    # @param [Integer] epochs Number of training.
+    # @param [Integer] batch_size Batch size used for one training.
+    # @param [Array or NilClass] test If you to test the model for every 1 epoch,
+    #     specify [x_test, y_test]. Don't test to the model, specify nil.                     
+    # @param [Bool] verbose Set true to display the log. If false is set, the log is not displayed.
+    # @param [Proc] batch_proc Set proc to process per batch.
+    # @yield [epoch] Process performed before one training.
     def train(x, y, epochs,
               batch_size: 1,
               test: nil,
@@ -187,24 +242,29 @@ module DNN
       end
     end
   
+    # Training once.
+    # Compile the model before use this method.
+    # @param [Numo::SFloat] x Input training data.
+    # @param [Numo::SFloat] y Output training data.
+    # @yield [x, y] batch_proc Set proc to process per batch.
     def train_on_batch(x, y, &batch_proc)
       raise DNN_Error.new("The model is not compiled.") unless compiled?
       check_xy_type(x, y)
       input_data_shape_check(x, y)
       x, y = batch_proc.call(x, y) if batch_proc
       out = forward(x, true)
-      loss_value = if @loss.is_a?(HuberLoss)
-        @loss.forward(out, y, get_all_layers)
-      else
-        @loss.forward(out, y) + @loss.regularize(get_all_layers)
-      end
+      loss_value = @loss.forward(out, y, get_all_layers)
       dout = @loss.backward(y)
-      backward(dout, true)
+      backward(dout)
       @loss.d_regularize(get_all_layers)
       update
       loss_value
     end
   
+    # Evaluate model and get accurate of test data.
+    # @param [Numo::SFloat] x Input test data.
+    # @param [Numo::SFloat] y Output test data.
+    # @yield [x, y] batch_proc Set proc to process per batch.
     def accurate(x, y, batch_size = 100, &batch_proc)
       check_xy_type(x, y)
       input_data_shape_check(x, y)
@@ -231,22 +291,28 @@ module DNN
       end
       correct.to_f / x.shape[0]
     end
-  
+
+    # Predict data.
+    # @param [Numo::SFloat] x Input data.
     def predict(x)
       check_xy_type(x)
       input_data_shape_check(x)
       forward(x, false)
     end
 
+    # Predict one data.
+    # @param [Numo::SFloat] x Input data. However, x is single data.
     def predict1(x)
       check_xy_type(x)
       predict(Xumo::SFloat.cast([x]))[0, false]
     end
 
+    # @return [DNN::Model] Copy this model.
     def copy
       Marshal.load(Marshal.dump(self))
     end
 
+    # Get the layer that the model has.
     def get_layer(*args)
       if args.length == 1
         index = args[0]
@@ -257,13 +323,17 @@ module DNN
       end
     end
 
+    # Get the all layers.
+    # @return [Array] all layers array.
     def get_all_layers
       @layers.map { |layer|
         layer.is_a?(Model) ? layer.get_all_layers : layer
       }.flatten
     end
   
-    def forward(x, learning_phase)
+    # TODO
+    # It is not good to write the Layer class name directly in the Model class. I will fix it later.
+    def forward(x, learning_phase)01
       @layers.each do |layer|
         x = if layer.is_a?(Layers::Dropout) || layer.is_a?(Layers::BatchNormalization) || layer.is_a?(Model)
           layer.forward(x, learning_phase)
@@ -274,13 +344,9 @@ module DNN
       x
     end
   
-    def backward(dout, learning_phase)
+    def backward(dout)
       @layers.reverse.each do |layer|
-        if layer.is_a?(Layers::Dropout) || layer.is_a?(Layers::BatchNormalization) || layer.is_a?(Model)
-          dout = layer.backward(dout, learning_phase)
-        else
-          dout = layer.backward(dout)
-        end
+        dout = layer.backward(dout)
       end
       dout
     end
@@ -364,12 +430,6 @@ module DNN
         raise TypeError.new("y:#{y.class.name} is not an instance of #{Xumo::SFloat.name} class.")
       end
     end
-
-    def type_check(var_name, var, type)
-      unless var.is_a?(type)
-        raise TypeError.new("#{var_name}:#{var.class} is not an instance of #{type} class.")
-      end
-    end
   end
 
 end

data/lib/dnn/core/rnn_layers.rb

@@ -3,19 +3,20 @@ module DNN
 
     # Super class of all RNN classes.
     class RNN < Connection
-      include Activations
+      include Initializers
 
+      # @return [Integer] number of nodes.
       attr_reader :num_nodes
+      # @return [Bool] Maintain state between batches.
       attr_reader :stateful
-      attr_reader :weight
-      attr_reader :weight2
-      attr_reader :bias
+      # @return [Bool] Only the last of each cell of RNN is left.
+      attr_reader :return_sequences
 
       def initialize(num_nodes,
                      stateful: false,
                      return_sequences: true,
-                     weight_initializer: Initializers::RandomNormal.new,
-                     bias_initializer: Initializers::Zeros.new,
+                     weight_initializer: RandomNormal.new,
+                     bias_initializer: Zeros.new,
                      l1_lambda: 0,
                      l2_lambda: 0)
         super(weight_initializer: weight_initializer, bias_initializer: bias_initializer,
@@ -25,6 +26,8 @@ module DNN
         @return_sequences = return_sequences
         @layers = []
         @hidden = @params[:h] = Param.new
+        # TODO
+        # Change to a good name.
         @params[:weight2] = @weight2 = Param.new
       end
 
@@ -74,10 +77,7 @@ module DNN
         super(hash)
       end
 
-      def shape
-        @return_sequences ? [@time_length, @num_nodes] : [@num_nodes]
-      end
-
+      # Reset the state of RNN.
       def reset_state
         @hidden.data = @hidden.data.fill(0) if @hidden.data
       end
@@ -152,6 +152,8 @@ module DNN
 
 
     class SimpleRNN < RNN
+      include Activations
+
       attr_reader :activation
       
       def self.load_hash(hash)
@@ -170,8 +172,8 @@ module DNN
                      stateful: false,
                      return_sequences: true,
                      activation: Tanh.new,
-                     weight_initializer: Initializers::RandomNormal.new,
-                     bias_initializer: Initializers::Zeros.new,
+                     weight_initializer: RandomNormal.new,
+                     bias_initializer: Zeros.new,
                      l1_lambda: 0,
                      l2_lambda: 0)
         super(num_nodes,
@@ -273,8 +275,8 @@ module DNN
       def initialize(num_nodes,
                      stateful: false,
                      return_sequences: true,
-                     weight_initializer: Initializers::RandomNormal.new,
-                     bias_initializer: Initializers::Zeros.new,
+                     weight_initializer: RandomNormal.new,
+                     bias_initializer: Zeros.new,
                      l1_lambda: 0,
                      l2_lambda: 0)
         super
@@ -416,8 +418,8 @@ module DNN
       def initialize(num_nodes,
                      stateful: false,
                      return_sequences: true,
-                     weight_initializer: Initializers::RandomNormal.new,
-                     bias_initializer: Initializers::Zeros.new,
+                     weight_initializer: RandomNormal.new,
+                     bias_initializer: Zeros.new,
                      l1_lambda: 0,
                      l2_lambda: 0)
         super

data/lib/dnn/core/utils.rb

@@ -20,6 +20,8 @@ module DNN
       dnn_class.new
     end
 
+    # TODO
+    # Don't want to write an implementation of the activation function in utils, so we will consider it later.
     def self.sigmoid(x)
       1 / (1 + NMath.exp(-x))
     end

data/lib/dnn/version.rb

@@ -1,3 +1,3 @@
 module DNN
-  VERSION = "0.9.1"
+  VERSION = "0.9.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-dnn
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.2
 platform: ruby
 authors:
 - unagiootoro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-04 00:00:00.000000000 Z
+date: 2019-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray