ruby-dnn 0.9.3 → 0.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2926b7a9ca6f3766ee46bba676837bd4cb98d55220ba37e4ef12109ef244a444
-  data.tar.gz: 57db99b2200d3d0946c6bb886f97308f6b09e550f5ab66802f61a7d13bd9b8fe
+  metadata.gz: 670d2d681e3929d20c5c855c97dbad1759560eb45018d20a9089c6bc62787833
+  data.tar.gz: a9360305a97aed1bac5300010fbfa3ea85565eab70d299a9d07acd78d6b24d63
 SHA512:
-  metadata.gz: 19c77e661bb6b208c279ce80d85a115572e124bad50dd97f3c8ec1097aa1bf4f245cef11bee58c3ffbc5c8c3a47f4aa6e9e3bb89db0ba8c1c7ff6ef2066764b3
-  data.tar.gz: 9a8d414fcb3401b883bde2cd54f6b320817ee66389e43971f62a4d9f85e1888db36842d704c3c2392f27f68266f6dcd8faf207723dfca25195651966e3713652
+  metadata.gz: d1f3b49241a81bf8c56b595bdaeecf097d8d3d461a02ab37860c6d2eede4a7e5cfadd0f83d2759cdbbdc7d541bbadebd11e55a80b0cfce9c2bd8a216efb2596d
+  data.tar.gz: e92230987d4c59cf4395ee33695fc974c65940f7df6aecfc3f89ebf65bbbeaae95433079df8855c448850923b5d230807eca2d2c135d475df6aa1aba6431988a

data/examples/iris_example.rb

@@ -0,0 +1,33 @@
+require "dnn"
+require "dnn/lib/iris"
+# require "numo/linalg/autoloader"
+
+include DNN::Layers
+include DNN::Activations
+include DNN::Optimizers
+include DNN::Losses
+Model = DNN::Model
+Iris = DNN::Iris
+
+x, y = Iris.load(true)
+x_train, y_train = x[0...100, true], y[0...100]
+x_test, y_test = x[100...150, true], y[100...150]
+
+x_train /= 255
+x_test /= 255
+
+y_train = DNN::Utils.to_categorical(y_train, 3, Numo::SFloat)
+y_test = DNN::Utils.to_categorical(y_test, 3, Numo::SFloat)
+
+model = Model.new
+
+model << InputLayer.new(4)
+
+model << Dense.new(64)
+model << ReLU.new
+
+model << Dense.new(3)
+
+model.compile(Adam.new, SoftmaxCrossEntropy.new)
+
+model.train(x_train, y_train, 1000, batch_size: 10, test: [x_test, y_test])

data/lib/dnn/core/activations.rb

@@ -3,7 +3,7 @@ module DNN
 
     class Sigmoid < Layers::Layer
       def forward(x)
-        @out = Utils.sigmoid(x)
+        @out = 1 / (1 + NMath.exp(-x))
       end
     
       def backward(dout)

data/lib/dnn/core/cnn_layers.rb

@@ -4,7 +4,7 @@ module DNN
     module Conv2DModule
       private
 
-      # img[bsize, out_h, out_w, channel] to col[bsize * out_h * out_w, fil_h * fil_w * ch]
+      # img[bsize, out_h, out_w, ch] to col[bsize * out_h * out_w, fil_h * fil_w * ch]
       def im2col(img, out_h, out_w, fil_h, fil_w, strides)
         bsize = img.shape[0]
         ch = img.shape[3]
@@ -19,7 +19,7 @@ module DNN
         col.reshape(bsize * out_h * out_w, fil_h * fil_w * ch)
       end
 
-      # col[bsize * out_h * out_w, fil_h * fil_w * ch] to img[bsize, out_h, out_w, channel]
+      # col[bsize * out_h * out_w, fil_h * fil_w * ch] to img[bsize, out_h, out_w, ch]
       def col2im(col, img_shape, out_h, out_w, fil_h, fil_w, strides)
         bsize, img_h, img_w, ch = img_shape
         col = col.reshape(bsize, out_h, out_w, fil_h, fil_w, ch)
@@ -84,7 +84,8 @@ module DNN
                    strides: hash[:strides],
                    padding: hash[:padding],
                    l1_lambda: hash[:l1_lambda],
-                   l2_lambda: hash[:l2_lambda])
+                   l2_lambda: hash[:l2_lambda],
+                   use_bias: hash[:use_bias])
       end
       
       # @param [Integer] num_filters number of filters.
@@ -93,13 +94,14 @@ module DNN
       # @param [Bool] padding Whether to padding.
       def initialize(num_filters, filter_size,
                      weight_initializer: Initializers::RandomNormal.new,
-                     bias_initializer: Initializers::RandomNormal.new,
+                     bias_initializer: Initializers::Zeros.new,
                      strides: 1,
                      padding: false,
                      l1_lambda: 0,
-                     l2_lambda: 0)
+                     l2_lambda: 0,
+                     use_bias: true)
         super(weight_initializer: weight_initializer, bias_initializer: bias_initializer,
-              l1_lambda: l1_lambda, l2_lambda: l2_lambda)
+              l1_lambda: l1_lambda, l2_lambda: l2_lambda, use_bias: use_bias)
         @num_filters = num_filters
         @filter_size = filter_size.is_a?(Integer) ? [filter_size, filter_size] : filter_size
         @strides = strides.is_a?(Integer) ? [strides, strides] : strides
@@ -120,14 +122,15 @@ module DNN
         x = padding(x, @pad_size) if @padding
         @x_shape = x.shape
         @col = im2col(x, *@out_size, *@filter_size, @strides)
-        out = @col.dot(@weight.data) + @bias.data
+        out = @col.dot(@weight.data)
+        out += @bias.data if @bias
         out.reshape(x.shape[0], *@out_size, out.shape[3])
       end
 
       def backward(dout)
         dout = dout.reshape(dout.shape[0..2].reduce(:*), dout.shape[3])
         @weight.grad = @col.transpose.dot(dout)
-        @bias.grad = dout.sum(0)
+        @bias.grad = dout.sum(0) if @bias
         dcol = dout.dot(@weight.data.transpose)
         dx = col2im(dcol, @x_shape, *@out_size, *@filter_size, @strides)
         @padding ? back_padding(dx, @pad_size) : dx
@@ -166,7 +169,7 @@ module DNN
       def init_params
         num_prev_filter = @input_shape[2]
         @weight.data = Xumo::SFloat.new(@filter_size.reduce(:*) * num_prev_filter, @num_filters)
-        @bias.data = Xumo::SFloat.new(@num_filters)
+        @bias.data = Xumo::SFloat.new(@num_filters) if @bias
         super()
       end
     end
@@ -305,18 +308,26 @@ module DNN
         @num_channel = input_shape[2]
       end
 
+      include Conv2DModule
+
       def forward(x)
         @x_shape = x.shape
         unpool_h, unpool_w = @unpool_size
         x2 = Xumo::SFloat.zeros(x.shape[0], x.shape[1], unpool_h, x.shape[2], unpool_w, @num_channel)
-        x2[true, true, 0, true, 0, true] = x
+        unpool_h.times do |i|
+          unpool_w.times do |j|
+            x2[true, true, i, true, j, true] = x
+          end
+        end
         x2.reshape(x.shape[0], *@out_size, x.shape[3])
       end
 
       def backward(dout)
-        unpool_h, unpool_w = @unpool_size
-        dout = dout.reshape(dout.shape[0], @x_shape[1], unpool_h, @x_shape[2], unpool_w, @num_channel)
-        dout[true, true, 0, true, 0, true].clone
+        in_size = input_shape[0..1]
+        col = im2col(dout, *input_shape[0..1], *@unpool_size, @unpool_size)
+        col = col.reshape(dout.shape[0] * in_size.reduce(:*), @unpool_size.reduce(:*), dout.shape[3]).transpose(0, 2, 1)
+                 .reshape(dout.shape[0] * in_size.reduce(:*) * dout.shape[3], @unpool_size.reduce(:*))
+        col.sum(1).reshape(dout.shape[0], *in_size, dout.shape[3])
       end
 
       def output_shape

data/lib/dnn/core/layers.rb

@@ -113,26 +113,34 @@ module DNN
       attr_reader :weight_initializer
       # @return [DNN::Initializers] bias initializer.
       attr_reader :bias_initializer
-      # @return [Float] L1 regularization
+      # @return [Float] L1 regularization.
       attr_reader :l1_lambda
-      # @return [Float] L2 regularization
+      # @return [Float] L2 regularization.
       attr_reader :l2_lambda
 
       # @param [DNN::Initializers] weight_initializer weight initializer.
       # @param [DNN::Initializers] bias_initializer bias initializer.
       # @param [Float] l1_lambda L1 regularization
       # @param [Float] l2_lambda L2 regularization
+      # @param [Bool] use_bias whether to use bias.
       def initialize(weight_initializer: Initializers::RandomNormal.new,
                      bias_initializer: Initializers::Zeros.new,
                      l1_lambda: 0,
-                     l2_lambda: 0)
+                     l2_lambda: 0,
+                     use_bias: true)
         super()
         @weight_initializer = weight_initializer
         @bias_initializer = bias_initializer
         @l1_lambda = l1_lambda
         @l2_lambda = l2_lambda
         @params[:weight] = @weight = Param.new
-        @params[:bias] = @bias = Param.new
+        # For compatibility on or before with v0.9.3, setting use_bias to nil use bias.
+        # Therefore, setting use_bias to nil is deprecated.
+        if use_bias || use_bias == nil
+          @params[:bias] = @bias = Param.new
+        else
+          @params[:bias] = @bias = nil
+        end
       end
 
       def regularizers
@@ -142,6 +150,11 @@ module DNN
         regularizers
       end
 
+      # @return [Bool] Return whether to use bias.
+      def use_bias
+        @bias ? true : false
+      end
+
       def to_hash(merge_hash)
         super({weight_initializer: @weight_initializer.to_hash,
                bias_initializer: @bias_initializer.to_hash,
@@ -153,7 +166,7 @@ module DNN
 
       def init_params
         @weight_initializer.init_param(self, @weight)
-        @bias_initializer.init_param(self, @bias)
+        @bias_initializer.init_param(self, @bias) if @bias
       end
     end
     
@@ -168,7 +181,8 @@ module DNN
                  weight_initializer: Utils.load_hash(hash[:weight_initializer]),
                  bias_initializer: Utils.load_hash(hash[:bias_initializer]),
                  l1_lambda: hash[:l1_lambda],
-                 l2_lambda: hash[:l2_lambda])
+                 l2_lambda: hash[:l2_lambda],
+                 use_bias: hash[:use_bias])
       end
 
       # @param [Integer] num_nodes number of nodes.
@@ -176,20 +190,23 @@ module DNN
                      weight_initializer: Initializers::RandomNormal.new,
                      bias_initializer: Initializers::Zeros.new,
                      l1_lambda: 0,
-                     l2_lambda: 0)
+                     l2_lambda: 0,
+                     use_bias: true)
         super(weight_initializer: weight_initializer, bias_initializer: bias_initializer,
-              l1_lambda: l1_lambda, l2_lambda: l2_lambda)
+              l1_lambda: l1_lambda, l2_lambda: l2_lambda, use_bias: use_bias)
         @num_nodes = num_nodes
       end
     
       def forward(x)
         @x = x
-        @x.dot(@weight.data) + @bias.data
+        out = x.dot(@weight.data)
+        out += @bias.data if @bias
+        out
       end
     
       def backward(dout)
         @weight.grad = @x.transpose.dot(dout)
-        @bias.grad = dout.sum(0)
+        @bias.grad = dout.sum(0) if @bias
         dout.dot(@weight.data.transpose)
       end
     
@@ -208,7 +225,7 @@ module DNN
       def init_params
         num_prev_nodes = @input_shape[0]
         @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
-        @bias.data = Xumo::SFloat.new(@num_nodes)
+        @bias.data = Xumo::SFloat.new(@num_nodes) if @bias
         super()
       end
     end

data/lib/dnn/core/losses.rb

@@ -100,8 +100,12 @@ module DNN
 
 
     class SoftmaxCrossEntropy < Loss
+      def self.softmax(x)
+        NMath.exp(x) / NMath.exp(x).sum(1).reshape(x.shape[0], 1)
+      end
+
       def loss(x, y)
-        @out = Utils.softmax(x)
+        @out = SoftmaxCrossEntropy.softmax(x)
         batch_size = y.shape[0]
         -(y * NMath.log(@out + 1e-7)).sum / batch_size
       end
@@ -113,10 +117,14 @@ module DNN
 
 
     class SigmoidCrossEntropy < Loss
+      def initialize
+        @sigmoid = Sigmoid.new
+      end
+
       def loss(x, y)
-        @out = Utils.sigmoid(x)
+        @out = @sigmoid.forward(x)
         batch_size = y.shape[0]
-        -(y * NMath.log(@out + 1e-7) + (1 - y) * NMath.log(1 - @out + 1e-7)).sum / batch_size
+        -(y * NMath.log(@out + 1e-7) + (1 - y) * NMath.log(1 - @out + 1e-7))
       end
 
       def backward(y)

data/lib/dnn/core/model.rb

@@ -213,8 +213,10 @@ module DNN
         puts "【 epoch #{epoch}/#{epochs} 】" if verbose
         (num_train_datas.to_f / batch_size).ceil.times do |index|
           x_batch, y_batch = dataset.get_batch(batch_size)
-          loss = train_on_batch(x_batch, y_batch, &batch_proc)
-          if loss.nan?
+          loss_value = train_on_batch(x_batch, y_batch, &batch_proc)
+          if loss_value.is_a?(Numo::SFloat)
+            loss_value = loss_value.mean
+          elsif loss_value.nan?
             puts "\nloss is nan" if verbose
             return
           end
@@ -230,7 +232,7 @@ module DNN
               log << "_"
             end
           end
-          log << "  #{num_trained_datas}/#{num_train_datas} loss: #{sprintf('%.8f', loss)}"
+          log << "  #{num_trained_datas}/#{num_train_datas} loss: #{sprintf('%.8f', loss_value)}"
           print log if verbose
         end
         if verbose && test
@@ -246,6 +248,7 @@ module DNN
     # Compile the model before use this method.
     # @param [Numo::SFloat] x Input training data.
     # @param [Numo::SFloat] y Output training data.
+    # @return [Float | Numo::SFloat] Return loss value in the form of Float or Numo::SFloat.
     # @yield [x, y] batch_proc Set proc to process per batch.
     def train_on_batch(x, y, &batch_proc)
       raise DNN_Error.new("The model is not compiled.") unless compiled?
@@ -304,7 +307,7 @@ module DNN
     # @param [Numo::SFloat] x Input data. However, x is single data.
     def predict1(x)
       check_xy_type(x)
-      predict(Xumo::SFloat.cast([x]))[0, false]
+      predict(x.reshape(1, *x.shape))[0, false]
     end
 
     # @return [DNN::Model] Copy this model.
@@ -333,7 +336,7 @@ module DNN
   
     # TODO
     # It is not good to write the Layer class name directly in the Model class. I will fix it later.
-    def forward(x, learning_phase)01
+    def forward(x, learning_phase)
       @layers.each do |layer|
         x = if layer.is_a?(Layers::Dropout) || layer.is_a?(Layers::BatchNormalization) || layer.is_a?(Model)
           layer.forward(x, learning_phase)

data/lib/dnn/core/rnn_layers.rb

@@ -18,9 +18,10 @@ module DNN
                      weight_initializer: RandomNormal.new,
                      bias_initializer: Zeros.new,
                      l1_lambda: 0,
-                     l2_lambda: 0)
+                     l2_lambda: 0,
+                     use_bias: true)
         super(weight_initializer: weight_initializer, bias_initializer: bias_initializer,
-              l1_lambda: l1_lambda, l2_lambda: l2_lambda)
+              l1_lambda: l1_lambda, l2_lambda: l2_lambda, use_bias: use_bias)
         @num_nodes = num_nodes
         @stateful = stateful
         @return_sequences = return_sequences
@@ -47,7 +48,7 @@ module DNN
       def backward(dh2s)
         @weight.grad = Xumo::SFloat.zeros(*@weight.data.shape)
         @weight2.grad = Xumo::SFloat.zeros(*@weight2.data.shape)
-        @bias.grad = Xumo::SFloat.zeros(*@bias.data.shape)
+        @bias.grad = Xumo::SFloat.zeros(*@bias.data.shape) if @bias
         unless @return_sequences
           dh = dh2s
           dh2s = Xumo::SFloat.zeros(dh.shape[0], @time_length, dh.shape[1])
@@ -114,7 +115,8 @@ module DNN
       def forward(x, h)
         @x = x
         @h = h
-        h2 = x.dot(@weight.data) + h.dot(@weight2.data) + @bias.data
+        h2 = x.dot(@weight.data) + h.dot(@weight2.data)
+        h2 += @bias.data if @bias
         @activation.forward(h2)
       end
 
@@ -122,7 +124,7 @@ module DNN
         dh2 = @activation.backward(dh2)
         @weight.grad += @x.transpose.dot(dh2)
         @weight2.grad += @h.transpose.dot(dh2)
-        @bias.grad += dh2.sum(0)
+        @bias.grad += dh2.sum(0) if @bias
         dx = dh2.dot(@weight.data.transpose)
         dh = dh2.dot(@weight2.data.transpose)
         [dx, dh]
@@ -143,7 +145,8 @@ module DNN
                               weight_initializer: Utils.load_hash(hash[:weight_initializer]),
                               bias_initializer: Utils.load_hash(hash[:bias_initializer]),
                               l1_lambda: hash[:l1_lambda],
-                              l2_lambda: hash[:l2_lambda])
+                              l2_lambda: hash[:l2_lambda],
+                              use_bias: hash[:use_bias])
         simple_rnn
       end
 
@@ -154,14 +157,16 @@ module DNN
                      weight_initializer: RandomNormal.new,
                      bias_initializer: Zeros.new,
                      l1_lambda: 0,
-                     l2_lambda: 0)
+                     l2_lambda: 0,
+                     use_bias: true)
         super(num_nodes,
               stateful: stateful,
               return_sequences: return_sequences,
               weight_initializer: weight_initializer,
               bias_initializer: bias_initializer,
               l1_lambda: l1_lambda,
-              l2_lambda: l2_lambda)
+              l2_lambda: l2_lambda,
+              use_bias: use_bias)
         @activation = activation
       end
 
@@ -176,10 +181,10 @@ module DNN
         num_prev_nodes = @input_shape[1]
         @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
         @weight2.data = Xumo::SFloat.new(@num_nodes, @num_nodes)
-        @bias.data = Xumo::SFloat.new(@num_nodes)
+        @bias.data = Xumo::SFloat.new(@num_nodes) if @bias
         @weight_initializer.init_param(self, @weight)
         @weight_initializer.init_param(self, @weight2)
-        @bias_initializer.init_param(self, @bias)
+        @bias_initializer.init_param(self, @bias) if @bias
         @time_length.times do |t|
           @layers << SimpleRNN_Dense.new(@weight, @weight2, @bias, @activation)
         end
@@ -204,7 +209,8 @@ module DNN
         @h = h
         @c = c
         num_nodes = h.shape[1]
-        a = x.dot(@weight.data) + h.dot(@weight2.data) + @bias.data
+        a = x.dot(@weight.data) + h.dot(@weight2.data)
+        a += @bias.data if @bias
 
         @forget = @forget_sigmoid.forward(a[true, 0...num_nodes])
         @g = @g_tanh.forward(a[true, num_nodes...(num_nodes * 2)])
@@ -230,7 +236,7 @@ module DNN
 
         @weight.grad += @x.transpose.dot(da)
         @weight2.grad += @h.transpose.dot(da)
-        @bias.grad += da.sum(0)
+        @bias.grad += da.sum(0) if @bias
         dx = da.dot(@weight.data.transpose)
         dh = da.dot(@weight2.data.transpose)
         dc = dc2_tmp * @forget
@@ -247,7 +253,8 @@ module DNN
                         weight_initializer: Utils.load_hash(hash[:weight_initializer]),
                         bias_initializer: Utils.load_hash(hash[:bias_initializer]),
                         l1_lambda: hash[:l1_lambda],
-                        l2_lambda: hash[:l2_lambda])
+                        l2_lambda: hash[:l2_lambda],
+                        use_bias: hash[:use_bias])
         lstm
       end
 
@@ -257,7 +264,8 @@ module DNN
                      weight_initializer: RandomNormal.new,
                      bias_initializer: Zeros.new,
                      l1_lambda: 0,
-                     l2_lambda: 0)
+                     l2_lambda: 0,
+                     use_bias: true)
         super
         @cell = @params[:c] = Param.new
       end
@@ -286,7 +294,7 @@ module DNN
       def backward(dh2s)
         @weight.grad = Xumo::SFloat.zeros(*@weight.data.shape)
         @weight2.grad = Xumo::SFloat.zeros(*@weight2.data.shape)
-        @bias.grad = Xumo::SFloat.zeros(*@bias.data.shape)
+        @bias.grad = Xumo::SFloat.zeros(*@bias.data.shape) if @bias
         unless @return_sequences
           dh = dh2s
           dh2s = Xumo::SFloat.zeros(dh.shape[0], @time_length, dh.shape[1])
@@ -315,10 +323,10 @@ module DNN
         num_prev_nodes = @input_shape[1]
         @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes * 4)
         @weight2.data = Xumo::SFloat.new(@num_nodes, @num_nodes * 4)
-        @bias.data = Xumo::SFloat.new(@num_nodes * 4)
+        @bias.data = Xumo::SFloat.new(@num_nodes * 4) if @bias
         @weight_initializer.init_param(self, @weight)
         @weight_initializer.init_param(self, @weight2)
-        @bias_initializer.init_param(self, @bias)
+        @bias_initializer.init_param(self, @bias) if @bias
         @time_length.times do |t|
           @layers << LSTM_Dense.new(@weight, @weight2, @bias)
         end
@@ -342,15 +350,19 @@ module DNN
         num_nodes = h.shape[1]
         @weight_a = @weight.data[true, 0...(num_nodes * 2)]
         @weight2_a = @weight2.data[true, 0...(num_nodes * 2)]
-        bias_a = @bias.data[0...(num_nodes * 2)]
-        a = x.dot(@weight_a) + h.dot(@weight2_a) + bias_a
+        a = x.dot(@weight_a) + h.dot(@weight2_a)
+        a += @bias.data[0...(num_nodes * 2)] if @bias
         @update = @update_sigmoid.forward(a[true, 0...num_nodes])
         @reset = @reset_sigmoid.forward(a[true, num_nodes..-1])
 
         @weight_h = @weight.data[true, (num_nodes * 2)..-1]
         @weight2_h = @weight2.data[true, (num_nodes * 2)..-1]
-        bias_h = @bias.data[(num_nodes * 2)..-1]
-        @tanh_h = @tanh.forward(x.dot(@weight_h) + (h * @reset).dot(@weight2_h) + bias_h)
+        @tanh_h = if @bias
+          bias_h = @bias.data[(num_nodes * 2)..-1]
+          @tanh.forward(x.dot(@weight_h) + (h * @reset).dot(@weight2_h) + bias_h)
+        else
+          @tanh.forward(x.dot(@weight_h) + (h * @reset).dot(@weight2_h))
+        end
         h2 = (1 - @update) * h + @update * @tanh_h
         h2
       end
@@ -363,7 +375,7 @@ module DNN
         dx = dtanh_h.dot(@weight_h.transpose)
         dweight2_h = (@h * @reset).transpose.dot(dtanh_h)
         dh += dtanh_h.dot(@weight2_h.transpose) * @reset
-        dbias_h = dtanh_h.sum(0)
+        dbias_h = dtanh_h.sum(0) if @bias
 
         dreset = @reset_sigmoid.backward(dtanh_h.dot(@weight2_h.transpose) * @h)
         dupdate = @update_sigmoid.backward(dh2 * @tanh_h - dh2 * @h)
@@ -372,11 +384,11 @@ module DNN
         dx += da.dot(@weight_a.transpose)
         dweight2_a = @h.transpose.dot(da)
         dh += da.dot(@weight2_a.transpose)
-        dbias_a = da.sum(0)
+        dbias_a = da.sum(0) if @bias
 
         @weight.grad += Xumo::SFloat.hstack([dweight_a, dweight_h])
         @weight2.grad += Xumo::SFloat.hstack([dweight2_a, dweight2_h])
-        @bias.grad += Xumo::SFloat.hstack([dbias_a, dbias_h])
+        @bias.grad += Xumo::SFloat.hstack([dbias_a, dbias_h]) if @bias
         [dx, dh]
       end
     end
@@ -390,7 +402,8 @@ module DNN
                        weight_initializer: Utils.load_hash(hash[:weight_initializer]),
                        bias_initializer: Utils.load_hash(hash[:bias_initializer]),
                        l1_lambda: hash[:l1_lambda],
-                       l2_lambda: hash[:l2_lambda])
+                       l2_lambda: hash[:l2_lambda],
+                       use_bias: hash[:use_bias])
         gru
       end
 
@@ -400,7 +413,8 @@ module DNN
                      weight_initializer: RandomNormal.new,
                      bias_initializer: Zeros.new,
                      l1_lambda: 0,
-                     l2_lambda: 0)
+                     l2_lambda: 0,
+                     use_bias: true)
         super
       end
 
@@ -411,10 +425,10 @@ module DNN
         num_prev_nodes = @input_shape[1]
         @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes * 3)
         @weight2.data = Xumo::SFloat.new(@num_nodes, @num_nodes * 3)
-        @bias.data = Xumo::SFloat.new(@num_nodes * 3)
+        @bias.data = Xumo::SFloat.new(@num_nodes * 3) if @bias
         @weight_initializer.init_param(self, @weight)
         @weight_initializer.init_param(self, @weight2)
-        @bias_initializer.init_param(self, @bias)
+        @bias_initializer.init_param(self, @bias) if @bias
         @time_length.times do |t|
           @layers << GRU_Dense.new(@weight, @weight2, @bias)
         end

data/lib/dnn/core/utils.rb

@@ -20,14 +20,14 @@ module DNN
       dnn_class.new
     end
 
-    # TODO
-    # Don't want to write an implementation of the activation function in utils, so we will consider it later.
+    # Return the result of the sigmoid function.
     def self.sigmoid(x)
-      1 / (1 + NMath.exp(-x))
+      Sigmoid.new.forward(x)
     end
 
+    # Return the result of the softmax function.
     def self.softmax(x)
-      NMath.exp(x) / NMath.exp(x).sum(1).reshape(x.shape[0], 1)
+      SoftmaxCrossEntropy.softmax(x)
     end
   end
 end

data/lib/dnn/lib/iris.rb

@@ -0,0 +1,60 @@
+require "csv"
+require_relative "downloader"
+
+module DNN
+  class DNN_Iris_LoadError < DNN_Error; end
+
+  module Iris
+    URL_CSV = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data"
+
+    # Iris-setosa
+    SETOSA = 0
+    # Iris-versicolor
+    VERSICOLOR = 1
+    # Iris-virginica
+    VIRGINICA = 2
+
+    def self.downloads
+      return if File.exist?(url_to_file_name(URL_CSV))
+      Downloader.download(URL_CSV)
+    end
+
+    def self.load(shuffle = false, shuffle_seed = rand(1 << 31))
+      downloads
+      csv_array = CSV.read(url_to_file_name(URL_CSV)).select { |a| a.length > 0 }
+      x = Numo::SFloat.zeros(csv_array.length, 4)
+      y = Numo::SFloat.zeros(csv_array.length)
+      csv_array.each.with_index do |(sepal_length, sepal_width, petal_length, petal_width, classes), i|
+        x[i, 0] = sepal_length.to_f
+        x[i, 1] = sepal_width.to_f
+        x[i, 2] = petal_length.to_f
+        x[i, 3] = petal_width.to_f
+        y[i] = case classes
+        when "Iris-setosa"
+          SETOSA
+        when "Iris-versicolor"
+          VERSICOLOR
+        when "Iris-virginica"
+          VIRGINICA
+        else
+          raise DNN_Iris_LoadError.new("Unknown class name '#{classes}' for iris")
+        end
+      end
+      if shuffle
+        orig_seed = Random::DEFAULT.seed
+        srand(shuffle_seed)
+        indexs = (0...csv_array.length).to_a.shuffle
+        x[indexs, true] = x
+        y[indexs] = y
+        srand(orig_seed)
+      end
+      [x, y]
+    end
+
+    private_class_method
+
+    def self.url_to_file_name(url)
+      __dir__ + "/" + url.match(%r`.+/(.+)$`)[1]
+    end
+  end
+end

data/lib/dnn/version.rb

@@ -1,3 +1,3 @@
 module DNN
-  VERSION = "0.9.3"
+  VERSION = "0.9.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-dnn
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.9.4
 platform: ruby
 authors:
 - unagiootoro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-08 00:00:00.000000000 Z
+date: 2019-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -101,6 +101,7 @@ files:
 - bin/console
 - bin/setup
 - examples/cifar10_example.rb
+- examples/iris_example.rb
 - examples/mnist_conv2d_example.rb
 - examples/mnist_example.rb
 - examples/mnist_lstm_example.rb
@@ -126,6 +127,7 @@ files:
 - lib/dnn/lib/cifar10.rb
 - lib/dnn/lib/downloader.rb
 - lib/dnn/lib/image.rb
+- lib/dnn/lib/iris.rb
 - lib/dnn/lib/mnist.rb
 - lib/dnn/version.rb
 - ruby-dnn.gemspec