ruby-dnn 1.1.4 → 1.2.2

data/examples/pix2pix/train.rb

@@ -23,8 +23,8 @@ epochs = 20
 batch_size = 128
 
 if initial_epoch == 1
-  gen = Generator.new([32, 32, 1])
-  dis = Discriminator.new([32, 32, 1], [32, 32, 3])
+  gen = Generator.new([32, 32, 1], 32)
+  dis = Discriminator.new([32, 32, 1], [32, 32, 3], 32)
   dcgan = DCGAN.new(gen, dis)
   gen.setup(Adam.new(alpha: 0.0002, beta1: 0.5), MeanAbsoluteError.new)
   dis.setup(Adam.new(alpha: 0.00001, beta1: 0.1), SigmoidCrossEntropy.new)

data/examples/vae.rb

@@ -28,24 +28,24 @@ end
 class Encoder < Model
   def initialize
     super
-    @l1 = Dense.new(196)
-    @l2 = Dense.new(49)
-    @l3_1 = Dense.new($z_dim)
-    @l3_2 = Dense.new($z_dim)
+    @d1 = Dense.new(196)
+    @d2 = Dense.new(49)
+    @d3_1 = Dense.new($z_dim)
+    @d3_2 = Dense.new($z_dim)
     @bn1 = BatchNormalization.new
     @bn2 = BatchNormalization.new
   end
 
   def forward(x)
     x = InputLayer.new(784).(x)
-    x = @l1.(x)
+    x = @d1.(x)
     x = @bn1.(x)
     x = ReLU.(x)
-    x = @l2.(x)
+    x = @d2.(x)
     x = @bn2.(x)
     x = ReLU.(x)
-    z_mean = @l3_1.(x)
-    z_sigma = @l3_2.(x)
+    z_mean = @d3_1.(x)
+    z_sigma = @d3_2.(x)
     [z_mean, z_sigma]
   end
 end
@@ -53,16 +53,16 @@ end
 class Decoder < Model
   def initialize
     super
-    @l3 = Dense.new(196)
-    @l4 = Dense.new(784)
+    @d1 = Dense.new(196)
+    @d2 = Dense.new(784)
     @bn1 = BatchNormalization.new
   end
 
   def forward(z)
-    x = @l3.(z)
+    x = @d1.(z)
     x = @bn1.(x)
     x = ReLU.(x)
-    x = @l4.(x)
+    x = @d2.(x)
     x
   end
 end
@@ -97,7 +97,7 @@ model = VAE.new
 dec = model.dec
 model.setup(Adam.new, VAELoss.new)
 
-model.train(x_train, x_train, 10, batch_size: 100)
+model.train(x_train, x_train, 10, batch_size: 128)
 
 images = []
 10.times do |i|

data/lib/dnn.rb

@@ -1,9 +1,29 @@
+require "numo/narray"
+
 module DNN
-  if defined? ::Cumo
+  if ENV["RUBY_DNN_USE_CUMO"] == "ENABLE"
+    require "cumo/narray"
     Xumo = ::Cumo
   else
-    require "numo/narray"
-    Xumo = ::Numo
+    if defined? ::Cumo
+      Xumo = ::Cumo
+    else
+      Xumo = ::Numo
+    end
+  end
+
+  def self.use_cumo?
+    defined? ::Cumo
+  end
+
+  def self.cudnn_available?
+    return false unless defined? ::Cumo
+    Cumo::CUDA::CUDNN.available?
+  end
+
+  def self.use_cudnn?
+    return false unless ENV["RUBY_DNN_USE_CUDNN"] == "ENABLE"
+    cudnn_available?
   end
 end
 
@@ -20,6 +40,7 @@ require_relative "dnn/core/layers/basic_layers"
 require_relative "dnn/core/layers/normalizations"
 require_relative "dnn/core/layers/activations"
 require_relative "dnn/core/layers/merge_layers"
+require_relative "dnn/core/layers/split_layers"
 require_relative "dnn/core/layers/cnn_layers"
 require_relative "dnn/core/layers/embedding"
 require_relative "dnn/core/layers/rnn_layers"

data/lib/dnn/core/callbacks.rb

@@ -104,6 +104,7 @@ module DNN
     # A callback that save the log.
     # The following logs will be recorded.
     # epoch:          Current epoch.
+    # step:           Current step in epoch.
     # train_loss:     Batch training loss.
     # test_loss:      Mean test loss.
     # test_accuracy:  Test accuracy.
@@ -111,6 +112,7 @@ module DNN
       def initialize
         @log = {
           epoch: [],
+          step: [],
           train_loss: [],
           test_loss: [],
           test_accuracy: [],
@@ -122,7 +124,7 @@ module DNN
       end
 
       def after_train_on_batch
-        logging(:train_loss)
+        logging(:train_loss, :step)
       end
 
       # Get a log.
@@ -130,10 +132,10 @@ module DNN
       # @return [Numo::NArray] Return the recorded log.
       def get_log(tag)
         case tag
-        when :epoch
-          Numo::UInt32.cast(@log[tag])
+        when :epoch, :step
+          Xumo::UInt32.cast(@log[tag])
         else
-          Numo::SFloat.cast(@log[tag])
+          Xumo::SFloat.cast(@log[tag])
         end
       end
 

data/lib/dnn/core/layers/basic_layers.rb

@@ -2,20 +2,21 @@ module DNN
   module Layers
 
     module LayerNode
-      def forward(input)
-        x = input.data
-        prev = (input.is_a?(Tensor) ? input.link : input)
-        y = forward_node(x)
-        link = Link.new(prev, self)
-        prev.next = link if prev.is_a?(Link)
-        Tensor.convert(y, link)
+      def forward(*inputs)
+        xs = inputs.map(&:data)
+        prevs = inputs.map { |input| input.is_a?(Tensor) ? input.link : input }
+        ys = forward_node(*xs)
+        num_outputs = (ys.is_a?(Array) ? ys.length : 1)
+        link = Link.new(prevs: prevs, layer_node: self, num_outputs: num_outputs)
+        prevs.map { |prev| prev.next = link if prev.is_a?(Link) }
+        Tensor.convert(ys, link)
       end
 
-      def forward_node(x)
+      def forward_node(*xs)
         raise NotImplementedError, "Class '#{self.class.name}' has implement method 'forward_node'"
       end
 
-      def backward_node(dy)
+      def backward_node(*dys)
         raise NotImplementedError, "Class '#{self.class.name}' has implement method 'backward_node'"
       end
     end
@@ -292,14 +293,8 @@ module DNN
     end
 
     class Flatten < Layer
-      include LayerNode
-
-      def forward_node(x)
-        x.reshape(x.shape[0], *@output_shape)
-      end
-
-      def backward_node(dy)
-        dy.reshape(dy.shape[0], *@input_shape)
+      def forward(x)
+        Reshape.(x, @output_shape)
       end
 
       def compute_output_shape
@@ -320,13 +315,37 @@ module DNN
       end
 
       def forward_node(x)
-        x.reshape(x.shape[0], *@output_shape)
+        if DNN.use_cumo?
+          _forward_gpu(x)
+        else
+          _forward_cpu(x)
+        end
       end
 
       def backward_node(dy)
+        if DNN.use_cumo?
+          _backward_gpu(dy)
+        else
+          _backward_cpu(dy)
+        end
+      end
+
+      def _forward_cpu(x)
+        x.reshape(x.shape[0], *@output_shape)
+      end
+
+      def _backward_cpu(dy)
         dy.reshape(dy.shape[0], *@input_shape)
       end
 
+      def _forward_gpu(x)
+        x.flatten.reshape(x.shape[0], *@output_shape)
+      end
+
+      def _backward_gpu(dy)
+        dy.flatten.reshape(dy.shape[0], *@input_shape)
+      end
+
       def to_hash
         super(shape: @shape)
       end
@@ -417,8 +436,8 @@ module DNN
       def forward_node(x)
         if DNN.learning_phase
           Xumo::SFloat.srand(@rnd.rand(1 << 31))
-          @mask = Xumo::SFloat.new(*x.shape).rand < @dropout_ratio
-          x[@mask] = 0
+          @mask = Xumo::SFloat.cast(Xumo::SFloat.new(*x.shape).rand >= @dropout_ratio)
+          x = x * @mask
         elsif @use_scale
           x *= (1 - @dropout_ratio)
         end
@@ -426,8 +445,7 @@ module DNN
       end
 
       def backward_node(dy)
-        dy[@mask] = 0
-        dy
+        dy * @mask
       end
 
       def to_hash

data/lib/dnn/core/layers/cnn_layers.rb

@@ -6,10 +6,27 @@ module DNN
       module_function
 
       # img[bsize, out_h, out_w, ch] to col[bsize * out_h * out_w, fil_h * fil_w * ch]
-      def im2col(img, out_h, out_w, fil_h, fil_w, strides)
+      def im2col(*args)
+        if DNN.use_cumo?
+          im2col_gpu(*args)
+        else
+          im2col_cpu(*args)
+        end
+      end
+
+      # col[bsize * out_h * out_w, fil_h * fil_w * ch] to img[bsize, out_h, out_w, ch]
+      def col2im(*args)
+        if DNN.use_cumo?
+          col2im_gpu(*args)
+        else
+          col2im_cpu(*args)
+        end
+      end
+
+      def im2col_cpu(img, out_h, out_w, fil_h, fil_w, strides)
         bsize = img.shape[0]
         ch = img.shape[3]
-        col = Xumo::SFloat.zeros(bsize, out_h, out_w, fil_h, fil_w, ch)
+        col = img.class.zeros(bsize, out_h, out_w, fil_h, fil_w, ch)
         (0...fil_h).each do |i|
           i_range = (i...(i + strides[0] * out_h)).step(strides[0]).to_a
           (0...fil_w).each do |j|
@@ -20,11 +37,16 @@ module DNN
         col.reshape(bsize * out_h * out_w, fil_h * fil_w * ch)
       end
 
-      # col[bsize * out_h * out_w, fil_h * fil_w * ch] to img[bsize, out_h, out_w, ch]
-      def col2im(col, img_shape, out_h, out_w, fil_h, fil_w, strides)
+      def im2col_gpu(img, out_h, out_w, fil_h, fil_w, strides)
+        img = Utils.cumo2numo(img)
+        col = im2col_cpu(img, out_h, out_w, fil_h, fil_w, strides)
+        Utils.numo2cumo(col)
+      end
+
+      def col2im_cpu(col, img_shape, out_h, out_w, fil_h, fil_w, strides)
         bsize, img_h, img_w, ch = img_shape
         col = col.reshape(bsize, out_h, out_w, fil_h, fil_w, ch)
-        img = Xumo::SFloat.zeros(bsize, img_h, img_w, ch)
+        img = col.class.zeros(bsize, img_h, img_w, ch)
         (0...fil_h).each do |i|
           i_range = (i...(i + strides[0] * out_h)).step(strides[0]).to_a
           (0...fil_w).each do |j|
@@ -35,6 +57,12 @@ module DNN
         img
       end
 
+      def col2im_gpu(col, img_shape, out_h, out_w, fil_h, fil_w, strides)
+        col = Utils.cumo2numo(col)
+        img = col2im_cpu(col, img_shape, out_h, out_w, fil_h, fil_w, strides)
+        Utils.numo2cumo(img)
+      end
+
       def zero_padding(img, pad)
         bsize, img_h, img_w, ch = img.shape
         img2 = Xumo::SFloat.zeros(bsize, img_h + pad[0], img_w + pad[1], ch)

data/lib/dnn/core/layers/math_layers.rb

@@ -61,7 +61,7 @@ module DNN
     end
 
     class Add < MergeLayer
-      include MergeLayerNode
+      include LayerNode
 
       def forward_node(x1, x2)
         @x1_shape = x1.shape
@@ -77,7 +77,7 @@ module DNN
     end
 
     class Sub < MergeLayer
-      include MergeLayerNode
+      include LayerNode
 
       def forward_node(x1, x2)
         @x1_shape = x1.shape
@@ -93,7 +93,7 @@ module DNN
     end
 
     class Mul < MergeLayer
-      include MergeLayerNode
+      include LayerNode
 
       def forward_node(x1, x2)
         @x1, @x2 = x1, x2
@@ -108,7 +108,7 @@ module DNN
     end
 
     class Div < MergeLayer
-      include MergeLayerNode
+      include LayerNode
 
       def forward_node(x1, x2)
         @x1, @x2 = x1, x2
@@ -123,7 +123,7 @@ module DNN
     end
 
     class Dot < MergeLayer
-      include MergeLayerNode
+      include LayerNode
 
       def forward_node(x1, x2)
         @x1, @x2 = x1, x2
@@ -205,8 +205,11 @@ module DNN
 
       def forward_node(x)
         @x_shape = x.shape
-        @dim = x.shape[@axis]
-        x.sum(axis: @axis, keepdims: true)
+        if @axis
+          x.sum(axis: @axis, keepdims: true)
+        else
+          x.sum
+        end
       end
 
       def backward_node(dy)
@@ -236,8 +239,13 @@ module DNN
 
       def forward_node(x)
         @x_shape = x.shape
-        @dim = x.shape[@axis]
-        x.mean(axis: @axis, keepdims: true)
+        if @axis
+          @dim = x.shape[@axis]
+          x.mean(axis: @axis, keepdims: true)
+        else
+          @dim = x.size
+          x.mean
+        end
       end
 
       def backward_node(dy)

data/lib/dnn/core/layers/merge_layers.rb

@@ -1,30 +1,6 @@
 module DNN
   module Layers
 
-    module MergeLayerNode
-      def forward(input1, input2)
-        x1 = input1.data
-        x2 = input2.data
-        prev1 = (input1.is_a?(Tensor) ? input1.link : input1)
-        prev2 = (input2.is_a?(Tensor) ? input2.link : input2)
-        y = forward_node(x1, x2)
-        link = TwoInputLink.new(prev1, prev2, self)
-        Tensor.convert(y, link)
-      end
-
-      def backward(dy)
-        backward_node(dy)
-      end
-
-      def forward_node(x1, x2)
-        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'forward_node'"
-      end
-
-      def backward_node(dy)
-        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'backward_node'"
-      end
-    end
-
     class MergeLayer < Layer
       def self.call(x1, x2, *args)
         new(*args).call(x1, x2)
@@ -33,7 +9,7 @@ module DNN
       def call(input1, input2)
         input1 = Tensor.convert(input1) if !input1.is_a?(Tensor) && !input1.is_a?(Param)
         input2 = Tensor.convert(input2) if !input2.is_a?(Tensor) && !input2.is_a?(Param)
-        if input1.data.is_a?(Numo::NArray)
+        if input1.data.is_a?(Xumo::NArray)
           build(input1.data.shape[1..-1]) unless built?
         else
           build([1]) unless built?
@@ -43,7 +19,7 @@ module DNN
     end
 
     class Concatenate < MergeLayer
-      include MergeLayerNode
+      include LayerNode
 
       attr_reader :axis