ruby-dnn 1.1.6 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b3228eaf257e50c55bdd51b348e71dcc45d74a7cc14668231c4a5e1c9fed318
-  data.tar.gz: ae3a1217cb1aa0a3d0f50ad2b042fd255385a713c7787ad318f34410170dbb83
+  metadata.gz: d152e645981ddb8f244abc88719c4c68a154679fa18abf6b02fa8852510e0c27
+  data.tar.gz: 2987a4ce59a8f7d60a160ac192125ab97a4e14735ae29e151d23412a22f3c13b
 SHA512:
-  metadata.gz: 977ddba86307314f9af523643e151f397705600f2aa244980dfefbda8af7c74114dd4b17d515c64d30e412d8e6aa3de8a127ba085dd8666e03a99ac2cb1fd573
-  data.tar.gz: 80e1d0a963e066aa74f0e9e9e786113ea8505112990de217b6d3a7b7b9f60b06f5ee777e94fb95fb7f006fd92f7ef4b11b3aaa87f605ae66115c30d007055657
+  metadata.gz: 13e9cb2995d1d2851850931a0aedb01ca4fb71b6b03fa81a0ac9e78176b8ed4f520ac6284a39e9c71aaf2629c55475eac33987e6f49535bfe27153729f37624c
+  data.tar.gz: a492f778d4094fce114149841a54e2b0d8a4eff77778b8074198c8df009916598cb96bfd67494c88520e10fd99618591862854773a1f20baf00247fe2bb1f1ea

data/README.md

@@ -95,7 +95,7 @@ If you want to know more detailed information, please refer to the source code.
 | Losses | MeanSquaredError, MeanAbsoluteError, Hinge, HuberLoss, SoftmaxCrossEntropy, SigmoidCrossEntropy |
 
 ## Datasets
-By setting the environment variable "RUBY_DNN_DOWNLOADS_PATH", you can specify the path to read dataset.
+By setting the environment variable 'RUBY_DNN_DOWNLOADS_PATH', you can specify the path to read dataset.
 
 ● Iris  
 ● MNIST  
@@ -109,10 +109,14 @@ By setting the environment variable "RUBY_DNN_DOWNLOADS_PATH", you can specify t
 ● DCGAN  
 ● Pix2pix
 
+## Use GPU
+If you do 'require "cumo/narray"' before 'require "dnn"', you can run it on GPU.
+Or, set the environment variable 'RUBY_DNN_USE_CUMO 'to'ENABLE' to force the GPU to be used.
+
 ## TODO
 ● Write a test.  
 ● Write a document.  
-● Support to GPU.  
+● Improve performance when using GPU.  
 
 ## Development
 

data/examples/mnist_conv2d_example.rb

@@ -21,13 +21,13 @@ model = Sequential.new
 
 model << InputLayer.new([28, 28, 1])
 
-model << Conv2D.new(16, 5)
+model << Conv2D.new(16, 3)
 model << BatchNormalization.new
 model << ReLU.new
 
 model << MaxPool2D.new(2)
 
-model << Conv2D.new(32, 5)
+model << Conv2D.new(32, 3)
 model << BatchNormalization.new
 model << ReLU.new
 
@@ -42,7 +42,7 @@ model << Dense.new(10)
 
 model.setup(Adam.new, SoftmaxCrossEntropy.new)
 
-model.train(x_train, y_train, 10, batch_size: 100, test: [x_test, y_test])
+model.train(x_train, y_train, 10, batch_size: 128, test: [x_test, y_test])
 
 accuracy, loss = model.evaluate(x_test, y_test)
 puts "accuracy: #{accuracy}"

data/examples/mnist_gpu.rb

@@ -0,0 +1,47 @@
+require "cumo/narray"
+require "dnn"
+require "dnn/datasets/mnist"
+
+include DNN::Models
+include DNN::Layers
+include DNN::Optimizers
+include DNN::Losses
+
+x_train, y_train = DNN::MNIST.load_train
+x_test, y_test = DNN::MNIST.load_test
+
+x_train = x_train.reshape(x_train.shape[0], 784)
+x_test = x_test.reshape(x_test.shape[0], 784)
+
+x_train = Numo::SFloat.cast(x_train) / 255
+x_test = Numo::SFloat.cast(x_test) / 255
+
+y_train = DNN::Utils.to_categorical(y_train, 10, Numo::SFloat)
+y_test = DNN::Utils.to_categorical(y_test, 10, Numo::SFloat)
+
+if DNN.use_cumo?
+  x_train = DNN::Utils.numo2cumo(x_train)
+  y_train = DNN::Utils.numo2cumo(y_train)
+  x_test = DNN::Utils.numo2cumo(x_test)
+  y_test = DNN::Utils.numo2cumo(y_test)
+end
+
+model = Sequential.new
+
+model << InputLayer.new(784)
+
+model << Dense.new(256)
+model << ReLU.new
+
+model << Dense.new(256)
+model << ReLU.new
+
+model << Dense.new(10)
+
+model.setup(Adam.new, SoftmaxCrossEntropy.new)
+
+model.train(x_train, y_train, 10, batch_size: 128, test: [x_test, y_test])
+
+accuracy, loss = model.evaluate(x_test, y_test)
+puts "accuracy: #{accuracy}"
+puts "loss: #{loss}"

data/examples/mnist_lstm_example.rb

@@ -31,7 +31,7 @@ model << Dense.new(10)
 
 model.setup(Adam.new, SoftmaxCrossEntropy.new)
 
-model.train(x_train, y_train, 10, batch_size: 100, test: [x_test, y_test])
+model.train(x_train, y_train, 10, batch_size: 128, test: [x_test, y_test])
 
 accuracy, loss = model.evaluate(x_test, y_test)
 puts "accuracy: #{accuracy}"

data/examples/vae.rb

@@ -97,7 +97,7 @@ model = VAE.new
 dec = model.dec
 model.setup(Adam.new, VAELoss.new)
 
-model.train(x_train, x_train, 10, batch_size: 100)
+model.train(x_train, x_train, 10, batch_size: 128)
 
 images = []
 10.times do |i|

data/lib/dnn.rb

@@ -1,9 +1,29 @@
+require "numo/narray"
+
 module DNN
-  if defined? ::Cumo
+  if ENV["RUBY_DNN_USE_CUMO"] == "ENABLE"
+    require "cumo/narray"
     Xumo = ::Cumo
   else
-    require "numo/narray"
-    Xumo = ::Numo
+    if defined? ::Cumo
+      Xumo = ::Cumo
+    else
+      Xumo = ::Numo
+    end
+  end
+
+  def self.use_cumo?
+    defined? ::Cumo
+  end
+
+  def self.cudnn_available?
+    return false unless defined? ::Cumo
+    Cumo::CUDA::CUDNN.available?
+  end
+
+  def self.use_cudnn?
+    return false unless ENV["RUBY_DNN_USE_CUDNN"] == "ENABLE"
+    cudnn_available?
   end
 end
 
@@ -20,6 +40,7 @@ require_relative "dnn/core/layers/basic_layers"
 require_relative "dnn/core/layers/normalizations"
 require_relative "dnn/core/layers/activations"
 require_relative "dnn/core/layers/merge_layers"
+require_relative "dnn/core/layers/split_layers"
 require_relative "dnn/core/layers/cnn_layers"
 require_relative "dnn/core/layers/embedding"
 require_relative "dnn/core/layers/rnn_layers"

data/lib/dnn/core/callbacks.rb

@@ -104,6 +104,7 @@ module DNN
     # A callback that save the log.
     # The following logs will be recorded.
     # epoch:          Current epoch.
+    # step:           Current step in epoch.
     # train_loss:     Batch training loss.
     # test_loss:      Mean test loss.
     # test_accuracy:  Test accuracy.
@@ -111,6 +112,7 @@ module DNN
       def initialize
         @log = {
           epoch: [],
+          step: [],
           train_loss: [],
           test_loss: [],
           test_accuracy: [],
@@ -122,7 +124,7 @@ module DNN
       end
 
       def after_train_on_batch
-        logging(:train_loss)
+        logging(:train_loss, :step)
       end
 
       # Get a log.
@@ -130,10 +132,10 @@ module DNN
       # @return [Numo::NArray] Return the recorded log.
       def get_log(tag)
         case tag
-        when :epoch
-          Numo::UInt32.cast(@log[tag])
+        when :epoch, :step
+          Xumo::UInt32.cast(@log[tag])
         else
-          Numo::SFloat.cast(@log[tag])
+          Xumo::SFloat.cast(@log[tag])
         end
       end
 

data/lib/dnn/core/layers/basic_layers.rb

@@ -2,20 +2,21 @@ module DNN
   module Layers
 
     module LayerNode
-      def forward(input)
-        x = input.data
-        prev = (input.is_a?(Tensor) ? input.link : input)
-        y = forward_node(x)
-        link = Link.new(prev, self)
-        prev.next = link if prev.is_a?(Link)
-        Tensor.convert(y, link)
+      def forward(*inputs)
+        xs = inputs.map(&:data)
+        prevs = inputs.map { |input| input.is_a?(Tensor) ? input.link : input }
+        ys = forward_node(*xs)
+        num_outputs = (ys.is_a?(Array) ? ys.length : 1)
+        link = Link.new(prevs: prevs, layer_node: self, num_outputs: num_outputs)
+        prevs.map { |prev| prev.next = link if prev.is_a?(Link) }
+        Tensor.convert(ys, link)
       end
 
-      def forward_node(x)
+      def forward_node(*xs)
         raise NotImplementedError, "Class '#{self.class.name}' has implement method 'forward_node'"
       end
 
-      def backward_node(dy)
+      def backward_node(*dys)
         raise NotImplementedError, "Class '#{self.class.name}' has implement method 'backward_node'"
       end
     end
@@ -292,14 +293,8 @@ module DNN
     end
 
     class Flatten < Layer
-      include LayerNode
-
-      def forward_node(x)
-        x.reshape(x.shape[0], *@output_shape)
-      end
-
-      def backward_node(dy)
-        dy.reshape(dy.shape[0], *@input_shape)
+      def forward(x)
+        Reshape.(x, @output_shape)
       end
 
       def compute_output_shape
@@ -320,13 +315,37 @@ module DNN
       end
 
       def forward_node(x)
-        x.reshape(x.shape[0], *@output_shape)
+        if DNN.use_cumo?
+          _forward_gpu(x)
+        else
+          _forward_cpu(x)
+        end
       end
 
       def backward_node(dy)
+        if DNN.use_cumo?
+          _backward_gpu(dy)
+        else
+          _backward_cpu(dy)
+        end
+      end
+
+      def _forward_cpu(x)
+        x.reshape(x.shape[0], *@output_shape)
+      end
+
+      def _backward_cpu(dy)
         dy.reshape(dy.shape[0], *@input_shape)
       end
 
+      def _forward_gpu(x)
+        x.flatten.reshape(x.shape[0], *@output_shape)
+      end
+
+      def _backward_gpu(dy)
+        dy.flatten.reshape(dy.shape[0], *@input_shape)
+      end
+
       def to_hash
         super(shape: @shape)
       end

data/lib/dnn/core/layers/cnn_layers.rb

@@ -6,10 +6,27 @@ module DNN
       module_function
 
       # img[bsize, out_h, out_w, ch] to col[bsize * out_h * out_w, fil_h * fil_w * ch]
-      def im2col(img, out_h, out_w, fil_h, fil_w, strides)
+      def im2col(*args)
+        if DNN.use_cumo?
+          im2col_gpu(*args)
+        else
+          im2col_cpu(*args)
+        end
+      end
+
+      # col[bsize * out_h * out_w, fil_h * fil_w * ch] to img[bsize, out_h, out_w, ch]
+      def col2im(*args)
+        if DNN.use_cumo?
+          col2im_gpu(*args)
+        else
+          col2im_cpu(*args)
+        end
+      end
+
+      def im2col_cpu(img, out_h, out_w, fil_h, fil_w, strides)
         bsize = img.shape[0]
         ch = img.shape[3]
-        col = Xumo::SFloat.zeros(bsize, out_h, out_w, fil_h, fil_w, ch)
+        col = img.class.zeros(bsize, out_h, out_w, fil_h, fil_w, ch)
         (0...fil_h).each do |i|
           i_range = (i...(i + strides[0] * out_h)).step(strides[0]).to_a
           (0...fil_w).each do |j|
@@ -20,11 +37,16 @@ module DNN
         col.reshape(bsize * out_h * out_w, fil_h * fil_w * ch)
       end
 
-      # col[bsize * out_h * out_w, fil_h * fil_w * ch] to img[bsize, out_h, out_w, ch]
-      def col2im(col, img_shape, out_h, out_w, fil_h, fil_w, strides)
+      def im2col_gpu(img, out_h, out_w, fil_h, fil_w, strides)
+        img = Utils.cumo2numo(img)
+        col = im2col_cpu(img, out_h, out_w, fil_h, fil_w, strides)
+        Utils.numo2cumo(col)
+      end
+
+      def col2im_cpu(col, img_shape, out_h, out_w, fil_h, fil_w, strides)
         bsize, img_h, img_w, ch = img_shape
         col = col.reshape(bsize, out_h, out_w, fil_h, fil_w, ch)
-        img = Xumo::SFloat.zeros(bsize, img_h, img_w, ch)
+        img = col.class.zeros(bsize, img_h, img_w, ch)
         (0...fil_h).each do |i|
           i_range = (i...(i + strides[0] * out_h)).step(strides[0]).to_a
           (0...fil_w).each do |j|
@@ -35,6 +57,12 @@ module DNN
         img
       end
 
+      def col2im_gpu(col, img_shape, out_h, out_w, fil_h, fil_w, strides)
+        col = Utils.cumo2numo(col)
+        img = col2im_cpu(col, img_shape, out_h, out_w, fil_h, fil_w, strides)
+        Utils.numo2cumo(img)
+      end
+
       def zero_padding(img, pad)
         bsize, img_h, img_w, ch = img.shape
         img2 = Xumo::SFloat.zeros(bsize, img_h + pad[0], img_w + pad[1], ch)

data/lib/dnn/core/layers/math_layers.rb

@@ -61,7 +61,7 @@ module DNN
     end
 
     class Add < MergeLayer
-      include MergeLayerNode
+      include LayerNode
 
       def forward_node(x1, x2)
         @x1_shape = x1.shape
@@ -77,7 +77,7 @@ module DNN
     end
 
     class Sub < MergeLayer
-      include MergeLayerNode
+      include LayerNode
 
       def forward_node(x1, x2)
         @x1_shape = x1.shape
@@ -93,7 +93,7 @@ module DNN
     end
 
     class Mul < MergeLayer
-      include MergeLayerNode
+      include LayerNode
 
       def forward_node(x1, x2)
         @x1, @x2 = x1, x2
@@ -108,7 +108,7 @@ module DNN
     end
 
     class Div < MergeLayer
-      include MergeLayerNode
+      include LayerNode
 
       def forward_node(x1, x2)
         @x1, @x2 = x1, x2
@@ -123,7 +123,7 @@ module DNN
     end
 
     class Dot < MergeLayer
-      include MergeLayerNode
+      include LayerNode
 
       def forward_node(x1, x2)
         @x1, @x2 = x1, x2

data/lib/dnn/core/layers/merge_layers.rb

@@ -1,30 +1,6 @@
 module DNN
   module Layers
 
-    module MergeLayerNode
-      def forward(input1, input2)
-        x1 = input1.data
-        x2 = input2.data
-        prev1 = (input1.is_a?(Tensor) ? input1.link : input1)
-        prev2 = (input2.is_a?(Tensor) ? input2.link : input2)
-        y = forward_node(x1, x2)
-        link = TwoInputLink.new(prev1, prev2, self)
-        Tensor.convert(y, link)
-      end
-
-      def backward(dy)
-        backward_node(dy)
-      end
-
-      def forward_node(x1, x2)
-        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'forward_node'"
-      end
-
-      def backward_node(dy)
-        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'backward_node'"
-      end
-    end
-
     class MergeLayer < Layer
       def self.call(x1, x2, *args)
         new(*args).call(x1, x2)
@@ -33,7 +9,7 @@ module DNN
       def call(input1, input2)
         input1 = Tensor.convert(input1) if !input1.is_a?(Tensor) && !input1.is_a?(Param)
         input2 = Tensor.convert(input2) if !input2.is_a?(Tensor) && !input2.is_a?(Param)
-        if input1.data.is_a?(Numo::NArray)
+        if input1.data.is_a?(Xumo::NArray)
           build(input1.data.shape[1..-1]) unless built?
         else
           build([1]) unless built?
@@ -43,7 +19,7 @@ module DNN
     end
 
     class Concatenate < MergeLayer
-      include MergeLayerNode
+      include LayerNode
 
       attr_reader :axis
 

data/lib/dnn/core/layers/split_layers.rb

@@ -0,0 +1,39 @@
+module DNN
+  module Layers
+
+    class Split < Layer
+      include LayerNode
+
+      attr_reader :axis
+      attr_reader :dim
+
+      def initialize(axis: 1, dim: nil)
+        super()
+        raise DNNError, "dim is nil" if dim == nil
+        @axis = axis
+        @dim = dim
+      end
+
+      def forward_node(x)
+        x1_dim = @dim
+        x2_dim = x.shape[@axis] - @dim
+        y1, y2others = x.split([x1_dim, x1_dim + x2_dim], axis: @axis)
+        y2 = y2others.is_a?(Array) ? y2others[0].concatenate(y2others[1..-1], axis: @axis) : y2others
+        [y1, y2]
+      end
+
+      def backward_node(dy1, dy2)
+        dy1.concatenate(dy2, axis: @axis)
+      end
+
+      def to_hash
+        super(axis: @axis, dim: @dim)
+      end
+
+      def load_hash(hash)
+        initialize(axis: hash[:axis], dim: hash[:dim])
+      end
+    end
+
+  end
+end

data/lib/dnn/core/link.rb

@@ -1,57 +1,38 @@
 module DNN
   class Link
-    attr_accessor :prev
+    attr_accessor :prevs
     attr_accessor :next
     attr_accessor :layer_node
+    attr_reader :num_outputs
 
-    def initialize(prev = nil, layer_node = nil)
-      @prev = prev
-      @layer_node = layer_node
-      @next = nil
-    end
-
-    def forward(x)
-      x = @layer_node.(x)
-      @next ? @next.forward(x) : x
-    end
-
-    def backward(dy = Xumo::SFloat[1])
-      dy = @layer_node.backward_node(dy)
-      @prev&.backward(dy)
-    end
-  end
-
-  class TwoInputLink
-    attr_accessor :prev1
-    attr_accessor :prev2
-    attr_accessor :next
-    attr_accessor :layer_node
-
-    def initialize(prev1 = nil, prev2 = nil, layer_node = nil)
-      @prev1 = prev1
-      @prev2 = prev2
+    def initialize(prevs: nil, layer_node: nil, num_outputs: 1)
+      @prevs = prevs
       @layer_node = layer_node
+      @num_outputs = num_outputs
       @next = nil
       @hold = []
     end
 
     def forward(x)
       @hold << x
-      return if @hold.length < 2
+      return if @hold.length < @prevs.length
       x = @layer_node.(*@hold)
       @hold = []
       @next ? @next.forward(x) : x
     end
 
     def backward(dy = Xumo::SFloat[1])
-      dys = @layer_node.backward_node(dy)
+      @hold << dy
+      return if @hold.length < @num_outputs
+      dys = @layer_node.backward_node(*@hold)
+      @hold = []
       if dys.is_a?(Array)
-        dy1, dy2 = *dys
+        dys.each.with_index do |dy, i|
+          @prevs[i]&.backward(dy)
+        end
       else
-        dy1 = dys
+        @prevs.first&.backward(dys)
       end
-      @prev1&.backward(dy1)
-      @prev2&.backward(dy2) if dy2
     end
   end
 end

data/lib/dnn/core/losses.rb

@@ -62,7 +62,7 @@ module DNN
     end
 
     class MeanSquaredError < Loss
-      include Layers::MergeLayerNode
+      include Layers::LayerNode
 
       def forward_node(y, t)
         @y = y
@@ -76,7 +76,7 @@ module DNN
     end
 
     class MeanAbsoluteError < Loss
-      include Layers::MergeLayerNode
+      include Layers::LayerNode
 
       def forward_node(y, t)
         @y = y
@@ -93,7 +93,7 @@ module DNN
     end
 
     class Hinge < Loss
-      include Layers::MergeLayerNode
+      include Layers::LayerNode
 
       def forward_node(y, t)
         @t = t
@@ -109,7 +109,7 @@ module DNN
     end
 
     class HuberLoss < Loss
-      include Layers::MergeLayerNode
+      include Layers::LayerNode
 
       def forward_node(y, t)
         @y = y
@@ -129,7 +129,7 @@ module DNN
     end
 
     class SoftmaxCrossEntropy < Loss
-      include Layers::MergeLayerNode
+      include Layers::LayerNode
 
       attr_accessor :eps
 
@@ -166,7 +166,7 @@ module DNN
     end
 
     class SigmoidCrossEntropy < Loss
-      include Layers::MergeLayerNode
+      include Layers::LayerNode
 
       attr_accessor :eps
 

data/lib/dnn/core/models.rb

@@ -230,6 +230,7 @@ module DNN
             puts "【 epoch #{epoch}/#{epochs} 】" if verbose
 
             train_iterator.foreach(batch_size) do |x_batch, y_batch, index|
+              @last_log[:step] = index
               train_step_met = train_step(x_batch, y_batch)
               num_trained_datas = (index + 1) * batch_size
               num_trained_datas = num_trained_datas > num_train_datas ? num_train_datas : num_trained_datas
@@ -305,13 +306,13 @@ module DNN
             loss_opt[:layers] = layers if i == 0
             loss_opt[:loss_weight] = @loss_weights[i] if @loss_weights
             loss = @loss_func[i].loss(out, Tensor.convert(y[i]), **loss_opt)
-            loss_data << loss.data.to_f
+            loss_data << Utils.to_f(loss.data)
             loss.link.backward(Xumo::SFloat.ones(y[i][0...1, false].shape[0], 1))
           end
         else
           out = output_tensors
           loss = @loss_func.loss(out, Tensor.convert(y), layers: layers)
-          loss_data = loss.data.to_f
+          loss_data = Utils.to_f(loss.data)
           loss.link.backward(Xumo::SFloat.ones(y[0...1, false].shape[0], 1))
         end
         @optimizer.update(get_all_trainable_params)
@@ -392,13 +393,13 @@ module DNN
           output_tensors.each.with_index do |out, i|
             correct << accuracy(out.data, y[i]) if accuracy
             loss = @loss_func[i].(out, Tensor.convert(y[i]))
-            loss_data << loss.data.to_f
+            loss_data << Utils.to_f(loss.data)
           end
         else
           out = output_tensors
           correct = accuracy(out.data, y) if accuracy
           loss = @loss_func.(out, Tensor.convert(y))
-          loss_data = loss.data.to_f
+          loss_data = Utils.to_f(loss.data)
         end
         call_callbacks(:after_test_on_batch)
         [correct, loss_data]
@@ -567,6 +568,56 @@ module DNN
         end
       end
 
+      # Convert the parameters of model and optimizer for cpu.
+      # @return [DNN::Models::Model] Return self.
+      def to_cpu
+        params_data = get_all_params_data
+        clean_layers
+        set_all_params_data(params_data)
+        trainable_layers.each do |layer|
+          layer.get_params.each do |key, param|
+            data = param.data
+            if DNN.use_cumo? && data.is_a?(Cumo::NArray)
+              param.data = Utils.cumo2numo(data)
+            end
+          end
+        end
+        @optimizer.status.each do |key, state|
+          next unless state
+          state.each do |param, data|
+            if DNN.use_cumo? && data.is_a?(Cumo::NArray)
+              state[param] = Utils.cumo2numo(data)
+            end
+          end
+        end
+        self
+      end
+
+      # Convert the parameters of model and optimizer for gpu.
+      # @return [DNN::Models::Model] Return self.
+      def to_gpu
+        params_data = get_all_params_data
+        clean_layers
+        set_all_params_data(params_data)
+        trainable_layers.each do |layer|
+          layer.get_params.each do |(key, param)|
+            data = param.data
+            if DNN.use_cumo? && data.is_a?(Numo::NArray)
+              param.data = Utils.numo2cumo(data)
+            end
+          end
+        end
+        @optimizer.status.each do |(key, state)|
+          next unless state
+          state.each do |(param, data)|
+            if DNN.use_cumo? && data.is_a?(Numo::NArray)
+              state[param] = Utils.numo2cumo(data)
+            end
+          end
+        end
+        self
+      end
+
       private
 
       def get_all_trainable_params
@@ -584,10 +635,10 @@ module DNN
       def metrics_to_str(mertics)
         mertics.map { |key, values|
           str_values = if values.is_a?(Array)
-                         values_fmt = values.map { |v| sprintf('%.4f', v) }
+                         values_fmt = values.map { |v| sprintf('%.4f', Utils.to_f(v)) }
                          "[#{values_fmt.join(", ")}]"
                        else
-                         sprintf('%.4f', values)
+                         sprintf('%.4f', Utils.to_f(values))
                        end
           "#{key}: #{str_values}"
         }.join(", ")
@@ -656,6 +707,7 @@ module DNN
           raise TypeError, "layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Chain class."
         end
         @stack.insert(index, layer)
+        self
       end
 
       # Remove layer to the model.

data/lib/dnn/core/optimizers.rb

@@ -3,6 +3,7 @@ module DNN
 
     # Super class of all optimizer classes.
     class Optimizer
+      attr_reader :status
       attr_accessor :clip_norm
 
       def self.from_hash(hash)
@@ -47,7 +48,7 @@ module DNN
       end
 
       private def clip_grads(params)
-        norm = Math.sqrt(params.reduce(0) { |total, param| total + (param.grad**2).sum })
+        norm = Math.sqrt(params.reduce(0) { |total, param| total + (param.grad**2).sum.to_f })
         return if norm <= @clip_norm
         rate = @clip_norm / (norm + 1e-7)
         params.each do |param|
@@ -71,6 +72,7 @@ module DNN
         @lr = lr
         @momentum = momentum
         @v = {}
+        @status = { v: @v }
       end
 
       def to_hash
@@ -120,6 +122,7 @@ module DNN
         @lr = lr
         @eps = eps
         @g = {}
+        @status = { g: @g }
       end
 
       private def update_params(params)
@@ -153,6 +156,7 @@ module DNN
         @alpha = alpha
         @eps = eps
         @g = {}
+        @status = { g: @g }
       end
 
       def to_hash
@@ -184,6 +188,7 @@ module DNN
         @eps = eps
         @h = {}
         @s = {}
+        @status = { h: @h, s: @s }
       end
 
       def to_hash
@@ -221,6 +226,7 @@ module DNN
         @eps = eps
         @m = {}
         @v = {}
+        @status = { m: @m, v: @v }
       end
 
       def to_hash
@@ -265,6 +271,7 @@ module DNN
         @m = {}
         @v = {}
         @s = amsgrad ? {} : nil
+        @status = { m: @m, v: @v, s: @s }
       end
 
       def to_hash

data/lib/dnn/core/utils.rb

@@ -43,5 +43,28 @@ module DNN
     def self.numerical_grad(x, func)
       (func.(x + 1e-7) - func.(x)) / 1e-7
     end
+
+    # Convert numo to cumo.
+    def self.numo2cumo(na)
+      b = na.to_binary
+      ca = Cumo::SFloat.from_binary(b)
+      ca.reshape(*na.shape)
+    end
+
+    # Convert cumo to numo.
+    def self.cumo2numo(ca)
+      b = ca.to_binary
+      na = Numo::SFloat.from_binary(b)
+      na.reshape(*ca.shape)
+    end
+
+    # Force convert to Float.
+    def self.to_f(x)
+      if x.is_a?(Xumo::NArray)
+        x[0].to_f
+      else
+        x.to_f
+      end
+    end
   end
 end

data/lib/dnn/version.rb

@@ -1,3 +1,3 @@
 module DNN
-  VERSION = "1.1.6"
+  VERSION = "1.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-dnn
 version: !ruby/object:Gem::Version
-  version: 1.1.6
+  version: 1.2.0
 platform: ruby
 authors:
 - unagiootoro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-03 00:00:00.000000000 Z
+date: 2020-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -135,6 +135,7 @@ files:
 - examples/mnist_conv2d_example.rb
 - examples/mnist_define_by_run.rb
 - examples/mnist_example.rb
+- examples/mnist_gpu.rb
 - examples/mnist_lstm_example.rb
 - examples/pix2pix/dcgan.rb
 - examples/pix2pix/imgen.rb
@@ -157,6 +158,7 @@ files:
 - lib/dnn/core/layers/merge_layers.rb
 - lib/dnn/core/layers/normalizations.rb
 - lib/dnn/core/layers/rnn_layers.rb
+- lib/dnn/core/layers/split_layers.rb
 - lib/dnn/core/link.rb
 - lib/dnn/core/losses.rb
 - lib/dnn/core/models.rb