ruby-dnn 0.16.1 → 0.16.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e04eef16303e2e223bfdb1ac17e2a35fc20d0eafff022f71e412d8b41dc49c9
-  data.tar.gz: d0cd83a209069eba6ea2c48fc6c3480e5edf5b99932b31cb9d7a73adb0eb573d
+  metadata.gz: b76b0eb5bf75a22e48726f93fa4faff413c196b1f1587ce145aba7042c84a532
+  data.tar.gz: f32d09cb89391583f51a557e5f72024f618d943af3becc25f8f19905fa395b3b
 SHA512:
-  metadata.gz: 2b7126723c81495c603a1b98fc64fdd111d4853db78e4f0c840f75e925d60d97a3dd6ba22a9a9e2a8f643562a1284bfe502e38e0e9e4870c7699f16ed151df40
-  data.tar.gz: c1c8900e92ec77a03f46853a9ff14d4b09c8442a597f6ae87d1ee049764a3316e1b7ff92cdb4c13c4b2787d2759ff5c8f6c28c11668503d1d0062a2f671a04b7
+  metadata.gz: f70784c49f71420df424c2077b430c0cb837b31b5493f62831cf49b296d834045796f2bae7749fa3b4bfb3ca365a97da856ccbebcc666c1489020eb55ec408fc
+  data.tar.gz: 52fd5cf850341b3d2bdd44da35f69254e508b0dd77e052a7217fe250a8d41a00d237b5ce0cb379e652bf2e6b7d0b99e95b3530ef5abe2d43a2b8f8c8f7dbca7e

data/examples/vae.rb

@@ -0,0 +1,118 @@
+require "dnn"
+require "dnn/datasets/mnist"
+require "dnn/image"
+require "numo/linalg/autoloader"
+
+include DNN::Models
+include DNN::Layers
+include DNN::Optimizers
+include DNN::Losses
+
+x_train, y_train = DNN::MNIST.load_train
+x_test, y_test = DNN::MNIST.load_test
+
+x_train = Numo::SFloat.cast(x_train).reshape(x_train.shape[0], 784)
+x_test = Numo::SFloat.cast(x_test).reshape(x_test.shape[0], 784)
+
+x_train /= 255
+x_test /= 255
+
+y_train = DNN::Utils.to_categorical(y_train, 10, Numo::SFloat)
+y_test = DNN::Utils.to_categorical(y_test, 10, Numo::SFloat)
+
+$z_dim = 2
+$z_mean = nil
+$z_sigma = nil
+
+class Sampling < MergeLayer
+  def forward(z_mean, z_sigma)
+    epsilon = DNN::Tensor.new(Numo::SFloat.new($z_dim).rand_norm(0, 1))
+    Tanh.(z_mean + z_sigma * epsilon)
+  end
+end
+
+class Encoder < Model
+  def initialize
+    super
+    @l1 = Dense.new(196)
+    @l2 = Dense.new(49)
+    @l3_1 = Dense.new($z_dim)
+    @l3_2 = Dense.new($z_dim)
+    @bn1 = BatchNormalization.new
+    @bn2 = BatchNormalization.new
+  end
+
+  def forward(x)
+    x = InputLayer.new(784).(x)
+    x = @l1.(x)
+    x = @bn1.(x)
+    x = ReLU.(x)
+    x = @l2.(x)
+    x = @bn2.(x)
+    x = ReLU.(x)
+    z_mean = @l3_1.(x)
+    z_sigma = @l3_2.(x)
+    [z_mean, z_sigma]
+  end
+end
+
+class Decoder < Model
+  def initialize
+    super
+    @l3 = Dense.new(196)
+    @l4 = Dense.new(784)
+    @bn1 = BatchNormalization.new
+  end
+
+  def forward(z)
+    x = @l3.(z)
+    x = @bn1.(x)
+    x = ReLU.(x)
+    x = @l4.(x)
+    x
+  end
+end
+
+class VAE < Model
+  attr_accessor :enc
+  attr_accessor :dec
+
+  def initialize(enc = nil, dec = nil)
+    super()
+    @enc = enc || Encoder.new
+    @dec = dec || Decoder.new
+  end
+
+  def forward(x)
+    z_mean, z_sigma = @enc.(x)
+    $z_mean, $z_sigma = z_mean, z_sigma
+    z = Sampling.(z_mean, z_sigma)
+    x = @dec.(z)
+    x
+  end
+end
+
+class VAELoss < Loss
+  def forward(y, t)
+    kl = -0.5 * Mean.(Sum.(1 + Log.($z_sigma**2) - $z_mean**2 - $z_sigma**2, axis: 1), axis: 0)
+    SigmoidCrossEntropy.(y, t) + kl
+  end
+end
+
+model = VAE.new
+dec = model.dec
+model.setup(Adam.new, VAELoss.new)
+
+model.train(x_train, x_train, 10, batch_size: 100)
+
+images = []
+10.times do |i|
+  10.times do |j|
+    z1 = (i / 4.5) - 1
+    z2 = (j / 4.5) - 1
+    z = Numo::SFloat[z1, z2]
+    out = DNN::Utils.sigmoid(dec.predict1(z))
+    img = Numo::UInt8.cast(out * 255).reshape(28, 28, 1)
+    DNN::Image.write("img/img_#{i}_#{j}.png", img)
+  end
+end

data/lib/dnn/core/layers/math_layers.rb

@@ -2,6 +2,8 @@ module DNN
   module Layers
 
     class Add < MergeLayer
+      include MergeLayerNode
+
       def forward_node(x1, x2)
         x1 + x2
       end
@@ -12,6 +14,8 @@ module DNN
     end
 
     class Sub < MergeLayer
+      include MergeLayerNode
+
       def forward_node(x1, x2)
         x1 - x2
       end
@@ -22,6 +26,8 @@ module DNN
     end
 
     class Mul < MergeLayer
+      include MergeLayerNode
+
       def forward_node(x1, x2)
         @x1, @x2 = x1, x2
         x1 * x2
@@ -33,6 +39,8 @@ module DNN
     end
 
     class Div < MergeLayer
+      include MergeLayerNode
+
       def forward_node(x1, x2)
         @x1, @x2 = x1, x2
         x1 / x2
@@ -46,6 +54,8 @@ module DNN
     end
 
     class Dot < MergeLayer
+      include MergeLayerNode
+
       def forward_node(x1, x2)
         @x1, @x2 = x1, x2
         x1.dot(x2)
@@ -60,12 +70,11 @@ module DNN
       include LayerNode
 
       def forward_node(x)
-        @x = x
-        Xumo::NMath.exp(x)
+        @y = Xumo::NMath.exp(x)
       end
 
       def backward_node(dy)
-        dy * Xumo::NMath.exp(@x)
+        dy * @y
       end
     end
 
@@ -122,20 +131,16 @@ module DNN
       end
 
       def forward_node(x)
-        if @axis
-          @dim = x.shape[@axis]
-          x.sum(axis: @axis, keepdims: true)
-        else
-          x.sum
-        end
+        @x_shape = x.shape
+        @dim = x.shape[@axis]
+        x.sum(axis: @axis, keepdims: true)
       end
 
       def backward_node(dy)
-        dx = dy.clone
-        if @axis
-          (@dim - 1).times do
-            dx = dx.concatenate(dy, axis: @axis)
-          end
+        return dy if @x_shape == dy.shape
+        dx = dy
+        (@dim - 1).times do
+          dx = dx.concatenate(dy, axis: @axis)
         end
         dx
       end
@@ -150,16 +155,16 @@ module DNN
       end
 
       def forward_node(x)
-        @dim = @axis ? x.shape[@axis] : x.size
+        @x_shape = x.shape
+        @dim = x.shape[@axis]
         x.mean(axis: @axis, keepdims: true)
       end
 
       def backward_node(dy)
+        return dy / @dim if @x_shape == dy.shape
         dx = dy
-        if @axis
-          (@dim - 1).times do
-            dx = dx.concatenate(dy, axis: @axis)
-          end
+        (@dim - 1).times do
+          dx = dx.concatenate(dy, axis: @axis)
         end
         dx / @dim
       end

data/lib/dnn/core/layers/merge_layers.rb

@@ -26,8 +26,6 @@ module DNN
     end
 
     class MergeLayer < Layers::Layer
-      include MergeLayerNode
-
       def self.call(x1, x2, *args)
         new(*args).call(x1, x2)
       end
@@ -45,6 +43,8 @@ module DNN
     end
 
     class Concatenate < MergeLayer
+      include MergeLayerNode
+
       attr_reader :axis
 
       def initialize(axis: 1)

data/lib/dnn/core/losses.rb

@@ -75,8 +75,8 @@ module DNN
         0.5 * ((y - t)**2).mean(0).sum
       end
 
-      def backward_node(dy)
-        @y - @t
+      def backward_node(d)
+        (@y - @t) / @y.shape[0]
       end
     end
 
@@ -90,10 +90,10 @@ module DNN
       end
 
       def backward_node(d)
-        dy = @y - @t
+        dy = (@y - @t)
         dy[dy >= 0] = 1
         dy[dy < 0] = -1
-        dy
+        dy / @y.shape[0]
       end
     end
 
@@ -109,7 +109,7 @@ module DNN
       def backward_node(d)
         a = Xumo::SFloat.ones(*@a.shape)
         a[@a <= 0] = 0
-        a * -@t
+        (a * -@t) / a.shape[0]
       end
     end
 
@@ -124,12 +124,12 @@ module DNN
       end
 
       def backward_node(d)
-        dy = @y - @t
+        dy = (@y - @t)
         if @loss_value > 1
           dy[dy >= 0] = 1
           dy[dy < 0] = -1
         end
-        dy
+        dy / @y.shape[0]
       end
 
       private
@@ -168,7 +168,7 @@ module DNN
       end
 
       def backward_node(d)
-        @x - @t
+        (@x - @t) / @x.shape[0]
       end
 
       def to_hash
@@ -205,7 +205,7 @@ module DNN
       end
 
       def backward_node(d)
-        @x - @t
+        (@x - @t) / @x.shape[0]
       end
 
       def to_hash

data/lib/dnn/core/models.rb

@@ -262,7 +262,7 @@ module DNN
         DNN.learning_phase = true
         out = call(Tensor.convert(x))
         loss = @loss_func.loss(out, Tensor.convert(y), layers)
-        loss.link.backward(Xumo::SFloat.zeros(y[0...1, false].shape))
+        loss.link.backward(Xumo::SFloat.ones(y[0...1, false].shape[0], 1))
         @optimizer.update(get_all_trainable_params)
         @last_log[:train_loss] = loss.data
         call_callbacks(:after_train_on_batch)
@@ -353,7 +353,12 @@ module DNN
       # @param [Numo::SFloat] x Input data. However, x is single data.
       def predict1(x, use_loss_activation: true)
         check_xy_type(x)
-        predict(x.reshape(1, *x.shape), use_loss_activation: use_loss_activation)[0, false]
+        input = if x.is_a?(Array)
+                  x.map { |v| v.reshape(1, *v.shape) }
+                else
+                  x.reshape(1, *x.shape)
+                end
+        predict(input, use_loss_activation: use_loss_activation)[0, false]
       end
 
       # Add callback function.
@@ -458,7 +463,7 @@ module DNN
         if !x.is_a?(Xumo::SFloat) && !x.is_a?(Array)
           raise TypeError, "x:#{x.class.name} is not an instance of #{Xumo::SFloat.name} class or Array class."
         end
-        if y && !y.is_a?(Xumo::SFloat) && !x.is_a?(Array)
+        if y && !y.is_a?(Xumo::SFloat) && !y.is_a?(Array)
           raise TypeError, "y:#{y.class.name} is not an instance of #{Xumo::SFloat.name} class or Array class."
         end
       end

data/lib/dnn/version.rb

@@ -1,3 +1,3 @@
 module DNN
-  VERSION = "0.16.1"
+  VERSION = "0.16.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-dnn
 version: !ruby/object:Gem::Version
-  version: 0.16.1
+  version: 0.16.2
 platform: ruby
 authors:
 - unagiootoro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-06 00:00:00.000000000 Z
+date: 2020-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -114,6 +114,7 @@ files:
 - examples/pix2pix/dcgan.rb
 - examples/pix2pix/imgen.rb
 - examples/pix2pix/train.rb
+- examples/vae.rb
 - examples/xor_example.rb
 - ext/rb_stb_image/extconf.rb
 - ext/rb_stb_image/rb_stb_image.c