ruby-dnn 0.15.3 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b213164ca3e4a7d781a673c4c49ec8dfbd6713468687a98c3739c0cf5629de73
-  data.tar.gz: 92a8133dc0ab085d387199f92c2a602bb3782b810c20a931f7ee8de6e247f2a4
+  metadata.gz: ea912bca075445de925fda876e2b003bf0f63936100a3c14adf58bf810a6a9af
+  data.tar.gz: 41a40ee396cbda27faa7719cfcf2bed5e1337dc2963c1315561e186e05b81d77
 SHA512:
-  metadata.gz: ea6652994f71d97142fe7357947701eff5f0c26708698a670a148e995fb10816b75b62b0bd60004d9ea99b6c837e5b599e790982ad12d010394e4ec7d6962135
-  data.tar.gz: 9d474a7bdd120f0e39fcb05e11f0f8370100dd21e79440fe0408bbccf676890726623f516ada78ae6b8829bcf0799ceeea78218f64d74a70ca7f5dd9286bdae3
+  metadata.gz: 16a12d59eb61b73f76b1361ddf2e5d3e2ca8929d5195f8f8d89299f829fa4a32284d2138ca6b8c4b8c5de5249c18e4a273390e9e80298e52d0e9a8109f4533a8
+  data.tar.gz: 859b180d139bff4f8904c939a41482574b1dd2c972fc83cf42743e078c3e04c28a1f8d740ba20dc970923bcb9411accefa86442fb7b77661de9ae015db1fc6b0

data/Rakefile

@@ -8,23 +8,15 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/*_test.rb", "test/layers_test/*_test.rb"]
 end
 
-task :build_cifar_loader do
-  sh "cd ext/cifar_loader; ruby extconf.rb; make"
-end
-
 task :build_rb_stb_image do
   sh "cd ext/rb_stb_image; ruby extconf.rb; make"
 end
 
-task :clean_cifar_loader do
-  sh "cd ext/cifar_loader; make clean; unlink Makefile"
-end
-
 task :clean_rb_stb_image do
   sh "cd ext/rb_stb_image; make clean; unlink Makefile"
 end
 
-task :default => [:test, :build_cifar_loader, :build_rb_stb_image]
+task :default => [:test, :build_rb_stb_image]
 
 task :doc do
   src_list = Dir["lib/dnn.rb"]

data/examples/api-examples/early_stopping_example.rb

@@ -34,7 +34,7 @@ class MLP < Model
     @bn2 = BatchNormalization.new
   end
 
-  def call(x)
+  def forward(x)
     x = InputLayer.(x)
     x = @l1.(x)
     x = @bn1.(x)

data/examples/api-examples/initializer_example.rb

@@ -35,7 +35,7 @@ class MLP < Model
     @bn2 = BatchNormalization.new
   end
 
-  def call(x)
+  def forward(x)
     x = InputLayer.(x)
     x = @l1.(x)
     x = @bn1.(x)

data/examples/api-examples/regularizer_example.rb

@@ -36,7 +36,7 @@ class MLP < Model
     @bn2 = BatchNormalization.new
   end
 
-  def call(x)
+  def forward(x)
     x = InputLayer.(x)
     x = @l1.(x)
     x = @bn1.(x)

data/examples/api-examples/save_example.rb

@@ -45,7 +45,7 @@ class MLP < Model
     @bn2 = BatchNormalization.new
   end
 
-  def call(x)
+  def forward(x)
     x = InputLayer.new(784).(x)
     x = @l1.(x)
     x = @bn1.(x)

data/examples/dcgan/dcgan.rb

@@ -19,7 +19,7 @@ class Generator < Model
     @bn6 = BatchNormalization.new
   end
 
-  def call(x)
+  def forward(x)
     x = InputLayer.new(20).(x)
     x = @l1.(x)
     x = @bn1.(x)
@@ -63,7 +63,7 @@ class Discriminator < Model
     @l6 = Dense.new(1)
   end
 
-  def call(x, trainable = true)
+  def forward(x, trainable = true)
     @l1.trainable = trainable
     @l2.trainable = trainable
     @l3.trainable = trainable
@@ -103,7 +103,7 @@ class DCGAN < Model
     @dis = dis
   end
 
-  def call(x)
+  def forward(x)
     x = @gen.(x)
     x = @dis.(x, false)
     x

data/examples/iris_example.rb

@@ -3,7 +3,6 @@ require "dnn/datasets/iris"
 # If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
-include DNN::Models
 include DNN::Layers
 include DNN::Optimizers
 include DNN::Losses
@@ -15,19 +14,44 @@ x_test, y_test = x[100...150, true], y[100...150]
 y_train = DNN::Utils.to_categorical(y_train, 3, Numo::SFloat)
 y_test = DNN::Utils.to_categorical(y_test, 3, Numo::SFloat)
 
-model = Sequential.new
-
-model << InputLayer.new(4)
-
-model << Dense.new(64)
-model << ReLU.new
-
-model << Dense.new(3)
-
-model.setup(Adam.new, SoftmaxCrossEntropy.new)
-
-model.train(x_train, y_train, 500, batch_size: 32, test: [x_test, y_test])
-
-accuracy, loss = model.evaluate(x_test, y_test)
-puts "accuracy: #{accuracy}"
-puts "loss: #{loss}"
+epochs = 1000
+batch_size = 32
+
+opt = Adam.new
+lf = SoftmaxCrossEntropy.new
+
+train_iter = DNN::Iterator.new(x_train, y_train)
+test_iter = DNN::Iterator.new(x_test, y_test, random: false)
+
+w1 = DNN::Param.new(Numo::SFloat.new(4, 16).rand_norm)
+b1 = DNN::Param.new(Numo::SFloat.zeros(16))
+w2 = DNN::Param.new(Numo::SFloat.new(16, 3).rand_norm)
+b2 = DNN::Param.new(Numo::SFloat.zeros(3))
+
+net = -> x, y do
+  h = Dot.(x, w1) + b1
+  h = Sigmoid.(h)
+  out = Dot.(h, w2) + b2
+  out
+end
+
+(1..epochs).each do |epoch|
+  train_iter.foreach(batch_size) do |x_batch, y_batch, step|
+    x = DNN::Tensor.convert(x_batch)
+    y = DNN::Tensor.convert(y_batch)
+    out = net.(x, y)
+    loss = lf.(out, y)
+    loss.link.backward(nil)
+    puts "epoch: #{epoch}, step: #{step}, loss = #{loss.data}"
+    opt.update([w1, b1, w2, b2])
+  end
+end
+
+correct = 0
+test_iter.foreach(batch_size) do |x_batch, y_batch, step|
+  x = DNN::Tensor.convert(x_batch)
+  y = DNN::Tensor.convert(y_batch)
+  out = net.(x, y)
+  correct += out.data.max_index(axis: 1).eq(y_batch.max_index(axis: 1)).count
+end
+puts "correct = #{correct}"

data/examples/mnist_define_by_run.rb

@@ -28,7 +28,7 @@ class MLP < Model
     @l3 = Dense.new(10)
   end
 
-  def call(x)
+  def forward(x)
     x = InputLayer.new(784).(x)
     x = @l1.(x)
     x = ReLU.(x)

data/examples/pix2pix/dcgan.rb

@@ -0,0 +1,157 @@
+include DNN::Models
+include DNN::Layers
+
+class Generator < Model
+  def initialize(input_shape)
+    super()
+    @input_shape = input_shape
+    @l1 = Conv2D.new(32, 4, padding: true)
+    @l2 = Conv2D.new(32, 4, strides: 2, padding: true)
+    @l3 = Conv2D.new(64, 4, padding: true)
+    @l4 = Conv2D.new(64, 4, strides: 2, padding: true)
+    @l5 = Conv2D.new(128, 4, padding: true)
+    @l6 = Conv2DTranspose.new(64, 4, strides: 2, padding: true)
+    @l7 = Conv2D.new(64, 4, padding: true)
+    @l8 = Conv2DTranspose.new(32, 4, strides: 2, padding: true)
+    @l9 = Conv2D.new(32, 4, padding: true)
+    @l10 = Conv2D.new(32, 4, padding: true)
+    @l11 = Conv2D.new(3, 4, padding: true)
+    @bn1 = BatchNormalization.new
+    @bn2 = BatchNormalization.new
+    @bn3 = BatchNormalization.new
+    @bn4 = BatchNormalization.new
+    @bn5 = BatchNormalization.new
+    @bn6 = BatchNormalization.new
+    @bn7 = BatchNormalization.new
+    @bn8 = BatchNormalization.new
+    @bn9 = BatchNormalization.new
+  end
+
+  def forward(x)
+    input = InputLayer.new(@input_shape).(x)
+    x = @l1.(input)
+    x = @bn1.(x)
+    h1 = ReLU.(x)
+
+    x = @l2.(h1)
+    x = @bn2.(x)
+    x = ReLU.(x)
+
+    x = @l3.(x)
+    x = @bn3.(x)
+    h2 = ReLU.(x)
+
+    x = @l4.(x)
+    x = @bn4.(x)
+    x = ReLU.(x)
+
+    x = @l5.(x)
+    x = @bn5.(x)
+    x = ReLU.(x)
+
+    x = @l6.(x)
+    x = @bn6.(x)
+    x = ReLU.(x)
+
+    x = @l7.(x)
+    x = @bn7.(x)
+    x = ReLU.(x)
+    x = Concatenate.(x, h2, axis: 3)
+
+    x = @l8.(x)
+    x = @bn8.(x)
+    x = ReLU.(x)
+
+    x = @l9.(x)
+    x = @bn9.(x)
+    x = ReLU.(x)
+    x = Concatenate.(x, h1, axis: 3)
+
+    x = @l10.(x)
+    x = ReLU.(x)
+
+    x = @l11.(x)
+    x = Tanh.(x)
+    x
+  end
+end
+
+class Discriminator < Model
+  def initialize(gen_input_shape, gen_output_shape)
+    super()
+    @gen_input_shape = gen_input_shape
+    @gen_output_shape = gen_output_shape
+    @l1_1 = Conv2D.new(32, 4, padding: true)
+    @l1_2 = Conv2D.new(32, 4, padding: true)
+    @l2 = Conv2D.new(32, 4, strides: 2, padding: true)
+    @l3 = Conv2D.new(32, 4, padding: true)
+    @l4 = Conv2D.new(64, 4, strides: 2, padding: true)
+    @l5 = Conv2D.new(64, 4, padding: true)
+    @l6 = Dense.new(1024)
+    @l7 = Dense.new(1)
+    @bn1 = BatchNormalization.new
+    @bn2 = BatchNormalization.new
+    @bn3 = BatchNormalization.new
+    @bn4 = BatchNormalization.new
+    @bn5 = BatchNormalization.new
+    @bn6 = BatchNormalization.new
+  end
+
+  def forward(inputs, trainable = true)
+    trainable_layers.each do |layer|
+      layer.trainable = trainable
+    end
+
+    input, images = *inputs
+    x = InputLayer.new(@gen_input_shape).(input)
+    x = @l1_1.(x)
+    x = @bn1.(x)
+    x1 = LeakyReLU.(x, 0.2)
+
+    x = InputLayer.new(@gen_output_shape).(images)
+    x = @l1_2.(x)
+    x = @bn2.(x)
+    x2 = LeakyReLU.(x, 0.2)
+
+    x = Concatenate.(x1, x2)
+    x = @l2.(x)
+    x = @bn3.(x)
+    x = LeakyReLU.(x, 0.2)
+
+    x = @l3.(x)
+    x = @bn4.(x)
+    x = LeakyReLU.(x, 0.2)
+
+    x = @l4.(x)
+    x = @bn5.(x)
+    x = LeakyReLU.(x, 0.2)
+
+    x = @l5.(x)
+    x = @bn6.(x)
+    x = LeakyReLU.(x, 0.2)
+
+    x = Flatten.(x)
+    x = @l6.(x)
+    x = LeakyReLU.(x, 0.2)
+
+    x = @l7.(x)
+    x
+  end
+end
+
+class DCGAN < Model
+  attr_reader :gen
+  attr_reader :dis
+
+  def initialize(gen, dis)
+    super()
+    @gen = gen
+    @dis = dis
+  end
+
+  def forward(input)
+    x = @gen.(input)
+    x = @dis.([input, x], false)
+    x
+  end
+end

data/examples/pix2pix/imgen.rb

@@ -0,0 +1,27 @@
+require "dnn"
+require "dnn/image"
+require "dnn/datasets/cifar10"
+require "numo/linalg/autoloader"
+require_relative "dcgan"
+
+def load_dataset
+  x, y = DNN::CIFAR10.load_test
+  x_out = Numo::SFloat.cast(x)
+  x_in = x_out.mean(axis: 3, keepdims: true)
+  x_in = (x_in / 127.5) - 1
+  x_out = (x_out / 127.5) - 1
+  [x_in, x_out]
+end
+
+batch_size = 100
+
+dcgan = DCGAN.load("trained/dcgan_model_epoch20.marshal")
+gen = dcgan.gen
+
+x_in, x_out = load_dataset
+images = gen.predict(x_in[0...batch_size, false])
+
+batch_size.times do |i|
+  img = Numo::UInt8.cast(((images[i, false] + 1) * 127.5).round)
+  DNN::Image.write("img/img_#{i}.jpg", img)
+end

data/examples/pix2pix/train.rb

@@ -0,0 +1,52 @@
+# This example converts a CIFAR10 grayscale image to a color image.
+
+require "dnn"
+require "dnn/datasets/cifar10"
+require "numo/linalg/autoloader"
+require_relative "dcgan"
+
+include DNN::Optimizers
+include DNN::Losses
+
+def load_dataset
+  x, y = DNN::CIFAR10.load_train
+  x_out = Numo::SFloat.cast(x)
+  x_in = x_out.mean(axis: 3, keepdims: true)
+  x_in = (x_in / 127.5) - 1
+  x_out = (x_out / 127.5) - 1
+  [x_in, x_out]
+end
+
+epochs = 20
+batch_size = 128
+
+gen = Generator.new([32, 32, 1])
+dis = Discriminator.new([32, 32, 1], [32, 32, 3])
+dcgan = DCGAN.new(gen, dis)
+
+dis.setup(Adam.new(alpha: 0.00001, beta1: 0.1), SigmoidCrossEntropy.new)
+dcgan.setup(Adam.new(alpha: 0.0002, beta1: 0.5), SigmoidCrossEntropy.new)
+
+x_in, x_out = load_dataset
+
+iter1 = DNN::Iterator.new(x_in, x_out)
+iter2 = DNN::Iterator.new(x_in, x_out)
+num_batchs = x_in.shape[0] / batch_size
+(1..epochs).each do |epoch|
+  num_batchs.times do |index|
+    x_in, x_out = iter1.next_batch(batch_size)
+    images = gen.predict(x_in)
+    y_real = Numo::SFloat.ones(batch_size, 1)
+    y_fake = Numo::SFloat.zeros(batch_size, 1)
+    dis_loss = dis.train_on_batch([x_in, x_out], y_real)
+    dis_loss += dis.train_on_batch([x_in, images], y_fake)
+
+    x_in, x_out = iter2.next_batch(batch_size)
+    dcgan_loss = dcgan.train_on_batch(x_in, y_real)
+
+    puts "epoch: #{epoch}, index: #{index}, dis_loss: #{dis_loss}, dcgan_loss: #{dcgan_loss}"
+  end
+  iter1.reset
+  iter2.reset
+  dcgan.save("trained/dcgan_model_epoch#{epoch}.marshal")
+end

data/lib/dnn.rb

@@ -8,6 +8,7 @@ module DNN
 end
 
 require_relative "dnn/version"
+require_relative "dnn/core/monkey_patch"
 require_relative "dnn/core/error"
 require_relative "dnn/core/global"
 require_relative "dnn/core/tensor"
@@ -22,6 +23,7 @@ require_relative "dnn/core/layers/merge_layers"
 require_relative "dnn/core/layers/cnn_layers"
 require_relative "dnn/core/layers/embedding"
 require_relative "dnn/core/layers/rnn_layers"
+require_relative "dnn/core/layers/math_layers"
 require_relative "dnn/core/optimizers"
 require_relative "dnn/core/losses"
 require_relative "dnn/core/initializers"

data/lib/dnn/core/layers/activations.rb

@@ -2,70 +2,84 @@ module DNN
   module Layers
 
     class Sigmoid < Layer
-      def forward(x)
+      include LayerNode
+
+      def forward_node(x)
         @y = 1 / (1 + Xumo::NMath.exp(-x))
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dy * (1 - @y) * @y
       end
     end
 
     class Tanh < Layer
-      def forward(x)
+      include LayerNode
+
+      def forward_node(x)
         @y = Xumo::NMath.tanh(x)
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dy * (1 - @y**2)
       end
     end
 
     class Softsign < Layer
-      def forward(x)
+      include LayerNode
+
+      def forward_node(x)
         @x = x
         x / (1 + x.abs)
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dy * (1 / (1 + @x.abs)**2)
       end
     end
 
     class Softplus < Layer
-      def forward(x)
+      include LayerNode
+
+      def forward_node(x)
         @x = x
         Xumo::NMath.log(1 + Xumo::NMath.exp(x))
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dy * (1 / (1 + Xumo::NMath.exp(-@x)))
       end
     end
 
     class Swish < Layer
-      def forward(x)
+      include LayerNode
+
+      def forward_node(x)
         @x = x
         @y = x * (1 / (1 + Xumo::NMath.exp(-x)))
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dy * (@y + (1 / (1 + Xumo::NMath.exp(-@x))) * (1 - @y))
       end
     end
 
     class ReLU < Layer
-      def forward(x)
+      include LayerNode
+
+      def forward_node(x)
         @x = x
         Xumo::SFloat.maximum(0, x)
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dy * Xumo::SFloat.cast(@x > 0)
       end
     end
 
     class LeakyReLU < Layer
+      include LayerNode
+
       attr_reader :alpha
 
       # @param [Float] alpha The slope when the output value is negative.
@@ -74,14 +88,14 @@ module DNN
         @alpha = alpha
       end
 
-      def forward(x)
+      def forward_node(x)
         @x = x
         a = Xumo::SFloat.ones(x.shape)
         a[x <= 0] = @alpha
         x * a
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dx = Xumo::SFloat.ones(@x.shape)
         dx[@x <= 0] = @alpha
         dy * dx
@@ -97,6 +111,8 @@ module DNN
     end
 
     class ELU < Layer
+      include LayerNode
+
       attr_reader :alpha
 
       # @param [Float] alpha The slope when the output value is negative.
@@ -105,7 +121,7 @@ module DNN
         @alpha = alpha
       end
 
-      def forward(x)
+      def forward_node(x)
         @x = x
         x1 = Xumo::SFloat.zeros(x.shape)
         x1[x >= 0] = 1
@@ -116,7 +132,7 @@ module DNN
         x1 + x2
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dx = Xumo::SFloat.ones(@x.shape)
         dx[@x < 0] = 0
         dx2 = Xumo::SFloat.zeros(@x.shape)
@@ -135,12 +151,14 @@ module DNN
     end
 
     class Mish < Layer
-      def forward(x)
+      include LayerNode
+
+      def forward_node(x)
         @x = x
-        x * Xumo::NMath.tanh(Softplus.new.forward(x))
+        x * Xumo::NMath.tanh(Softplus.new.forward_node(x))
       end
 
-      def backward(dy)
+      def backward_node(dy)
         omega = 4 * (@x + 1) + 4 * Xumo::NMath.exp(2 * @x) + Xumo::NMath.exp(3 * @x) + Xumo::NMath.exp(@x) * (4 * @x + 6)
         delta = 2 * Xumo::NMath.exp(@x) + Xumo::NMath.exp(2 * @x) + 2
         dy * (Xumo::NMath.exp(@x) * omega) / delta**2