ruby-dnn 0.10.4 → 0.12.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 589418cabcfddc5f0066011a7094a9740903d891c0a53248c2dec2ae6ba050f9
-  data.tar.gz: af86c3663a8e855a8ccf2ffed6825037ebe452c0e83810d0595282959715516f
+  metadata.gz: 2835cd619265328ffa05c41515915aaeb54924ca4ddd4958edba992ea4feda89
+  data.tar.gz: 5ab01dbcf1219cdfac7b3a0e20bc5783fa15baaf82225ea1a9a2d076569cb7f1
 SHA512:
-  metadata.gz: d842fe4bbbd6801726e97177ab1212b6eab9d09bcb553a5bf35e082a9685ffea362fcf729ea41a4f438790bc5d2ddd0c962ad629af6c2afe45d95049989ddd37
-  data.tar.gz: fa13a086a3232e6117d5976a467a1a0bb3640f2b8cd7c6db1b2ef5a6c17d42a3d47ca9ce493f6278c331beda12380b4344d502b3cebef0b6978891c383d19167
+  metadata.gz: 1335b77cf701501930e32f07a9faa027706b908d79a24b1b369228b45c5ef15a38ac476c6951d14b7ce398b1eb7789bf0c96566427b84fffb1ed892af7e43629
+  data.tar.gz: baf8ceeec91699a75a14bde79e815561189e26c9317e34f09580f90f45562120ba12540a7a64e42e562d4d26162490b6c2170721459b8e635731e900812a357b

data/.travis.yml

@@ -1,7 +1,6 @@
 sudo: false
 language: ruby
 rvm:
-  - 2.5.1
-before_install: gem install bundler -v 1.16.2
+  - 2.6.1
 script:
   - rake test

data/README.md

@@ -26,7 +26,7 @@ Or install it yourself as:
 ### MNIST MLP example
 
 ```ruby
-model = Model.new
+model = Sequential.new
 
 model << InputLayer.new(784)
 
@@ -38,10 +38,38 @@ model << ReLU.new
 
 model << Dense.new(10)
 
-model.compile(RMSProp.new, SoftmaxCrossEntropy.new)
+model.setup(RMSProp.new, SoftmaxCrossEntropy.new)
 
 model.train(x_train, y_train, 10, batch_size: 100, test: [x_test, y_test])
+```
+
+When create a model with 'define by run' style:  
 
+```ruby
+class MLP < Model
+  def initialize
+    super
+    @l1 = Dense.new(256)
+    @l2 = Dense.new(256)
+    @l3 = Dense.new(10)
+  end
+
+  def call(x)
+    x = InputLayer.(x)
+    x = @l1.(x)
+    x = ReLU.(x)
+    x = @l2.(x)
+    x = ReLU.(x)
+    x = @l3.(x)
+    x
+  end
+end
+
+model = MLP.new
+
+model.setup(Adam.new, SoftmaxCrossEntropy.new)
+
+model.train(x_train, y_train, 10, batch_size: 100, test: [x_test, y_test])
 ```
 
 Please refer to examples for basic usage.  
@@ -50,14 +78,13 @@ If you want to know more detailed information, please refer to the source code.
 ## Implemented
 || Implemented classes |
 |:-----------|------------:|
-| Connections | Dense, Conv2D, Conv2D_Transpose, Embedding, SimpleRNN, LSTM, GRU |
+| Connections | Dense, Conv2D, Conv2DTranspose, Embedding, SimpleRNN, LSTM, GRU |
 | Layers | Flatten, Reshape, Dropout, BatchNormalization, MaxPool2D, AvgPool2D, UnPool2D |
 | Activations | Sigmoid, Tanh, Softsign, Softplus, Swish, ReLU, LeakyReLU, ELU |
-| Optimizers | SGD, Nesterov, AdaGrad, RMSProp, AdaDelta, Adam, RMSPropGraves |
-| Losses | MeanSquaredError, MeanAbsoluteError, HuberLoss, SoftmaxCrossEntropy, SigmoidCrossEntropy |
+| Optimizers | SGD, Nesterov, AdaGrad, RMSProp, AdaDelta, RMSPropGraves, Adam, AdaBound |
+| Losses | MeanSquaredError, MeanAbsoluteError, Hinge, HuberLoss, SoftmaxCrossEntropy, SigmoidCrossEntropy |
 
 ## TODO
-● Support to define by run model.  
 ● Write a test.  
 ● Write a document.  
 ● Support to GPU.  

data/examples/cifar100_example.rb

@@ -7,7 +7,7 @@ include DNN::Layers
 include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-Model = DNN::Model
+include DNN::Models
 CIFAR100 = DNN::CIFAR100
 
 x_train, y_train = CIFAR100.load_train
@@ -25,7 +25,7 @@ y_test = y_test[true, 1]
 y_train = DNN::Utils.to_categorical(y_train, 100, Numo::SFloat)
 y_test = DNN::Utils.to_categorical(y_test, 100, Numo::SFloat)
 
-model = Model.new
+model = Sequential.new
 
 model << InputLayer.new([32, 32, 3])
 
@@ -66,6 +66,6 @@ model << Dropout.new(0.5)
 
 model << Dense.new(100)
 
-model.compile(Adam.new, SoftmaxCrossEntropy.new)
+model.setup(Adam.new, SoftmaxCrossEntropy.new)
 
 model.train(x_train, y_train, 10, batch_size: 100, test: [x_test, y_test])

data/examples/cifar10_example.rb

@@ -7,7 +7,7 @@ include DNN::Layers
 include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-Model = DNN::Model
+include DNN::Models
 CIFAR10 = DNN::CIFAR10
 
 x_train, y_train = CIFAR10.load_train
@@ -22,7 +22,7 @@ x_test /= 255
 y_train = DNN::Utils.to_categorical(y_train, 10, Numo::SFloat)
 y_test = DNN::Utils.to_categorical(y_test, 10, Numo::SFloat)
 
-model = Model.new
+model = Sequential.new
 
 model << InputLayer.new([32, 32, 3])
 
@@ -63,6 +63,6 @@ model << Dropout.new(0.5)
 
 model << Dense.new(10)
 
-model.compile(Adam.new, SoftmaxCrossEntropy.new)
+model.setup(Adam.new, SoftmaxCrossEntropy.new)
 
 model.train(x_train, y_train, 10, batch_size: 100, test: [x_test, y_test])

data/examples/dcgan/dcgan.rb

@@ -0,0 +1,112 @@
+include DNN::Layers
+include DNN::Activations
+include DNN::Optimizers
+include DNN::Losses
+include DNN::Models
+
+class Generator < Model
+  def initialize
+    super
+    @l1 = Dense.new(1024)
+    @l2 = Dense.new(7 * 7 * 64)
+    @l3 = Conv2DTranspose.new(64, 4, strides: 2, padding: true)
+    @l4 = Conv2D.new(64, 4, padding: true)
+    @l5 = Conv2DTranspose.new(32, 4, strides: 2, padding: true)
+    @l6 = Conv2D.new(32, 4, padding: true)
+    @l7 = Conv2D.new(1, 4, padding: true)
+    @bn1 = BatchNormalization.new
+    @bn2 = BatchNormalization.new
+    @bn3 = BatchNormalization.new
+    @bn4 = BatchNormalization.new
+    @bn5 = BatchNormalization.new
+    @bn6 = BatchNormalization.new
+  end
+
+  def call(x)
+    x = InputLayer.new(20).(x)
+    x = @l1.(x)
+    x = @bn1.(x)
+    x = ReLU.(x)
+
+    x = @l2.(x)
+    x = @bn2.(x)
+    x = ReLU.(x)
+
+    x = Reshape.(x, [7, 7, 64])
+    x = @l3.(x)
+    x = @bn3.(x)
+    x = ReLU.(x)
+
+    x = @l4.(x)
+    x = @bn4.(x)
+    x = ReLU.(x)
+
+    x = @l5.(x)
+    x = @bn5.(x)
+    x = ReLU.(x)
+
+    x = @l6.(x)
+    x = @bn6.(x)
+    x = ReLU.(x)
+
+    x = @l7.(x)
+    x = Tanh.(x)
+    x
+  end
+end
+
+
+class Discriminator < Model
+  def initialize
+    super
+    @l1 = Conv2D.new(32, 4, strides: 2, padding: true)
+    @l2 = Conv2D.new(32, 4, padding: true)
+    @l3 = Conv2D.new(64, 4, strides: 2, padding: true)
+    @l4 = Conv2D.new(64, 4, padding: true)
+    @l5 = Dense.new(1024)
+    @l6 = Dense.new(1)
+  end
+
+  def call(x, trainable = true)
+    has_param_layers.each do |layer|
+      layer.trainable = trainable
+    end
+    x = InputLayer.new([28, 28, 1]).(x)
+    x = @l1.(x)
+    x = LeakyReLU.(x, 0.2)
+
+    x = @l2.(x)
+    x = LeakyReLU.(x, 0.2)
+
+    x = @l3.(x)
+    x = LeakyReLU.(x, 0.2)
+
+    x = @l4.(x)
+    x = LeakyReLU.(x, 0.2)
+
+    x = Flatten.(x)
+    x = @l5.(x)
+    x = LeakyReLU.(x, 0.2)
+
+    x = @l6.(x)
+    x
+  end
+end
+
+
+class DCGAN < Model
+  attr_reader :gen
+  attr_reader :dis
+
+  def initialize(gen, dis)
+    super()
+    @gen = gen
+    @dis = dis
+  end
+
+  def call(x)
+    x = @gen.(x)
+    x = @dis.(x, false)
+    x
+  end
+end

data/examples/dcgan/imgen.rb

@@ -0,0 +1,20 @@
+require "dnn"
+require "dnn/image"
+require "numo/linalg/autoloader"
+require_relative "dcgan"
+
+Image = DNN::Image
+
+batch_size = 100
+
+dcgan = Model.load("trained/dcgan_model_epoch20.marshal")
+gen = dcgan.gen
+
+Numo::SFloat.srand(rand(1 << 31))
+noise = Numo::SFloat.new(batch_size, 20).rand(-1, 1)
+images = gen.predict(noise)
+
+batch_size.times do |i|
+  img = Numo::UInt8.cast(((images[i, false] + 1) * 127.5).round)
+  Image.write("img/img_#{i}.jpg", img)
+end

data/examples/dcgan/train.rb

@@ -0,0 +1,41 @@
+require "dnn"
+require "dnn/mnist"
+require "numo/linalg/autoloader"
+require_relative "dcgan"
+
+MNIST = DNN::MNIST
+
+Numo::SFloat.srand(rand(1 << 31))
+
+epochs = 20
+batch_size = 128
+
+gen = Generator.new
+dis = Discriminator.new
+dcgan = DCGAN.new(gen, dis)
+
+dis.setup(Adam.new(alpha: 0.00001, beta1: 0.1), SigmoidCrossEntropy.new)
+dcgan.setup(Adam.new(alpha: 0.0002, beta1: 0.5), SigmoidCrossEntropy.new)
+
+x_train, y_train = MNIST.load_train
+x_train = Numo::SFloat.cast(x_train)
+x_train = x_train / 127.5 - 1
+
+iter = DNN::Iterator.new(x_train, y_train)
+(1..epochs).each do |epoch|
+  puts "epoch: #{epoch}"
+  iter.foreach(batch_size) do |x_batch, y_batch, index|
+    noise = Numo::SFloat.new(batch_size, 20).rand(-1, 1)
+    images = gen.predict(noise)
+    x = x_batch.concatenate(images)
+    y = Numo::SFloat.cast([1] * batch_size + [0] * batch_size).reshape(batch_size * 2, 1)
+    dis_loss = dis.train_on_batch(x, y)
+
+    noise = Numo::SFloat.new(batch_size, 20).rand(-1, 1)
+    label = Numo::SFloat.cast([1] * batch_size).reshape(batch_size, 1)
+    dcgan_loss = dcgan.train_on_batch(noise, label)
+
+    puts "index: #{index}, dis_loss: #{dis_loss.mean}, dcgan_loss: #{dcgan_loss.mean}"
+  end
+  dcgan.save("trained/dcgan_model_epoch#{epoch}.marshal")
+end

data/examples/iris_example.rb

@@ -7,20 +7,17 @@ include DNN::Layers
 include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-Model = DNN::Model
+include DNN::Models
 Iris = DNN::Iris
 
 x, y = Iris.load(true)
 x_train, y_train = x[0...100, true], y[0...100]
 x_test, y_test = x[100...150, true], y[100...150]
 
-x_train /= 255
-x_test /= 255
-
 y_train = DNN::Utils.to_categorical(y_train, 3, Numo::SFloat)
 y_test = DNN::Utils.to_categorical(y_test, 3, Numo::SFloat)
 
-model = Model.new
+model = Sequential.new
 
 model << InputLayer.new(4)
 
@@ -29,6 +26,6 @@ model << ReLU.new
 
 model << Dense.new(3)
 
-model.compile(Adam.new, SoftmaxCrossEntropy.new)
+model.setup(Adam.new, SoftmaxCrossEntropy.new)
 
 model.train(x_train, y_train, 1000, batch_size: 10, test: [x_test, y_test])

data/examples/mnist_conv2d_example.rb

@@ -7,14 +7,14 @@ include DNN::Layers
 include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-Model = DNN::Model
+include DNN::Models
 MNIST = DNN::MNIST
 
 x_train, y_train = MNIST.load_train
 x_test, y_test = MNIST.load_test
 
-x_train = Numo::SFloat.cast(x_train).reshape(x_train.shape[0], 28, 28, 1)
-x_test = Numo::SFloat.cast(x_test).reshape(x_test.shape[0], 28, 28, 1)
+x_train = Numo::SFloat.cast(x_train)
+x_test = Numo::SFloat.cast(x_test)
 
 x_train /= 255
 x_test /= 255
@@ -22,7 +22,7 @@ x_test /= 255
 y_train = DNN::Utils.to_categorical(y_train, 10, Numo::SFloat)
 y_test = DNN::Utils.to_categorical(y_test, 10, Numo::SFloat)
 
-model = Model.new
+model = Sequential.new
 
 model << InputLayer.new([28, 28, 1])
 
@@ -45,6 +45,6 @@ model << Dropout.new(0.5)
 
 model << Dense.new(10)
 
-model.compile(Adam.new, SoftmaxCrossEntropy.new)
+model.setup(Adam.new, SoftmaxCrossEntropy.new)
 
 model.train(x_train, y_train, 10, batch_size: 100, test: [x_test, y_test])

data/examples/mnist_define_by_run.rb

@@ -0,0 +1,52 @@
+require "dnn"
+require "dnn/mnist"
+# If you use numo/linalg then please uncomment out.
+# require "numo/linalg/autoloader"
+
+include DNN::Layers
+include DNN::Activations
+include DNN::Optimizers
+include DNN::Losses
+include DNN::Models
+MNIST = DNN::MNIST
+
+x_train, y_train = MNIST.load_train
+x_test, y_test = MNIST.load_test
+
+x_train = Numo::SFloat.cast(x_train).reshape(x_train.shape[0], 784)
+x_test = Numo::SFloat.cast(x_test).reshape(x_test.shape[0], 784)
+
+x_train /= 255
+x_test /= 255
+
+y_train = DNN::Utils.to_categorical(y_train, 10, Numo::SFloat)
+y_test = DNN::Utils.to_categorical(y_test, 10, Numo::SFloat)
+
+class MLP < Model
+  def initialize
+    super
+    @l1 = Dense.new(256)
+    @l2 = Dense.new(256)
+    @l3 = Dense.new(10)
+    @bn1 = BatchNormalization.new
+    @bn2 = BatchNormalization.new
+  end
+
+  def call(x)
+    x = InputLayer.(x)
+    x = @l1.(x)
+    x = @bn1.(x)
+    x = ReLU.(x)
+    x = @l2.(x)
+    x = @bn2.(x)
+    x = ReLU.(x)
+    x = @l3.(x)
+    x
+  end
+end
+
+model = MLP.new
+
+model.setup(Adam.new, SoftmaxCrossEntropy.new)
+
+model.train(x_train, y_train, 10, batch_size: 100, test: [x_test, y_test])

data/examples/mnist_example.rb

@@ -7,7 +7,7 @@ include DNN::Layers
 include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-Model = DNN::Model
+include DNN::Models
 MNIST = DNN::MNIST
 
 x_train, y_train = MNIST.load_train
@@ -22,7 +22,7 @@ x_test /= 255
 y_train = DNN::Utils.to_categorical(y_train, 10, Numo::SFloat)
 y_test = DNN::Utils.to_categorical(y_test, 10, Numo::SFloat)
 
-model = Model.new
+model = Sequential.new
 
 model << InputLayer.new(784)
 
@@ -34,6 +34,6 @@ model << ReLU.new
 
 model << Dense.new(10)
 
-model.compile(RMSProp.new, SoftmaxCrossEntropy.new)
+model.setup(RMSProp.new, SoftmaxCrossEntropy.new)
 
 model.train(x_train, y_train, 10, batch_size: 100, test: [x_test, y_test])