ruby-dnn 0.13.4 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cfe06541a0afe5fd3465839f02865efe7105c7635003840f60fd46e2976b0c26
-  data.tar.gz: d0f2150f86fe1ee423231c410af8809cecf9d412ecefebb0a3bee1ea14abeee7
+  metadata.gz: '08960a5015b086d49e27dc44ea1076cb56c6548cf2945698dd81ae30a1f738e7'
+  data.tar.gz: 2b78bfa3f91b224226fa2f268f0fb0cc94c2d711bc1d7324dddf32d7e50be409
 SHA512:
-  metadata.gz: 65ad242e2647f2035409432b6568a72d73b95a10b096d863938f9e0cef3bc07aac9a770a9d900d021af2afc3f912a9a3ed4725ba6488411c758b5f5a717a8964
-  data.tar.gz: d4102ded5ade667087736ecea6f897f1136b138682c93a53869e68b3fa70683a3bc8570e3ef3eb840c0a5f13ceb38a7f41b6c9a6f6b4d9bee9543b52ead2a95a
+  metadata.gz: ccdc09a73a918fd8eb206283d9d2db396a528f737d9981d49ecb90affcfa1d417429c3fb0a7070d57563a3168c3283a6aa762e04271df58d9b57bb2232a96545
+  data.tar.gz: 5e01577078cfee549f2f5b963fa3565a59ea1fc805eca9bebea5a368b95e32f5c248db0e42d28c883eb71b138ce70372abb7f799acc3e2fde72550b3db0996f3

data/examples/cifar100_example.rb

@@ -3,11 +3,10 @@ require "dnn/datasets/cifar100"
 # If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
+include DNN::Models
 include DNN::Layers
-include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-include DNN::Models
 CIFAR100 = DNN::CIFAR100
 
 x_train, y_train = CIFAR100.load_train

data/examples/cifar10_example.rb

@@ -3,11 +3,10 @@ require "dnn/datasets/cifar10"
 # If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
+include DNN::Models
 include DNN::Layers
-include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-include DNN::Models
 CIFAR10 = DNN::CIFAR10
 
 x_train, y_train = CIFAR10.load_train

data/examples/dcgan/dcgan.rb

@@ -1,8 +1,5 @@
-include DNN::Layers
-include DNN::Activations
-include DNN::Optimizers
-include DNN::Losses
 include DNN::Models
+include DNN::Layers
 
 class Generator < Model
   def initialize
@@ -55,7 +52,6 @@ class Generator < Model
   end
 end
 
-
 class Discriminator < Model
   def initialize
     super
@@ -97,12 +93,11 @@ class Discriminator < Model
   end
 end
 
-
 class DCGAN < Model
-  attr_reader :gen
-  attr_reader :dis
+  attr_accessor :gen
+  attr_accessor :dis
 
-  def initialize(gen, dis)
+  def initialize(gen = nil, dis = nil)
     super()
     @gen = gen
     @dis = dis
@@ -113,4 +108,19 @@ class DCGAN < Model
     x = @dis.(x, false)
     x
   end
+
+  def train_step(x_batch, y_batch)
+    batch_size = x_batch.shape[0]
+    noise = Numo::SFloat.new(batch_size, 20).rand(-1, 1)
+    images = @gen.predict(noise)
+    x = x_batch.concatenate(images)
+    y = Numo::SFloat.cast([1] * batch_size + [0] * batch_size).reshape(batch_size * 2, 1)
+    dis_loss = @dis.train_on_batch(x, y)
+
+    noise = Numo::SFloat.new(batch_size, 20).rand(-1, 1)
+    label = Numo::SFloat.cast([1] * batch_size).reshape(batch_size, 1)
+    dcgan_loss = train_on_batch(noise, label)
+
+    { dis_loss: dis_loss.mean, dcgan_loss: dcgan_loss.mean }
+  end
 end

data/examples/dcgan/imgen.rb

@@ -3,15 +3,22 @@ require "dnn/image"
 require "numo/linalg/autoloader"
 require_relative "dcgan"
 
+include DNN::Loaders
 Image = DNN::Image
 
 batch_size = 100
 
-dcgan = DCGAN.load("trained/dcgan_model_epoch20.marshal")
-gen = dcgan.gen
+gen = Generator.new
+dis = Discriminator.new
+dcgan = DCGAN.new(gen, dis)
+dcgan.predict1(Numo::SFloat.zeros(20))
+
+loader = MarshalLoader.new(dcgan)
+loader.load("trained/dcgan_model_epoch20.marshal")
 
 Numo::SFloat.srand(rand(1 << 31))
 noise = Numo::SFloat.new(batch_size, 20).rand(-1, 1)
+
 images = gen.predict(noise)
 
 batch_size.times do |i|

data/examples/dcgan/train.rb

@@ -3,6 +3,9 @@ require "dnn/datasets/mnist"
 require "numo/linalg/autoloader"
 require_relative "dcgan"
 
+include DNN::Optimizers
+include DNN::Losses
+include DNN::Callbacks
 MNIST = DNN::MNIST
 
 Numo::SFloat.srand(rand(1 << 31))
@@ -17,28 +20,10 @@ dcgan = DCGAN.new(gen, dis)
 dis.setup(Adam.new(alpha: 0.00001, beta1: 0.1), SigmoidCrossEntropy.new)
 dcgan.setup(Adam.new(alpha: 0.0002, beta1: 0.5), SigmoidCrossEntropy.new)
 
-x_train, y_train = MNIST.load_train
+x_train, * = MNIST.load_train
 x_train = Numo::SFloat.cast(x_train)
 x_train = x_train / 127.5 - 1
 
-iter = DNN::Iterator.new(x_train, y_train)
-num_batchs = x_train.shape[0] / batch_size
-(1..epochs).each do |epoch|
-  puts "epoch: #{epoch}"
-  num_batchs.times do |index|
-    x_batch, y_batch = iter.next_batch(batch_size)
-    noise = Numo::SFloat.new(batch_size, 20).rand(-1, 1)
-    images = gen.predict(noise)
-    x = x_batch.concatenate(images)
-    y = Numo::SFloat.cast([1] * batch_size + [0] * batch_size).reshape(batch_size * 2, 1)
-    dis_loss = dis.train_on_batch(x, y)
-
-    noise = Numo::SFloat.new(batch_size, 20).rand(-1, 1)
-    label = Numo::SFloat.cast([1] * batch_size).reshape(batch_size, 1)
-    dcgan_loss = dcgan.train_on_batch(noise, label)
-
-    puts "index: #{index}, dis_loss: #{dis_loss.mean}, dcgan_loss: #{dcgan_loss.mean}"
-  end
-  iter.reset
-  dcgan.save("trained/dcgan_model_epoch#{epoch}.marshal")
-end
+dcgan.add_callback(CheckPoint.new("trained/dcgan_model"))
+dcgan.predict1(Numo::SFloat.zeros(20))
+dcgan.train(x_train, x_train, epochs, batch_size: batch_size, last_round_down: true)

data/examples/iris_example.rb

@@ -3,11 +3,10 @@ require "dnn/datasets/iris"
 # If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
+include DNN::Models
 include DNN::Layers
-include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-include DNN::Models
 Iris = DNN::Iris
 
 x, y = Iris.load(true)

data/examples/mnist_conv2d_example.rb

@@ -3,11 +3,10 @@ require "dnn/datasets/mnist"
 # If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
+include DNN::Models
 include DNN::Layers
-include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-include DNN::Models
 MNIST = DNN::MNIST
 
 x_train, y_train = MNIST.load_train

data/examples/mnist_define_by_run.rb

@@ -3,11 +3,10 @@ require "dnn/datasets/mnist"
 # If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
+include DNN::Models
 include DNN::Layers
-include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-include DNN::Models
 MNIST = DNN::MNIST
 
 x_train, y_train = MNIST.load_train

data/examples/mnist_example.rb

@@ -3,11 +3,10 @@ require "dnn/datasets/mnist"
 # If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
+include DNN::Models
 include DNN::Layers
-include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-include DNN::Models
 MNIST = DNN::MNIST
 
 x_train, y_train = MNIST.load_train

data/examples/mnist_lstm_example.rb

@@ -3,11 +3,10 @@ require "dnn/datasets/mnist"
 # If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
+include DNN::Models
 include DNN::Layers
-include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-include DNN::Models
 MNIST = DNN::MNIST
 
 x_train, y_train = MNIST.load_train

data/examples/xor_example.rb

@@ -1,10 +1,9 @@
 require "dnn"
 
+include DNN::Models
 include DNN::Layers
-include DNN::Activations
 include DNN::Optimizers
 include DNN::Losses
-include DNN::Models
 
 x = Numo::SFloat[[0, 0], [1, 0], [0, 1], [1, 1]]
 y = Numo::SFloat[[0], [1], [1], [0]]
@@ -20,4 +19,4 @@ model.setup(SGD.new, SigmoidCrossEntropy.new)
 
 model.train(x, y, 20000, batch_size: 4, verbose: false)
 
-p DNN::Utils.sigmoid(model.predict(x))
+p model.predict(x)

data/lib/dnn.rb

@@ -10,6 +10,7 @@ end
 require_relative "dnn/version"
 require_relative "dnn/core/error"
 require_relative "dnn/core/global"
+require_relative "dnn/core/tensor"
 require_relative "dnn/core/models"
 require_relative "dnn/core/param"
 require_relative "dnn/core/link"
@@ -25,5 +26,6 @@ require_relative "dnn/core/cnn_layers"
 require_relative "dnn/core/embedding"
 require_relative "dnn/core/rnn_layers"
 require_relative "dnn/core/optimizers"
+require_relative "dnn/core/callbacks"
 require_relative "dnn/core/savers"
 require_relative "dnn/core/utils"

data/lib/dnn/core/activations.rb

@@ -1,7 +1,7 @@
 module DNN
-  module Activations
+  module Layers
 
-    class Sigmoid < Layers::Layer
+    class Sigmoid < Layer
       def forward(x)
         @y = 1 / (1 + Xumo::NMath.exp(-x))
       end
@@ -11,31 +11,28 @@ module DNN
       end
     end
 
-
-    class Tanh < Layers::Layer
+    class Tanh < Layer
       def forward(x)
         @y = Xumo::NMath.tanh(x)
       end
 
       def backward(dy)
-        dy * (1 - @y ** 2)
+        dy * (1 - @y**2)
       end
     end
 
-
-    class Softsign < Layers::Layer
+    class Softsign < Layer
       def forward(x)
         @x = x
         x / (1 + x.abs)
       end
 
       def backward(dy)
-        dy * (1 / (1 + @x.abs) ** 2)
+        dy * (1 / (1 + @x.abs)**2)
       end
     end
 
-
-    class Softplus < Layers::Layer
+    class Softplus < Layer
       def forward(x)
         @x = x
         Xumo::NMath.log(1 + Xumo::NMath.exp(x))
@@ -46,8 +43,7 @@ module DNN
       end
     end
 
-
-    class Swish < Layers::Layer
+    class Swish < Layer
       def forward(x)
         @x = x
         @y = x * (1 / (1 + Xumo::NMath.exp(-x)))
@@ -58,8 +54,7 @@ module DNN
       end
     end
 
-
-    class ReLU < Layers::Layer
+    class ReLU < Layer
       def forward(x)
         @x = x
         Xumo::SFloat.maximum(0, x)
@@ -70,8 +65,7 @@ module DNN
       end
     end
 
-
-    class LeakyReLU < Layers::Layer
+    class LeakyReLU < Layer
       attr_reader :alpha
 
       # @param [Float] alpha The slope when the output value is negative.
@@ -102,8 +96,7 @@ module DNN
       end
     end
 
-
-    class ELU < Layers::Layer
+    class ELU < Layer
       attr_reader :alpha
 
       # @param [Float] alpha The slope when the output value is negative.

data/lib/dnn/core/callbacks.rb

@@ -0,0 +1,136 @@
+module DNN
+  module Callbacks
+
+    class Callback
+      attr_accessor :model
+
+      # Please implement the method used for callback event.
+
+      # Process performed before one training.
+      # def before_epoch; end
+
+      # Process performed after one training.
+      # def after_epoch; end
+
+      # Set the proc to be performed before train on batch processing.
+      # def before_train_on_batch; end
+
+      # Set the proc to be performed after train on batch processing.
+      # def after_train_on_batch; end
+
+      # Set the proc to be performed before test on batch processing.
+      # def before_test_on_batch; end
+
+      # Set the proc to be performed after test on batch processing.
+      # def after_test_on_batch; end
+    end
+
+    # This callback wrap the lambda function.
+    class LambdaCallback < Callback
+      def initialize(event, lambda)
+        instance_eval do
+          define_singleton_method(event) { lambda.call }
+        end
+      end
+    end
+
+    # A callback that save the model at the after of the epoch.
+    class CheckPoint < Callback
+      def initialize(base_file_name)
+        @base_file_name = base_file_name
+      end
+
+      def after_epoch
+        model.save(@base_file_name + "_epoch#{model.last_log[:epoch]}.marshal")
+      end
+    end
+
+    # A callback to stop training the model early after test on batch.
+    class EarlyStopping < Callback
+      def initialize(trigger, tolerance)
+        @trigger = trigger
+        @tolerance = tolerance
+      end
+
+      def after_train_on_batch
+        throw :stop, "Early stopped." if judge_early_stopping_train
+      end
+
+      def after_epoch
+        throw :stop, "Early stopped." if judge_early_stopping_test
+      end
+
+      private
+
+      def judge_early_stopping_train
+        case @trigger
+        when :train_loss
+          return true if model.last_log[@trigger].mean <= @tolerance
+        end
+        false
+      end
+
+      def judge_early_stopping_test
+        case @trigger
+        when :test_loss
+          return true if model.last_log[@trigger].mean <= @tolerance
+        when :test_accuracy
+          return true if model.last_log[@trigger] >= @tolerance
+        end
+        false
+      end
+    end
+
+    # A callback to stop training the model if loss is NaN by after train on batch.
+    class NaNStopping < Callback
+      def after_train_on_batch
+        throw :stop, "loss is NaN." if model.last_log[:train_loss].nan?
+      end
+    end
+
+    # A callback that save the log.
+    # The following logs will be recorded.
+    # epoch:          Current epoch.
+    # train_loss:     Batch training loss.
+    # test_loss:      Mean test loss.
+    # test_accuracy:  Test accuracy.
+    class Logger < Callback
+      def initialize
+        @log = {
+          epoch: [],
+          train_loss: [],
+          test_loss: [],
+          test_accuracy: [],
+        }
+      end
+
+      def after_epoch
+        logging(:epoch, :test_loss, :test_accuracy)
+      end
+
+      def after_train_on_batch
+        logging(:train_loss)
+      end
+
+      # Get a log.
+      # @param [Symbol] tag Tag indicating the type of Log.
+      # @return [Numo::NArray] Return the recorded log.
+      def get_log(tag)
+        case tag
+        when :epoch
+          Numo::UInt32.cast(@log[tag])
+        else
+          Numo::SFloat.cast(@log[tag])
+        end
+      end
+
+      private def logging(*tags)
+        tags.each do |tag|
+          @log[tag] ||= []
+          @log[tag] << model.last_log[tag]
+        end
+      end
+    end
+
+  end
+end