ruby-dnn 0.16.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b76b0eb5bf75a22e48726f93fa4faff413c196b1f1587ce145aba7042c84a532
-  data.tar.gz: f32d09cb89391583f51a557e5f72024f618d943af3becc25f8f19905fa395b3b
+  metadata.gz: 0db9ac3047ba8c15d903ace901f5e4e332835d11dffca2f441664ae843049d1d
+  data.tar.gz: f1b4bf61da8a48b8ad483eb806ab443bb40f1b0d88573c2d901ae45299abf86d
 SHA512:
-  metadata.gz: f70784c49f71420df424c2077b430c0cb837b31b5493f62831cf49b296d834045796f2bae7749fa3b4bfb3ca365a97da856ccbebcc666c1489020eb55ec408fc
-  data.tar.gz: 52fd5cf850341b3d2bdd44da35f69254e508b0dd77e052a7217fe250a8d41a00d237b5ce0cb379e652bf2e6b7d0b99e95b3530ef5abe2d43a2b8f8c8f7dbca7e
+  metadata.gz: 880fe0688bb5b15c016fdddb15b18f5e0b3ba2a45ae36292182adf8def20d93ca3ae176747dbf3d1369ea28cdaaf23e7cd9e96d0a0c6c4bb92db27131d8f4d93
+  data.tar.gz: 4d00dc6831f0c82e0dc1b4128d98dc391456410cbf26e0d0192b19092333664774eccabd2db0a609b739c1c40664e148dcd69152cdc45394690e21d785e1acc0

data/README.md

@@ -42,6 +42,11 @@ model << Dense.new(10)
 model.setup(Adam.new, SoftmaxCrossEntropy.new)
 
 model.train(x_train, y_train, 10, batch_size: 128, test: [x_test, y_test])
+
+
+accuracy, loss = model.evaluate(x_test, y_test)
+puts "accuracy: #{accuracy}"
+puts "loss: #{loss}"
 ```
 
 When create a model with 'define by run' style:  
@@ -71,6 +76,10 @@ model = MLP.new
 model.setup(Adam.new, SoftmaxCrossEntropy.new)
 
 model.train(x_train, y_train, 10, batch_size: 128, test: [x_test, y_test])
+
+accuracy, loss = model.evaluate(x_test, y_test)
+puts "accuracy: #{accuracy}"
+puts "loss: #{loss}"
 ```
 
 Please refer to examples for basic usage.  
@@ -86,6 +95,19 @@ If you want to know more detailed information, please refer to the source code.
 | Optimizers | SGD, Nesterov, AdaGrad, RMSProp, AdaDelta, RMSPropGraves, Adam, AdaBound |
 | Losses | MeanSquaredError, MeanAbsoluteError, Hinge, HuberLoss, SoftmaxCrossEntropy, SigmoidCrossEntropy |
 
+## Datasets
+● Iris  
+● MNIST  
+● Fashion-MNIST  
+● CIFAR-10  
+● CIFAR-100  
+● STL-10
+
+## Examples
+● VAE  
+● DCGAN  
+● Pix2pix
+
 ## TODO
 ● Write a test.  
 ● Write a document.  

data/examples/api-examples/early_stopping_example.rb

@@ -35,7 +35,7 @@ class MLP < Model
   end
 
   def forward(x)
-    x = InputLayer.(x)
+    x = InputLayer.new(784).(x)
     x = @l1.(x)
     x = @bn1.(x)
     x = ReLU.(x)

data/examples/api-examples/initializer_example.rb

@@ -36,7 +36,7 @@ class MLP < Model
   end
 
   def forward(x)
-    x = InputLayer.(x)
+    x = InputLayer.new(784).(x)
     x = @l1.(x)
     x = @bn1.(x)
     x = ReLU.(x)

data/examples/api-examples/regularizer_example.rb

@@ -37,7 +37,7 @@ class MLP < Model
   end
 
   def forward(x)
-    x = InputLayer.(x)
+    x = InputLayer.new(784).(x)
     x = @l1.(x)
     x = @bn1.(x)
     x = ReLU.(x)

data/examples/dcgan/dcgan.rb

@@ -61,6 +61,9 @@ class Discriminator < Model
     @l4 = Conv2D.new(64, 4, padding: true)
     @l5 = Dense.new(1024)
     @l6 = Dense.new(1)
+    @bn1 = BatchNormalization.new
+    @bn2 = BatchNormalization.new
+    @bn3 = BatchNormalization.new
   end
 
   def forward(x)
@@ -69,12 +72,15 @@ class Discriminator < Model
     x = LeakyReLU.(x, 0.2)
 
     x = @l2.(x)
+    x = @bn1.(x)
     x = LeakyReLU.(x, 0.2)
 
     x = @l3.(x)
+    x = @bn2.(x)
     x = LeakyReLU.(x, 0.2)
 
     x = @l4.(x)
+    x = @bn3.(x)
     x = LeakyReLU.(x, 0.2)
 
     x = Flatten.(x)
@@ -119,10 +125,11 @@ class DCGAN < Model
     batch_size = x_batch.shape[0]
     noise = Numo::SFloat.new(batch_size, 20).rand(-1, 1)
     images = @gen.predict(noise)
-    x = x_batch.concatenate(images)
-    y = Numo::SFloat.cast([1] * batch_size + [0] * batch_size).reshape(batch_size * 2, 1)
+    y_real = Numo::SFloat.ones(batch_size, 1)
+    y_fake = Numo::SFloat.zeros(batch_size, 1)
     @dis.enable_training
-    dis_loss = @dis.train_on_batch(x, y)
+    dis_loss = @dis.train_on_batch(x_batch, y_real)
+    dis_loss + @dis.train_on_batch(images, y_fake)
 
     noise = Numo::SFloat.new(batch_size, 20).rand(-1, 1)
     label = Numo::SFloat.cast([1] * batch_size).reshape(batch_size, 1)

data/examples/pix2pix/dcgan.rb

@@ -2,6 +2,8 @@ include DNN::Models
 include DNN::Layers
 
 class Generator < Model
+  attr_reader :generate_images
+
   def initialize(input_shape)
     super()
     @input_shape = input_shape
@@ -25,6 +27,7 @@ class Generator < Model
     @bn7 = BatchNormalization.new
     @bn8 = BatchNormalization.new
     @bn9 = BatchNormalization.new
+    @generate_images = nil
   end
 
   def forward(x)
@@ -72,6 +75,7 @@ class Generator < Model
 
     x = @l11.(x)
     x = Tanh.(x)
+    @generate_images = x.data
     x
   end
 end

data/examples/pix2pix/train.rb

@@ -24,6 +24,7 @@ gen = Generator.new([32, 32, 1])
 dis = Discriminator.new([32, 32, 1], [32, 32, 3])
 dcgan = DCGAN.new(gen, dis)
 
+gen.setup(Adam.new(alpha: 0.0002, beta1: 0.5), MeanAbsoluteError.new)
 dis.setup(Adam.new(alpha: 0.00001, beta1: 0.1), SigmoidCrossEntropy.new)
 dcgan.setup(Adam.new(alpha: 0.0002, beta1: 0.5), SigmoidCrossEntropy.new)
 
@@ -35,7 +36,9 @@ num_batchs = x_in.shape[0] / batch_size
 (1..epochs).each do |epoch|
   num_batchs.times do |index|
     x_in, x_out = iter1.next_batch(batch_size)
-    images = gen.predict(x_in)
+    gen_loss = gen.train_on_batch(x_in, x_out)
+
+    images = gen.generate_images
     y_real = Numo::SFloat.ones(batch_size, 1)
     y_fake = Numo::SFloat.zeros(batch_size, 1)
     dis.enable_training
@@ -45,7 +48,7 @@ num_batchs = x_in.shape[0] / batch_size
     x_in, x_out = iter2.next_batch(batch_size)
     dcgan_loss = dcgan.train_on_batch(x_in, y_real)
 
-    puts "epoch: #{epoch}, index: #{index}, dis_loss: #{dis_loss}, dcgan_loss: #{dcgan_loss}"
+    puts "epoch: #{epoch}, index: #{index}, gen_loss: #{gen_loss}, dis_loss: #{dis_loss}, dcgan_loss: #{dcgan_loss}"
   end
   iter1.reset
   iter2.reset

data/examples/vae.rb

@@ -9,16 +9,10 @@ include DNN::Optimizers
 include DNN::Losses
 
 x_train, y_train = DNN::MNIST.load_train
-x_test, y_test = DNN::MNIST.load_test
 
 x_train = Numo::SFloat.cast(x_train).reshape(x_train.shape[0], 784)
-x_test = Numo::SFloat.cast(x_test).reshape(x_test.shape[0], 784)
 
 x_train /= 255
-x_test /= 255
-
-y_train = DNN::Utils.to_categorical(y_train, 10, Numo::SFloat)
-y_test = DNN::Utils.to_categorical(y_test, 10, Numo::SFloat)
 
 $z_dim = 2
 $z_mean = nil

data/lib/dnn/core/callbacks.rb

@@ -27,10 +27,11 @@ module DNN
 
     # This callback wrap the lambda function.
     class LambdaCallback < Callback
-      def initialize(event, lambda = nil, &block)
-        lambda = block unless lambda
+      # @param [Symbol] event Event to execute callback.
+      # @yield Register the contents of the callback.
+      def initialize(event, &block)
         instance_eval do
-          define_singleton_method(event) { lambda.call }
+          define_singleton_method(event) { block.call }
         end
       end
     end
@@ -55,6 +56,9 @@ module DNN
     end
 
     # A callback to stop training the model early after test on batch.
+    # @param [Symbol] trigger A log that triggers early stopping.
+    #                         Specify one of train_loss, test_loss, test_accuracy.
+    # @param [Float] tolerance Tolerance value for early stopping.
     class EarlyStopping < Callback
       def initialize(trigger, tolerance)
         @trigger = trigger

data/lib/dnn/core/error.rb

@@ -1,5 +1,5 @@
 module DNN
-  class DNN_Error < StandardError; end
+  class DNNError < StandardError; end
 
-  class DNN_ShapeError < DNN_Error; end
+  class DNNShapeError < DNNError; end
 end

data/lib/dnn/core/initializers.rb

@@ -6,7 +6,7 @@ module DNN
         return nil unless hash
         initializer_class = DNN.const_get(hash[:class])
         initializer = initializer_class.allocate
-        raise DNN_Error, "#{initializer.class} is not an instance of #{self} class." unless initializer.is_a?(self)
+        raise DNNError, "#{initializer.class} is not an instance of #{self} class." unless initializer.is_a?(self)
         initializer.load_hash(hash)
         initializer
       end
@@ -122,8 +122,8 @@ module DNN
 
       def init_param(layer, param)
         Xumo::SFloat.srand(@seed)
-        num_prev_nodes = layer.input_shape.reduce(:*)
-        param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes)
+        num_prev_units = layer.input_shape.reduce(:*)
+        param.data = param.data.rand_norm / Math.sqrt(num_prev_units)
       end
     end
 
@@ -134,8 +134,8 @@ module DNN
 
       def init_param(layer, param)
         Xumo::SFloat.srand(@seed)
-        num_prev_nodes = layer.input_shape.reduce(:*)
-        param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes) * Math.sqrt(2)
+        num_prev_units = layer.input_shape.reduce(:*)
+        param.data = param.data.rand_norm / Math.sqrt(num_prev_units) * Math.sqrt(2)
       end
     end
 

data/lib/dnn/core/iterator.rb

@@ -21,7 +21,7 @@ module DNN
     # @param [Integer] batch_size Required batch size.
     # @return [Array] Returns the mini batch in the form [x_batch, y_batch].
     def next_batch(batch_size)
-      raise DNN_Error, "This iterator has not next batch. Please call reset." unless has_next?
+      raise DNNError, "This iterator has not next batch. Please call reset." unless has_next?
       if @indexes.length <= batch_size
         batch_indexes = @indexes
         @has_next = false
@@ -60,6 +60,9 @@ module DNN
       @has_next
     end
 
+    # Run a loop with all data separated by batch
+    # @param [Integer] batch_size Batch size.
+    # @yield Executes block by receiving the specified arguments (x_batch, y_batch).
     def foreach(batch_size, &block)
       steps = @last_round_down ? @num_datas / batch_size : (@num_datas.to_f / batch_size).ceil
       steps.times do |step|

data/lib/dnn/core/layers/basic_layers.rb

@@ -2,18 +2,14 @@ module DNN
   module Layers
 
     module LayerNode
-      def forward(input_tensor)
-        x = input_tensor.data
-        prev_link = (input_tensor.is_a?(Tensor) ? input_tensor.link : input_tensor)
+      def forward(input)
+        x = input.data
+        prev = (input.is_a?(Tensor) ? input.link : input)
         y = forward_node(x)
-        link = Link.new(prev_link, self)
+        link = Link.new(prev, self)
         Tensor.new(y, link)
       end
 
-      def backward(dy)
-        backward_node(dy)
-      end
-
       def forward_node(x)
         raise NotImplementedError, "Class '#{self.class.name}' has implement method 'forward_node'"
       end
@@ -26,6 +22,7 @@ module DNN
     # Super class of all layer classes.
     class Layer
       attr_reader :input_shape
+      attr_reader :output_shape
 
       def self.call(x, *args)
         new(*args).(x)
@@ -35,7 +32,7 @@ module DNN
         return nil unless hash
         layer_class = DNN.const_get(hash[:class])
         layer = layer_class.allocate
-        raise DNN_Error, "#{layer.class} is not an instance of #{self} class." unless layer.is_a?(self)
+        raise DNNError, "#{layer.class} is not an instance of #{self} class." unless layer.is_a?(self)
         layer.load_hash(hash)
         layer
       end
@@ -45,18 +42,19 @@ module DNN
       end
 
       # Forward propagation and create a link.
-      # @param [Tensor] input_tensor Input tensor.
+      # @param [Tensor | Param] input Input tensor or param.
       # @return [Tensor] Output tensor.
-      def call(input_tensor)
-        input_tensor = Tensor.new(input_tensor) if !input_tensor.is_a?(Tensor) && !input_tensor.is_a?(Param)
-        build(input_tensor.data.shape[1..-1]) unless built?
-        forward(input_tensor)
+      def call(input)
+        input = Tensor.new(input) if !input.is_a?(Tensor) && !input.is_a?(Param)
+        build(input.data.shape[1..-1]) unless built?
+        forward(input)
       end
 
       # Build the layer.
       # @param [Array] input_shape Setting the shape of the input data.
       def build(input_shape)
         @input_shape = input_shape
+        @output_shape = compute_output_shape
         @built = true
       end
 
@@ -66,16 +64,16 @@ module DNN
       end
 
       # Forward propagation.
-      # @param [Tensor] input_tensor Input tensor.
+      # @param [Tensor] input Input tensor or param.
       # @return [Tensor] Output tensor.
-      def forward(input_tensor)
+      def forward(input)
         raise NotImplementedError, "Class '#{self.class.name}' has implement method 'forward'"
       end
 
       # Please reimplement this method as needed.
       # The default implementation return input_shape.
       # @return [Array] Return the shape of the output data.
-      def output_shape
+      def compute_output_shape
         @input_shape
       end
 
@@ -135,60 +133,37 @@ module DNN
     end
 
     class InputLayer < Layer
-      include LayerNode
-
-      def self.call(input)
-        shape = input.is_a?(Tensor) ? input.data.shape : input.shape
-        new(shape[1..-1]).(input)
-      end
-
       # @param [Array] input_dim_or_shape Setting the shape or dimension of the input data.
       def initialize(input_dim_or_shape)
         super()
         @input_shape = input_dim_or_shape.is_a?(Array) ? input_dim_or_shape : [input_dim_or_shape]
       end
 
-      def call(input)
-        build(@input_shape) unless built?
-        if input.is_a?(Tensor)
-          x = input.data
-          prev_link = input&.link
-        else
-          x = input
-          prev_link = nil
-        end
-        Tensor.new(forward_node(x), Link.new(prev_link, self))
-      end
-
       def build(input_shape)
-        @built = true
+        super(@input_shape)
       end
 
-      def forward_node(x)
+      def forward(x)
         unless x.shape[1..-1] == @input_shape
-          raise DNN_ShapeError, "The shape of x does not match the input shape. input shape is #{@input_shape}, but x shape is #{x.shape[1..-1]}."
+          raise DNNShapeError, "The shape of x does not match the input shape. input shape is #{@input_shape}, but x shape is #{x.shape[1..-1]}."
         end
         x
       end
 
-      def backward_node(dy)
-        dy
-      end
-
       def to_proc
         method(:call).to_proc
       end
 
       def >>(layer)
         if RUBY_VERSION < "2.6.0"
-          raise DNN_Error, "Function composition is not supported before ruby version 2.6.0."
+          raise DNNError, "Function composition is not supported before ruby version 2.6.0."
         end
         to_proc >> layer
       end
 
       def <<(layer)
         if RUBY_VERSION < "2.6.0"
-          raise DNN_Error, "Function composition is not supported before ruby version 2.6.0."
+          raise DNNError, "Function composition is not supported before ruby version 2.6.0."
         end
         to_proc << layer
       end
@@ -267,10 +242,10 @@ module DNN
     class Dense < Connection
       include LayerNode
 
-      attr_reader :num_nodes
+      attr_reader :num_units
 
-      # @param [Integer] num_nodes Number of nodes.
-      def initialize(num_nodes,
+      # @param [Integer] num_units Number of nodes.
+      def initialize(num_units,
                      weight_initializer: Initializers::RandomNormal.new,
                      bias_initializer: Initializers::Zeros.new,
                      weight_regularizer: nil,
@@ -278,17 +253,17 @@ module DNN
                      use_bias: true)
         super(weight_initializer: weight_initializer, bias_initializer: bias_initializer,
               weight_regularizer: weight_regularizer, bias_regularizer: bias_regularizer, use_bias: use_bias)
-        @num_nodes = num_nodes
+        @num_units = num_units
       end
 
       def build(input_shape)
         unless input_shape.length == 1
-          raise DNN_ShapeError, "Input shape is #{input_shape}. But input shape must be 1 dimensional."
+          raise DNNShapeError, "Input shape is #{input_shape}. But input shape must be 1 dimensional."
         end
         super
-        num_prev_nodes = input_shape[0]
-        @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
-        @bias.data = Xumo::SFloat.new(@num_nodes) if @bias
+        num_prev_units = input_shape[0]
+        @weight.data = Xumo::SFloat.new(num_prev_units, @num_units)
+        @bias.data = Xumo::SFloat.new(@num_units) if @bias
         init_weight_and_bias
       end
 
@@ -307,16 +282,16 @@ module DNN
         dy.dot(@weight.data.transpose)
       end
 
-      def output_shape
-        [@num_nodes]
+      def compute_output_shape
+        [@num_units]
       end
 
       def to_hash
-        super(num_nodes: @num_nodes)
+        super(num_units: @num_units)
       end
 
       def load_hash(hash)
-        initialize(hash[:num_nodes],
+        initialize(hash[:num_units],
                    weight_initializer: Initializers::Initializer.from_hash(hash[:weight_initializer]),
                    bias_initializer: Initializers::Initializer.from_hash(hash[:bias_initializer]),
                    weight_regularizer: Regularizers::Regularizer.from_hash(hash[:weight_regularizer]),
@@ -329,14 +304,14 @@ module DNN
       include LayerNode
 
       def forward_node(x)
-        x.reshape(x.shape[0], *output_shape)
+        x.reshape(x.shape[0], *@output_shape)
       end
 
       def backward_node(dy)
         dy.reshape(dy.shape[0], *@input_shape)
       end
 
-      def output_shape
+      def compute_output_shape
         [@input_shape.reduce(:*)]
       end
     end
@@ -344,11 +319,13 @@ module DNN
     class Reshape < Layer
       include LayerNode
 
-      attr_reader :output_shape
-
-      def initialize(output_shape)
+      def initialize(shape)
         super()
-        @output_shape = output_shape
+        @shape = shape
+      end
+
+      def compute_output_shape
+        @shape
       end
 
       def forward_node(x)
@@ -360,11 +337,11 @@ module DNN
       end
 
       def to_hash
-        super(output_shape: @output_shape)
+        super(shape: @shape)
       end
 
       def load_hash(hash)
-        initialize(hash[:output_shape])
+        initialize(hash[:shape])
       end
     end