ruby-dnn 0.14.0 → 0.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08960a5015b086d49e27dc44ea1076cb56c6548cf2945698dd81ae30a1f738e7'
-  data.tar.gz: 2b78bfa3f91b224226fa2f268f0fb0cc94c2d711bc1d7324dddf32d7e50be409
+  metadata.gz: a07d117f60cfce7327a63514df0ffb2bf2228d4dfaa531d5b1c849ad581c5aa3
+  data.tar.gz: 460158875978df670f99a501dc1a5878c392517d2c5123f13c3c8c96718f3a5e
 SHA512:
-  metadata.gz: ccdc09a73a918fd8eb206283d9d2db396a528f737d9981d49ecb90affcfa1d417429c3fb0a7070d57563a3168c3283a6aa762e04271df58d9b57bb2232a96545
-  data.tar.gz: 5e01577078cfee549f2f5b963fa3565a59ea1fc805eca9bebea5a368b95e32f5c248db0e42d28c883eb71b138ce70372abb7f799acc3e2fde72550b3db0996f3
+  metadata.gz: e8cbb2fc0992b458f5af4176d2757f9e1222aba1cf1a980db5f8c4e64c20f8442bb1c6c103f78adbe5855bfdd7a8f01b33ffe3978b369b20b52efbdd9de7af4c
+  data.tar.gz: a67ced6f09ea4d1fd5c81a7e13a1ac1b35d46512b54e06fa3773025036fed3cf5200b5cefb985defe4debd8fc2b8a669fa9a48a2ea21ee6721ff539c89c1d944

data/examples/api-examples/early_stopping_example.rb

@@ -0,0 +1,58 @@
+require "dnn"
+require "dnn/datasets/mnist"
+# If you use numo/linalg then please uncomment out.
+# require "numo/linalg/autoloader"
+
+include DNN::Models
+include DNN::Layers
+include DNN::Optimizers
+include DNN::Losses
+include DNN::Callbacks
+MNIST = DNN::MNIST
+
+EPOCHS = 3
+BATCH_SIZE = 128
+
+x_train, y_train = MNIST.load_train
+x_test, y_test = MNIST.load_test
+
+x_train = Numo::SFloat.cast(x_train).reshape(x_train.shape[0], 784)
+x_test = Numo::SFloat.cast(x_test).reshape(x_test.shape[0], 784)
+
+x_train /= 255
+x_test /= 255
+
+y_train = DNN::Utils.to_categorical(y_train, 10, Numo::SFloat)
+y_test = DNN::Utils.to_categorical(y_test, 10, Numo::SFloat)
+
+class MLP < Model
+  def initialize
+    super
+    @l1 = Dense.new(256)
+    @l2 = Dense.new(256)
+    @l3 = Dense.new(10)
+    @bn1 = BatchNormalization.new
+    @bn2 = BatchNormalization.new
+  end
+
+  def call(x)
+    x = InputLayer.(x)
+    x = @l1.(x)
+    x = @bn1.(x)
+    x = ReLU.(x)
+    x = @l2.(x)
+    x = @bn2.(x)
+    x = ReLU.(x)
+    x = @l3.(x)
+    x
+  end
+end
+
+model = MLP.new
+model.setup(Adam.new, SoftmaxCrossEntropy.new)
+
+# Add EarlyStopping callback for model.
+# This callback is stop the training when test accuracy is over 0.9.
+model.add_callback(EarlyStopping.new(:test_accuracy, 0.9))
+
+model.train(x_train, y_train, EPOCHS, batch_size: BATCH_SIZE, test: [x_test, y_test])

data/examples/api-examples/initializer_example.rb

@@ -0,0 +1,54 @@
+require "dnn"
+require "dnn/datasets/mnist"
+# If you use numo/linalg then please uncomment out.
+# require "numo/linalg/autoloader"
+
+include DNN::Models
+include DNN::Layers
+include DNN::Initializers
+include DNN::Optimizers
+include DNN::Losses
+MNIST = DNN::MNIST
+
+EPOCHS = 3
+BATCH_SIZE = 128
+
+x_train, y_train = MNIST.load_train
+x_test, y_test = MNIST.load_test
+
+x_train = Numo::SFloat.cast(x_train).reshape(x_train.shape[0], 784)
+x_test = Numo::SFloat.cast(x_test).reshape(x_test.shape[0], 784)
+
+x_train /= 255
+x_test /= 255
+
+y_train = DNN::Utils.to_categorical(y_train, 10, Numo::SFloat)
+y_test = DNN::Utils.to_categorical(y_test, 10, Numo::SFloat)
+
+class MLP < Model
+  def initialize
+    super
+    # Set the initial values of weight and bias to the initial values of He.
+    @l1 = Dense.new(256, weight_initializer: He.new, bias_initializer: He.new)
+    @l2 = Dense.new(256, weight_initializer: He.new, bias_initializer: He.new)
+    @l3 = Dense.new(10, weight_initializer: He.new, bias_initializer: He.new)
+    @bn1 = BatchNormalization.new
+    @bn2 = BatchNormalization.new
+  end
+
+  def call(x)
+    x = InputLayer.(x)
+    x = @l1.(x)
+    x = @bn1.(x)
+    x = ReLU.(x)
+    x = @l2.(x)
+    x = @bn2.(x)
+    x = ReLU.(x)
+    x = @l3.(x)
+    x
+  end
+end
+
+model = MLP.new
+model.setup(Adam.new, SoftmaxCrossEntropy.new)
+model.train(x_train, y_train, EPOCHS, batch_size: BATCH_SIZE, test: [x_test, y_test])

data/examples/api-examples/regularizer_example.rb

@@ -0,0 +1,55 @@
+require "dnn"
+require "dnn/datasets/mnist"
+# If you use numo/linalg then please uncomment out.
+# require "numo/linalg/autoloader"
+
+include DNN::Models
+include DNN::Layers
+include DNN::Regularizers
+include DNN::Optimizers
+include DNN::Losses
+MNIST = DNN::MNIST
+
+EPOCHS = 3
+BATCH_SIZE = 128
+L2_LAMBDA = 0.01
+
+x_train, y_train = MNIST.load_train
+x_test, y_test = MNIST.load_test
+
+x_train = Numo::SFloat.cast(x_train).reshape(x_train.shape[0], 784)
+x_test = Numo::SFloat.cast(x_test).reshape(x_test.shape[0], 784)
+
+x_train /= 255
+x_test /= 255
+
+y_train = DNN::Utils.to_categorical(y_train, 10, Numo::SFloat)
+y_test = DNN::Utils.to_categorical(y_test, 10, Numo::SFloat)
+
+class MLP < Model
+  def initialize
+    super
+    # Set L2 regularizer(weight decay) for weight and bias.
+    @l1 = Dense.new(256, weight_regularizer: L2.new(L2_LAMBDA), bias_regularizer: L2.new(L2_LAMBDA))
+    @l2 = Dense.new(256, weight_regularizer: L2.new(L2_LAMBDA), bias_regularizer: L2.new(L2_LAMBDA))
+    @l3 = Dense.new(10, weight_regularizer: L2.new(L2_LAMBDA), bias_regularizer: L2.new(L2_LAMBDA))
+    @bn1 = BatchNormalization.new
+    @bn2 = BatchNormalization.new
+  end
+
+  def call(x)
+    x = InputLayer.(x)
+    x = @l1.(x)
+    x = @bn1.(x)
+    x = ReLU.(x)
+    x = @l2.(x)
+    x = @bn2.(x)
+    x = ReLU.(x)
+    x = @l3.(x)
+    x
+  end
+end
+
+model = MLP.new
+model.setup(Adam.new, SoftmaxCrossEntropy.new)
+model.train(x_train, y_train, EPOCHS, batch_size: BATCH_SIZE, test: [x_test, y_test])

data/examples/api-examples/save_example.rb

@@ -0,0 +1,85 @@
+require "dnn"
+require "dnn/datasets/mnist"
+# If you use numo/linalg then please uncomment out.
+# require "numo/linalg/autoloader"
+
+include DNN::Models
+include DNN::Layers
+include DNN::Optimizers
+include DNN::Losses
+include DNN::Savers
+include DNN::Loaders
+MNIST = DNN::MNIST
+
+USE_MARSHAL = 0
+USE_JSON = 1
+
+EPOCHS = 3
+BATCH_SIZE = 128
+
+# Select save style from USE_MARSHAL or USE_JSON.
+SAVE_STYLE = USE_MARSHAL
+
+# When set a true, save data included optimizer status.
+# This setting is enabled when SAVE_STYLE is USE_MARSHAL.
+INCLUDE_OPTIMIZER = false
+
+x_train, y_train = MNIST.load_train
+x_test, y_test = MNIST.load_test
+
+x_train = Numo::SFloat.cast(x_train).reshape(x_train.shape[0], 784)
+x_test = Numo::SFloat.cast(x_test).reshape(x_test.shape[0], 784)
+
+x_train /= 255
+x_test /= 255
+
+y_train = DNN::Utils.to_categorical(y_train, 10, Numo::SFloat)
+y_test = DNN::Utils.to_categorical(y_test, 10, Numo::SFloat)
+
+class MLP < Model
+  def initialize
+    super
+    @l1 = Dense.new(256)
+    @l2 = Dense.new(256)
+    @l3 = Dense.new(10)
+    @bn1 = BatchNormalization.new
+    @bn2 = BatchNormalization.new
+  end
+
+  def call(x)
+    x = InputLayer.(x)
+    x = @l1.(x)
+    x = @bn1.(x)
+    x = ReLU.(x)
+    x = @l2.(x)
+    x = @bn2.(x)
+    x = ReLU.(x)
+    x = @l3.(x)
+    x
+  end
+end
+
+model = MLP.new
+model.setup(Adam.new, SoftmaxCrossEntropy.new)
+model.train(x_train, y_train, EPOCHS, batch_size: BATCH_SIZE, test: [x_test, y_test])
+
+if SAVE_STYLE == USE_MARSHAL
+  saver = MarshalSaver.new(model, include_optimizer: INCLUDE_OPTIMIZER)
+  saver.save("trained_mnist.marshal")
+  # model.save("trained_mnist.marshal") # This code is equivalent to the code above.
+elsif SAVE_STYLE == USE_JSON
+  saver = JSONSaver.new(model)
+  saver.save("trained_mnist.json")
+end
+
+model2 = MLP.new
+if SAVE_STYLE == USE_MARSHAL
+  loader = MarshalLoader.new(model2)
+  loader.load("trained_mnist.marshal")
+  # MLP.load("trained_mnist.marshal") # This code is equivalent to the code above.
+elsif SAVE_STYLE == USE_JSON
+  loader = JSONLoader.new(model2)
+  loader.load("trained_mnist.json")
+end
+
+puts model2.accuracy(x_test, y_test)

data/lib/dnn/core/callbacks.rb

@@ -65,7 +65,7 @@ module DNN
       def judge_early_stopping_train
         case @trigger
         when :train_loss
-          return true if model.last_log[@trigger].mean <= @tolerance
+          return true if model.last_log[@trigger] <= @tolerance
         end
         false
       end
@@ -73,7 +73,7 @@ module DNN
       def judge_early_stopping_test
         case @trigger
         when :test_loss
-          return true if model.last_log[@trigger].mean <= @tolerance
+          return true if model.last_log[@trigger] <= @tolerance
         when :test_accuracy
           return true if model.last_log[@trigger] >= @tolerance
         end

data/lib/dnn/core/layers.rb

@@ -97,8 +97,9 @@ module DNN
     end
 
     class InputLayer < Layer
-      def self.call(input_tensor)
-        new(input_tensor.data.shape[1..-1]).(input_tensor)
+      def self.call(input)
+        shape = input.is_a?(Tensor) ? input.data.shape : input.shape
+        new(shape[1..-1]).(input)
       end
 
       # @param [Array] input_dim_or_shape Setting the shape or dimension of the input data.
@@ -107,9 +108,16 @@ module DNN
         @input_shape = input_dim_or_shape.is_a?(Array) ? input_dim_or_shape : [input_dim_or_shape]
       end
 
-      def call(input_tensor)
+      def call(input)
         build unless built?
-        Tensor.new(forward(input_tensor.data), Link.new(input_tensor&.link, self))
+        if input.is_a?(Tensor)
+          x = input.data
+          prev_link = input&.link
+        else
+          x = input
+          prev_link = nil
+        end
+        Tensor.new(forward(x), Link.new(prev_link, self))
       end
 
       def build
@@ -333,7 +341,7 @@ module DNN
       def forward(x)
         if DNN.learning_phase
           Xumo::SFloat.srand(@rnd.rand(1 << 31))
-          @mask = Xumo::SFloat.ones(*x.shape).rand < @dropout_ratio
+          @mask = Xumo::SFloat.new(*x.shape).rand < @dropout_ratio
           x[@mask] = 0
         elsif @use_scale
           x *= (1 - @dropout_ratio)

data/lib/dnn/core/losses.rb

@@ -17,7 +17,7 @@ module DNN
         end
         loss_value = forward(y, t)
         loss_value += regularizers_forward(layers) if layers
-        loss_value.is_a?(Float) ? Xumo::SFloat[loss_value] : loss_value
+        loss_value.is_a?(Float) ? loss_value : loss_value.sum
       end
 
       def forward(y, t)

data/lib/dnn/core/models.rb

@@ -135,7 +135,7 @@ module DNN
       # @return [Hash] Hash of contents to be output to log.
       private def train_step(x, y)
         loss_value = train_on_batch(x, y)
-        { loss: loss_value.mean }
+        { loss: loss_value }
       end
 
       # Implement the test process to be performed.
@@ -185,7 +185,7 @@ module DNN
         iter.foreach(batch_size) do |x_batch, y_batch|
           correct, loss_value = test_on_batch(x_batch, y_batch)
           total_correct += correct
-          sum_loss += loss_value.mean
+          sum_loss += loss_value
         end
         mean_loss = sum_loss / max_steps
         acc = total_correct.to_f / num_test_datas
@@ -259,8 +259,9 @@ module DNN
 
       # Save the model in marshal format.
       # @param [String] file_name Name to save model.
-      def save(file_name)
-        saver = Savers::MarshalSaver.new(self)
+      # @param [Boolean] include_optimizer Set true to save data included optimizer status.
+      def save(file_name, include_optimizer: true)
+        saver = Savers::MarshalSaver.new(self, include_optimizer: include_optimizer)
         saver.save(file_name)
       end
 
@@ -312,7 +313,12 @@ module DNN
       def forward(x, learning_phase)
         DNN.learning_phase = learning_phase
         @layers_cache = nil
-        output_tensor = call(Tensor.new(x, nil))
+        inputs = if x.is_a?(Array)
+                   x.map { |a| Tensor.new(a, nil) }
+                 else
+                   Tensor.new(x, nil)
+                 end
+        output_tensor = call(inputs)
         @last_link = output_tensor.link
         unless @built
           @built = true

data/lib/dnn/image.rb

@@ -18,9 +18,9 @@ module DNN
     # @param [String] file_name File name to read.
     # @param [Integer] channel_type Specify channel type of image.
     def self.read(file_name, channel_type = RGB)
-      raise ImageReadError.new("#{file_name} is not found.") unless File.exist?(file_name)
+      raise ImageReadError, "#{file_name} is not found." unless File.exist?(file_name)
       bin, w, h, n = Stb.stbi_load(file_name, channel_type)
-      raise ImageReadError.new("#{file_name} load failed.") if bin == ""
+      raise ImageReadError, "#{file_name} load failed." if bin == ""
       img = Numo::UInt8.from_binary(bin)
       img.reshape(h, w, channel_type)
     end
@@ -38,18 +38,26 @@ module DNN
       end
       h, w, ch = img.shape
       bin = img.to_binary
-      case file_name
-      when /\.png$/i
+      match_data = file_name.match(/\.(\w+)$/i)
+      if match_data
+        ext = match_data[1]
+      else
+        raise ImageWriteError, "File name has not extension."
+      end
+      case ext
+      when "png"
         stride_in_bytes = w * ch
         res = Stb.stbi_write_png(file_name, w, h, ch, bin, stride_in_bytes)
-      when /\.bmp$/i
+      when "bmp"
         res = Stb.stbi_write_bmp(file_name, w, h, ch, bin)
-      when /\.jpg$/i, /\.jpeg/i
+      when "jpg", "jpeg"
         raise TypeError, "quality:#{quality.class} is not an instance of Integer class." unless quality.is_a?(Integer)
         raise ArgumentError, "quality should be between 1 and 100." unless quality.between?(1, 100)
         res = Stb.stbi_write_jpg(file_name, w, h, ch, bin, quality)
+      else
+        raise ImageWriteError, "Extension '#{ext}' is not support."
       end
-      raise ImageWriteError.new("Image write failed.") if res == 0
+      raise ImageWriteError, "Image write failed." if res == 0
     end
 
     # Resize the image.
@@ -90,11 +98,11 @@ module DNN
     end
 
     private_class_method def self.img_check(img)
-      raise TypeError.new("img: #{img.class} is not an instance of the Numo::UInt8 class.") unless img.is_a?(Numo::UInt8)
+      raise TypeError, "img: #{img.class} is not an instance of the Numo::UInt8 class." unless img.is_a?(Numo::UInt8)
       if img.shape.length != 3
-        raise ImageShapeError.new("img shape is #{img.shape}. But img shape must be 3 dimensional.")
+        raise ImageShapeError, "img shape is #{img.shape}. But img shape must be 3 dimensional."
       elsif !img.shape[2].between?(1, 4)
-        raise ImageShapeError.new("img channel is #{img.shape[2]}. But img channel must be 1 or 2 or 3 or 4.")
+        raise ImageShapeError, "img channel is #{img.shape[2]}. But img channel must be 1 or 2 or 3 or 4."
       end
     end
   end

data/lib/dnn/version.rb

@@ -1,3 +1,3 @@
 module DNN
-  VERSION = "0.14.0"
+  VERSION = "0.14.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-dnn
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.14.1
 platform: ruby
 authors:
 - unagiootoro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-19 00:00:00.000000000 Z
+date: 2019-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -98,6 +98,10 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- examples/api-examples/early_stopping_example.rb
+- examples/api-examples/initializer_example.rb
+- examples/api-examples/regularizer_example.rb
+- examples/api-examples/save_example.rb
 - examples/cifar100_example.rb
 - examples/cifar10_example.rb
 - examples/dcgan/dcgan.rb