ruby-dnn 0.9.4 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 670d2d681e3929d20c5c855c97dbad1759560eb45018d20a9089c6bc62787833
-  data.tar.gz: a9360305a97aed1bac5300010fbfa3ea85565eab70d299a9d07acd78d6b24d63
+  metadata.gz: 9fbaca2827a3877b9c215d3a93b7c0e20a4a456471541e3e3f0140a95abf695b
+  data.tar.gz: 0e57474b603a0ba4d08d66176fea34b77e9dbc0c51639635bb861779a5bd047e
 SHA512:
-  metadata.gz: d1f3b49241a81bf8c56b595bdaeecf097d8d3d461a02ab37860c6d2eede4a7e5cfadd0f83d2759cdbbdc7d541bbadebd11e55a80b0cfce9c2bd8a216efb2596d
-  data.tar.gz: e92230987d4c59cf4395ee33695fc974c65940f7df6aecfc3f89ebf65bbbeaae95433079df8855c448850923b5d230807eca2d2c135d475df6aa1aba6431988a
+  metadata.gz: e02bcedc98d20dda9c4aae46955af777586cb433ca1d4f479248e56b2ec08a38fbb1b1023c644620664c40fda420a4584d98c1dd0080ab29212c0a0176b669d7
+  data.tar.gz: bb1486c4a85edbb5b5aed8e8feaffd15937893883d5d31b831427e7bcf663e4157452dd565f0bf4909b444e1241213f8f0a682f56d03d267954e66b9d312e7ab

data/README.md

@@ -1,4 +1,5 @@
 # ruby-dnn
+[![Gem Version](https://badge.fury.io/rb/ruby-dnn.svg)](https://badge.fury.io/rb/ruby-dnn)
 
 ruby-dnn is a ruby deep learning library. This library supports full connected neural network and convolution neural network.
 Currently, you can get 99% accuracy with MNIST and 74% with CIFAR 10.
@@ -21,10 +22,45 @@ Or install it yourself as:
 
 ## Usage
 
-Currently, only simple Japanese documents are available.
-Please refer to examples for basic usage.
+### MNIST MLP example
+
+```ruby
+model = Model.new
+
+model << InputLayer.new(784)
+
+model << Dense.new(256)
+model << ReLU.new
+
+model << Dense.new(256)
+model << ReLU.new
+
+model << Dense.new(10)
+
+model.compile(RMSProp.new, SoftmaxCrossEntropy.new)
+
+model.train(x_train, y_train, 10, batch_size: 100, test: [x_test, y_test])
+
+```
+
+Please refer to examples for basic usage.  
 If you want to know more detailed information, please refer to the source code.
 
+## Implemented
+|| Implemented classes |
+|:-----------|------------:|
+| Connections | Dense, Conv2D, Conv2D_Transpose, SimpleRNN, LSTM, GRU |
+| Layers | Flatten, Reshape, Dropout, BatchNormalization, MaxPool2D, AvgPool2D, UnPool2D |
+| Activations | Sigmoid, Tanh, Softsign, Softplus, Swish, ReLU, LeakyReLU, ELU |
+| Optimizers | SGD, Nesterov, AdaGrad, RMSProp, AdaDelta, Adam, RMSPropGraves |
+| Losses | MeanSquaredError, MeanAbsoluteError, HuberLoss, SoftmaxCrossEntropy, SigmoidCrossEntropy |
+
+## TODO
+● Add CI badge.  
+● Write a test.  
+● Write a document.  
+● Support to GPU.  
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake "spec"` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
@@ -41,4 +77,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Dnn project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/dnn/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the ruby-dnn project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/dnn/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -17,3 +17,9 @@ task :build_image_io do
 end
 
 task :default => [:test, :build_dataset_loader, :build_image_io]
+
+task :doc do
+  src_list = Dir["lib/dnn/core/*.rb"]
+  src_list += Dir["lib/dnn/*.rb"]
+  sh "yardoc #{src_list.join(' ')}"
+end

data/examples/cifar100_example.rb

@@ -0,0 +1,71 @@
+require "dnn"
+require "dnn/cifar100"
+# If you use numo/linalg then please uncomment out.
+# require "numo/linalg/autoloader"
+
+include DNN::Layers
+include DNN::Activations
+include DNN::Optimizers
+include DNN::Losses
+Model = DNN::Model
+CIFAR100 = DNN::CIFAR100
+
+x_train, y_train = CIFAR100.load_train
+x_test, y_test = CIFAR100.load_test
+
+x_train = Numo::SFloat.cast(x_train)
+x_test = Numo::SFloat.cast(x_test)
+
+x_train /= 255
+x_test /= 255
+
+y_train = y_train[true, 1]
+y_test = y_test[true, 1]
+
+y_train = DNN::Utils.to_categorical(y_train, 100, Numo::SFloat)
+y_test = DNN::Utils.to_categorical(y_test, 100, Numo::SFloat)
+
+model = Model.new
+
+model << InputLayer.new([32, 32, 3])
+
+model << Conv2D.new(16, 5, padding: true)
+model << BatchNormalization.new
+model << ReLU.new
+
+model << Conv2D.new(16, 5, padding: true)
+model << BatchNormalization.new
+model << ReLU.new
+
+model << MaxPool2D.new(2)
+
+model << Conv2D.new(32, 5, padding: true)
+model << BatchNormalization.new
+model << ReLU.new
+
+model << Conv2D.new(32, 5, padding: true)
+model << BatchNormalization.new
+model << ReLU.new
+
+model << MaxPool2D.new(2)
+
+model << Conv2D.new(64, 5, padding: true)
+model << BatchNormalization.new
+model << ReLU.new
+
+model << Conv2D.new(64, 5, padding: true)
+model << BatchNormalization.new
+model << ReLU.new
+
+model << Flatten.new
+
+model << Dense.new(1024)
+model << BatchNormalization.new
+model << ReLU.new
+model << Dropout.new(0.5)
+
+model << Dense.new(100)
+
+model.compile(Adam.new, SoftmaxCrossEntropy.new)
+
+model.train(x_train, y_train, 10, batch_size: 100, test: [x_test, y_test])

data/examples/cifar10_example.rb

@@ -1,5 +1,6 @@
 require "dnn"
-require "dnn/lib/cifar10"
+require "dnn/cifar10"
+# If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
 include DNN::Layers

data/examples/iris_example.rb

@@ -1,5 +1,6 @@
 require "dnn"
-require "dnn/lib/iris"
+require "dnn/iris"
+# If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
 include DNN::Layers

data/examples/mnist_conv2d_example.rb

@@ -1,5 +1,6 @@
 require "dnn"
-require "dnn/lib/mnist"
+require "dnn/mnist"
+# If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
 include DNN::Layers

data/examples/mnist_example.rb

@@ -1,5 +1,6 @@
 require "dnn"
-require "dnn/lib/mnist"
+require "dnn/mnist"
+# If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
 include DNN::Layers
@@ -26,11 +27,9 @@ model = Model.new
 model << InputLayer.new(784)
 
 model << Dense.new(256)
-model << BatchNormalization.new
 model << ReLU.new
 
 model << Dense.new(256)
-model << BatchNormalization.new
 model << ReLU.new
 
 model << Dense.new(10)

data/examples/mnist_lstm_example.rb

@@ -1,5 +1,6 @@
 require "dnn"
-require "dnn/lib/mnist"
+require "dnn/mnist"
+# If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
 include DNN::Layers

data/ext/cifar_loader/cifar_loader.c

@@ -0,0 +1,77 @@
+#include <ruby.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define CIFAR_WIDTH 32
+#define CIFAR_HEIGHT 32
+#define CIFAR_CHANNEL 3
+
+static VALUE cifar10_load_binary(VALUE self, VALUE rb_bin, VALUE rb_num_datas) {
+  uint8_t* bin = (uint8_t*)StringValuePtr(rb_bin);
+  int32_t num_datas = FIX2INT(rb_num_datas);
+  VALUE rb_x_bin;
+  VALUE rb_y_bin;
+  int32_t i;
+  int32_t j = 0;
+  int32_t k = 0;
+  int32_t size = CIFAR_WIDTH * CIFAR_HEIGHT * CIFAR_CHANNEL;
+  int32_t x_bin_size = num_datas * size;
+  int32_t y_bin_size = num_datas;
+  uint8_t* x_bin;
+  uint8_t* y_bin;
+
+  x_bin = (uint8_t*)malloc(x_bin_size);
+  y_bin = (uint8_t*)malloc(y_bin_size);
+  for (i = 0; i < num_datas; i++) {
+    y_bin[i] = bin[j];
+    j++;
+    memcpy(&x_bin[k], &bin[j], size);
+    j += size;
+    k += size;
+  }
+  rb_x_bin = rb_str_new((char*)x_bin, x_bin_size);
+  rb_y_bin = rb_str_new((char*)y_bin, y_bin_size);
+  free(x_bin);
+  free(y_bin);
+  return rb_ary_new3(2, rb_x_bin, rb_y_bin);
+}
+
+static VALUE cifar100_load_binary(VALUE self, VALUE rb_bin, VALUE rb_num_datas) {
+  uint8_t* bin = (uint8_t*)StringValuePtr(rb_bin);
+  int32_t num_datas = FIX2INT(rb_num_datas);
+  VALUE rb_x_bin;
+  VALUE rb_y_bin;
+  int32_t i;
+  int32_t j = 0;
+  int32_t k = 0;
+  int32_t size = CIFAR_WIDTH * CIFAR_HEIGHT * CIFAR_CHANNEL;
+  int32_t x_bin_size = num_datas * size;
+  int32_t y_bin_size = num_datas * 2;
+  uint8_t* x_bin;
+  uint8_t* y_bin;
+
+  x_bin = (uint8_t*)malloc(x_bin_size);
+  y_bin = (uint8_t*)malloc(y_bin_size);
+  for (i = 0; i < num_datas; i++) {
+    y_bin[i * 2] = bin[j];
+    y_bin[i * 2 + 1] = bin[j + 1];
+    j += 2;
+    memcpy(&x_bin[k], &bin[j], size);
+    j += size;
+    k += size;
+  }
+  rb_x_bin = rb_str_new((char*)x_bin, x_bin_size);
+  rb_y_bin = rb_str_new((char*)y_bin, y_bin_size);
+  free(x_bin);
+  free(y_bin);
+  return rb_ary_new3(2, rb_x_bin, rb_y_bin);
+}
+
+void Init_cifar_loader() {
+  VALUE rb_dnn = rb_define_module("DNN");
+  VALUE rb_cifar10 = rb_define_module_under(rb_dnn, "CIFAR10");
+  VALUE rb_cifar100 = rb_define_module_under(rb_dnn, "CIFAR100");
+
+  rb_define_singleton_method(rb_cifar10, "load_binary", cifar10_load_binary, 2);
+  rb_define_singleton_method(rb_cifar100, "load_binary", cifar100_load_binary, 2);
+}

data/ext/cifar_loader/extconf.rb

@@ -0,0 +1,3 @@
+require "mkmf"
+
+create_makefile("cifar_loader")

data/lib/dnn.rb

@@ -16,6 +16,7 @@ require_relative "dnn/core/param"
 require_relative "dnn/core/dataset"
 require_relative "dnn/core/initializers"
 require_relative "dnn/core/layers"
+require_relative "dnn/core/normalizations"
 require_relative "dnn/core/activations"
 require_relative "dnn/core/losses"
 require_relative "dnn/core/regularizers"

data/lib/dnn/{lib/cifar10.rb → cifar10.rb}

@@ -1,24 +1,22 @@
 require "zlib"
 require "archive/tar/minitar"
-require_relative "../../../ext/cifar10_loader/cifar10_loader"
+require_relative "../../ext/cifar_loader/cifar_loader"
 require_relative "downloader"
 
 URL_CIFAR10 = "https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz"
-CIFAR10_DIR = "cifar-10-batches-bin"
+DIR_CIFAR10 = "cifar-10-batches-bin"
 
 module DNN
   module CIFAR10
     class DNN_CIFAR10_LoadError < DNN_Error; end
 
-    private_class_method :load_binary
-
     def self.downloads
-      return if Dir.exist?(__dir__ + "/" + CIFAR10_DIR)
+      return if Dir.exist?(__dir__ + "/downloads/" + DIR_CIFAR10)
       Downloader.download(URL_CIFAR10)
-      cifar10_binary_file_name = __dir__ + "/" + URL_CIFAR10.match(%r`.+/(.+)`)[1]
+      cifar10_binary_file_name = __dir__ + "/downloads/" + URL_CIFAR10.match(%r`.+/(.+)`)[1]
       begin
         Zlib::GzipReader.open(cifar10_binary_file_name) do |gz|
-          Archive::Tar::Minitar::unpack(gz, __dir__)
+          Archive::Tar::Minitar::unpack(gz, __dir__ + "/downloads")
         end
       ensure
         File.unlink(cifar10_binary_file_name)
@@ -29,11 +27,11 @@ module DNN
       downloads
       bin = ""
       (1..5).each do |i|
-        fname = __dir__ + "/#{CIFAR10_DIR}/data_batch_#{i}.bin"
+        fname = __dir__ + "/downloads/#{DIR_CIFAR10}/data_batch_#{i}.bin"
         raise DNN_CIFAR10_LoadError.new(%`file "#{fname}" is not found.`) unless File.exist?(fname)
         bin << File.binread(fname)
       end
-      x_bin, y_bin = load_binary(bin, 50000)
+      x_bin, y_bin = CIFAR10.load_binary(bin, 50000)
       x_train = Numo::UInt8.from_binary(x_bin).reshape(50000, 3, 32, 32).transpose(0, 2, 3, 1).clone
       y_train = Numo::UInt8.from_binary(y_bin)
       [x_train, y_train]
@@ -41,10 +39,10 @@ module DNN
 
     def self.load_test
       downloads
-      fname = __dir__ + "/#{CIFAR10_DIR}/test_batch.bin"
+      fname = __dir__ + "/downloads/#{DIR_CIFAR10}/test_batch.bin"
       raise DNN_CIFAR10_LoadError.new(%`file "#{fname}" is not found.`) unless File.exist?(fname)
       bin = File.binread(fname)
-      x_bin, y_bin = load_binary(bin, 10000)
+      x_bin, y_bin = CIFAR10.load_binary(bin, 10000)
       x_test = Numo::UInt8.from_binary(x_bin).reshape(10000, 3, 32, 32).transpose(0, 2, 3, 1).clone
       y_test = Numo::UInt8.from_binary(y_bin)
       [x_test, y_test]

data/lib/dnn/cifar100.rb

@@ -0,0 +1,49 @@
+require "zlib"
+require "archive/tar/minitar"
+require_relative "../../ext/cifar_loader/cifar_loader"
+require_relative "downloader"
+
+URL_CIFAR100 = "https://www.cs.toronto.edu/~kriz/cifar-100-binary.tar.gz"
+DIR_CIFAR100 = "cifar-100-binary"
+
+module DNN
+  module CIFAR100
+    class DNN_CIFAR100_LoadError < DNN_Error; end
+
+    def self.downloads
+      return if Dir.exist?(__dir__ + "/downloads/" + DIR_CIFAR100)
+      Downloader.download(URL_CIFAR100)
+      cifar100_binary_file_name = __dir__ + "/downloads/" + URL_CIFAR100.match(%r`.+/(.+)`)[1]
+      begin
+        Zlib::GzipReader.open(cifar100_binary_file_name) do |gz|
+          Archive::Tar::Minitar::unpack(gz, __dir__ + "/downloads")
+        end
+      ensure
+        File.unlink(cifar100_binary_file_name)
+      end
+    end
+
+    def self.load_train
+      downloads
+      bin = ""
+      fname = __dir__ + "/downloads/#{DIR_CIFAR100}/train.bin"
+      raise DNN_CIFAR100_LoadError.new(%`file "#{fname}" is not found.`) unless File.exist?(fname)
+      bin << File.binread(fname)
+      x_bin, y_bin = CIFAR100.load_binary(bin, 50000)
+      x_train = Numo::UInt8.from_binary(x_bin).reshape(50000, 3, 32, 32).transpose(0, 2, 3, 1).clone
+      y_train = Numo::UInt8.from_binary(y_bin).reshape(50000, 2)
+      [x_train, y_train]
+    end
+
+    def self.load_test
+      downloads
+      fname = __dir__ + "/downloads/#{DIR_CIFAR100}/test.bin"
+      raise DNN_CIFAR100_LoadError.new(%`file "#{fname}" is not found.`) unless File.exist?(fname)
+      bin = File.binread(fname)
+      x_bin, y_bin = CIFAR100.load_binary(bin, 10000)
+      x_test = Numo::UInt8.from_binary(x_bin).reshape(10000, 3, 32, 32).transpose(0, 2, 3, 1).clone
+      y_test = Numo::UInt8.from_binary(y_bin).reshape(10000, 2)
+      [x_test, y_test]
+    end
+  end
+end

data/lib/dnn/core/activations.rb

@@ -3,22 +3,22 @@ module DNN
 
     class Sigmoid < Layers::Layer
       def forward(x)
-        @out = 1 / (1 + NMath.exp(-x))
+        @y = 1 / (1 + NMath.exp(-x))
       end
     
-      def backward(dout)
-        dout * (1 - @out) * @out
+      def backward(dy)
+        dy * (1 - @y) * @y
       end
     end
 
 
     class Tanh < Layers::Layer
       def forward(x)
-        @out = NMath.tanh(x)
+        @y = NMath.tanh(x)
       end
       
-      def backward(dout)
-        dout * (1 - @out**2)
+      def backward(dy)
+        dy * (1 - @y**2)
       end
     end
 
@@ -29,8 +29,8 @@ module DNN
         x / (1 + x.abs)
       end
 
-      def backward(dout)
-        dout * (1 / (1 + @x.abs)**2)
+      def backward(dy)
+        dy * (1 / (1 + @x.abs)**2)
       end
     end
 
@@ -41,8 +41,8 @@ module DNN
         NMath.log(1 + NMath.exp(x))
       end
 
-      def backward(dout)
-        dout * (1 / (1 + NMath.exp(-@x)))
+      def backward(dy)
+        dy * (1 / (1 + NMath.exp(-@x)))
       end
     end
 
@@ -50,11 +50,11 @@ module DNN
     class Swish < Layers::Layer
       def forward(x)
         @x = x
-        @out = x * (1 / (1 + NMath.exp(-x)))
+        @y = x * (1 / (1 + NMath.exp(-x)))
       end
     
-      def backward(dout)
-        dout * (@out + (1 / (1 + NMath.exp(-@x))) * (1 - @out))
+      def backward(dy)
+        dy * (@y + (1 / (1 + NMath.exp(-@x))) * (1 - @y))
       end
     end
     
@@ -66,23 +66,25 @@ module DNN
         x
       end
     
-      def backward(dout)
+      def backward(dy)
         @x[@x > 0] = 1
         @x[@x <= 0] = 0
-        dout * @x
+        dy * @x
       end
     end
 
 
     class LeakyReLU < Layers::Layer
+      # @return [Float] Return the alpha value.
       attr_reader :alpha
 
-      def initialize(alpha = 0.3)
-        @alpha = alpha
+      def self.from_hash(hash)
+        self.new(hash[:alpha])
       end
 
-      def self.load_hash(hash)
-        self.new(hash[:alpha])
+      # @param [Float] alpha The slope when the output value is negative.
+      def initialize(alpha = 0.3)
+        @alpha = alpha
       end
 
       def forward(x)
@@ -92,10 +94,10 @@ module DNN
         x * a
       end
 
-      def backward(dout)
+      def backward(dy)
         @x[@x > 0] = 1
         @x[@x <= 0] = @alpha
-        dout * @x
+        dy * @x
       end
 
       def to_hash
@@ -105,12 +107,14 @@ module DNN
 
 
     class ELU < Layers::Layer
+      # @return [Float] Return the alpha value.
       attr_reader :alpha
 
-      def self.load_hash(hash)
+      def self.from_hash(hash)
         self.new(hash[:alpha])
       end
 
+      # @param [Float] alpha The slope when the output value is negative.
       def initialize(alpha = 1.0)
         @alpha = alpha
       end
@@ -126,13 +130,13 @@ module DNN
         x1 + x2
       end
 
-      def backward(dout)
+      def backward(dy)
         dx = Xumo::SFloat.ones(@x.shape)
         dx[@x < 0] = 0
         dx2 = Xumo::SFloat.zeros(@x.shape)
         dx2[@x < 0] = 1
         dx2 *= @alpha * NMath.exp(@x)
-        dout * (dx + dx2)
+        dy * (dx + dx2)
       end
 
       def to_hash