ruby-dnn 0.14.3 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e4633ce695dc370e62c6653d5ff07d81f9306625f7165750f8e455c4a28ca61
-  data.tar.gz: e4275720191b14592e2fb93581e9826e437ce9fcc75f6473cb963892d8f9f893
+  metadata.gz: 6be46c1c89fbb1f1f2091c95f15ccab21473d52571ee47953a4c07446ea8544c
+  data.tar.gz: e01eb929c13b2a33a350ee42b8bb8584446722ac3ba3dc7e163f7f3429ec068a
 SHA512:
-  metadata.gz: d9831b1b3a73423742bce7f989a13c7ac3d63426a2bd8366143be1717a1e90fbdf6aa79a9e90508e193e00fbcd120942b7ccd0c8b6b13eb9cc4e237471712030
-  data.tar.gz: 60344f9eb3645b0560dd9d1f63a86dcc554f7c9b923a3b84a0bc301264120d50ca034d2887c1cc41989f223891ace4d64c570c9935e2b3f6f467a05f0fab910f
+  metadata.gz: cc5bd6608d17a90cb97f6f5530877e5441a2dc73fc18e7b821c33d606469d6a15c22d519570b8c8821282915de84b677b3527221b1eb076a145a85997a58d2d9
+  data.tar.gz: 721c578f41a7648dfc5c462a1b9a81f6b9604b50540e8fa4a58bb801adc8059ee7cff9f27d3c1b1a710ce0857e426ca2087f4902bf46d9c621870e3ae7cb664a

data/README.md

@@ -2,8 +2,9 @@
 [![Gem Version](https://badge.fury.io/rb/ruby-dnn.svg)](https://badge.fury.io/rb/ruby-dnn)  
 [![Build Status](https://travis-ci.org/unagiootoro/ruby-dnn.svg?branch=master)](https://travis-ci.org/unagiootoro/ruby-dnn)
 
-ruby-dnn is a ruby deep learning library. This library supports full connected neural network and convolution neural network.
-Currently, you can get 99% accuracy with MNIST and 74% with CIFAR 10.
+ruby-dnn is a ruby deep learning library. This library supports full connected neural network and convolution neural network
+and recurrent neural network.
+Currently, you can get 99% accuracy with MNIST and 78% with CIFAR 10.
 
 ## Installation
 
@@ -79,8 +80,9 @@ If you want to know more detailed information, please refer to the source code.
 || Implemented classes |
 |:-----------|------------:|
 | Connections | Dense, Conv2D, Conv2DTranspose, Embedding, SimpleRNN, LSTM, GRU |
-| Layers | Flatten, Reshape, Dropout, BatchNormalization, MaxPool2D, AvgPool2D, UnPool2D |
 | Activations | Sigmoid, Tanh, Softsign, Softplus, Swish, ReLU, LeakyReLU, ELU |
+| Basic | Flatten, Reshape, Dropout, BatchNormalization |
+| Pooling | MaxPool2D, AvgPool2D, GlobalAvgPool2D, UnPool2D |
 | Optimizers | SGD, Nesterov, AdaGrad, RMSProp, AdaDelta, RMSPropGraves, Adam, AdaBound |
 | Losses | MeanSquaredError, MeanAbsoluteError, Hinge, HuberLoss, SoftmaxCrossEntropy, SigmoidCrossEntropy |
 

data/Rakefile

@@ -5,7 +5,7 @@ Rake::TestTask.new(:test) do |t|
   t.libs << "test"
   t.libs << "ext"
   t.libs << "lib"
-  t.test_files = FileList["test/*_test.rb"]
+  t.test_files = FileList["test/*_test.rb", "test/layers_test/*_test.rb"]
 end
 
 task :build_cifar_loader do
@@ -27,7 +27,9 @@ end
 task :default => [:test, :build_cifar_loader, :build_rb_stb_image]
 
 task :doc do
-  src_list = Dir["lib/dnn/core/*.rb"]
+  src_list = Dir["lib/dnn.rb"]
+  src_list += Dir["lib/dnn/core/*.rb"]
+  src_list += Dir["lib/dnn/core/layers/*.rb"]
   src_list += Dir["lib/dnn/*.rb"]
   sh "yardoc #{src_list.join(' ')}"
 end

data/examples/api-examples/save_example.rb

@@ -20,9 +20,9 @@ BATCH_SIZE = 128
 # Select save style from USE_MARSHAL or USE_JSON.
 SAVE_STYLE = USE_MARSHAL
 
-# When set a true, save data included optimizer status.
+# When set a true, save data included model structure.
 # This setting is enabled when SAVE_STYLE is USE_MARSHAL.
-INCLUDE_OPTIMIZER = false
+INCLUDE_MODEL = true
 
 x_train, y_train = MNIST.load_train
 x_test, y_test = MNIST.load_test
@@ -47,7 +47,7 @@ class MLP < Model
   end
 
   def call(x)
-    x = InputLayer.(x)
+    x = InputLayer.new(784).(x)
     x = @l1.(x)
     x = @bn1.(x)
     x = ReLU.(x)
@@ -64,7 +64,7 @@ model.setup(Adam.new, SoftmaxCrossEntropy.new)
 model.train(x_train, y_train, EPOCHS, batch_size: BATCH_SIZE, test: [x_test, y_test])
 
 if SAVE_STYLE == USE_MARSHAL
-  saver = MarshalSaver.new(model, include_optimizer: INCLUDE_OPTIMIZER)
+  saver = MarshalSaver.new(model, include_model: INCLUDE_MODEL)
   saver.save("trained_mnist.marshal")
   # model.save("trained_mnist.marshal") # This code is equivalent to the code above.
 elsif SAVE_STYLE == USE_JSON
@@ -73,6 +73,8 @@ elsif SAVE_STYLE == USE_JSON
 end
 
 model2 = MLP.new
+model2.setup(Adam.new, SoftmaxCrossEntropy.new)
+model2.predict1(Numo::SFloat.zeros(784))
 if SAVE_STYLE == USE_MARSHAL
   loader = MarshalLoader.new(model2)
   loader.load("trained_mnist.marshal")
@@ -82,4 +84,4 @@ elsif SAVE_STYLE == USE_JSON
   loader.load("trained_mnist.json")
 end
 
-puts model2.accuracy(x_test, y_test)
+puts model2.evaluate(x_test, y_test)

data/examples/dcgan/imgen.rb

@@ -8,13 +8,8 @@ Image = DNN::Image
 
 batch_size = 100
 
-gen = Generator.new
-dis = Discriminator.new
-dcgan = DCGAN.new(gen, dis)
-dcgan.predict1(Numo::SFloat.zeros(20))
-
-loader = MarshalLoader.new(dcgan)
-loader.load("trained/dcgan_model_epoch20.marshal")
+dcgan = DCGAN.load("trained/dcgan_model_epoch20.marshal")
+gen = dcgan.gen
 
 Numo::SFloat.srand(rand(1 << 31))
 noise = Numo::SFloat.new(batch_size, 20).rand(-1, 1)

data/examples/dcgan/train.rb

@@ -20,7 +20,6 @@ dcgan = DCGAN.new(gen, dis)
 dis.setup(Adam.new(alpha: 0.00001, beta1: 0.1), SigmoidCrossEntropy.new)
 dcgan.setup(Adam.new(alpha: 0.0002, beta1: 0.5), SigmoidCrossEntropy.new)
 dcgan.add_callback(CheckPoint.new("trained/dcgan_model"))
-dcgan.predict1(Numo::SFloat.zeros(20))
 
 x_train, * = MNIST.load_train
 x_train = Numo::SFloat.cast(x_train)

data/lib/dnn.rb

@@ -11,21 +11,21 @@ require_relative "dnn/version"
 require_relative "dnn/core/error"
 require_relative "dnn/core/global"
 require_relative "dnn/core/tensor"
-require_relative "dnn/core/models"
 require_relative "dnn/core/param"
 require_relative "dnn/core/link"
 require_relative "dnn/core/iterator"
-require_relative "dnn/core/initializers"
-require_relative "dnn/core/layers"
-require_relative "dnn/core/normalizations"
-require_relative "dnn/core/activations"
-require_relative "dnn/core/merge_layers"
+require_relative "dnn/core/models"
+require_relative "dnn/core/layers/basic_layers"
+require_relative "dnn/core/layers/normalizations"
+require_relative "dnn/core/layers/activations"
+require_relative "dnn/core/layers/merge_layers"
+require_relative "dnn/core/layers/cnn_layers"
+require_relative "dnn/core/layers/embedding"
+require_relative "dnn/core/layers/rnn_layers"
+require_relative "dnn/core/optimizers"
 require_relative "dnn/core/losses"
+require_relative "dnn/core/initializers"
 require_relative "dnn/core/regularizers"
-require_relative "dnn/core/cnn_layers"
-require_relative "dnn/core/embedding"
-require_relative "dnn/core/rnn_layers"
-require_relative "dnn/core/optimizers"
 require_relative "dnn/core/callbacks"
 require_relative "dnn/core/savers"
 require_relative "dnn/core/utils"

data/lib/dnn/core/callbacks.rb

@@ -35,13 +35,17 @@ module DNN
     end
 
     # A callback that save the model at the after of the epoch.
+    # @param [String] base_file_name Base file name for saving.
+    # @param [Boolean] include_model When set a true, save data included model structure.
     class CheckPoint < Callback
-      def initialize(base_file_name)
+      def initialize(base_file_name, include_model: true)
         @base_file_name = base_file_name
+        @include_model = include_model
       end
 
       def after_epoch
-        model.save(@base_file_name + "_epoch#{model.last_log[:epoch]}.marshal")
+        saver = Savers::MarshalSaver.new(@model, include_model: @include_model)
+        saver.save(@base_file_name + "_epoch#{model.last_log[:epoch]}.marshal")
       end
     end
 

data/lib/dnn/core/iterator.rb

@@ -4,8 +4,8 @@ module DNN
     attr_reader :num_datas
     attr_reader :last_round_down
 
-    # @param [Numo::SFloat] x_datas input datas.
-    # @param [Numo::SFloat] y_datas output datas.
+    # @param [Numo::SFloat | Array] x_datas input datas.
+    # @param [Numo::SFloat | Array] y_datas output datas.
     # @param [Boolean] random Set true to return batches randomly. Setting false returns batches in order of index.
     # @param [Boolean] last_round_down Set true to round down for last batch data when call foreach.
     def initialize(x_datas, y_datas, random: true, last_round_down: false)
@@ -19,6 +19,7 @@ module DNN
 
     # Return the next batch.
     # @param [Integer] batch_size Required batch size.
+    # @return [Array] Returns the mini batch in the form [x_batch, y_batch].
     def next_batch(batch_size)
       raise DNN_Error, "This iterator has not next batch. Please call reset." unless has_next?
       if @indexes.length <= batch_size
@@ -27,6 +28,13 @@ module DNN
       else
         batch_indexes = @indexes.shift(batch_size)
       end
+      get_batch(batch_indexes)
+    end
+
+    # Implement a process to get mini batch.
+    # @param [Array] batch_indexes Index of batch to get.
+    # @return [Array] Returns the mini batch in the form [x_batch, y_batch].
+    private def get_batch(batch_indexes)
       x_batch = if @x_datas.is_a?(Array)
                   @x_datas.map { |datas| datas[batch_indexes, false] }
                 else

data/lib/dnn/core/{layers.rb → layers/basic_layers.rb}

@@ -3,7 +3,6 @@ module DNN
 
     # Super class of all layer classes.
     class Layer
-      attr_accessor :name
       attr_reader :input_shape
 
       def self.call(x, *args)
@@ -16,13 +15,11 @@ module DNN
         layer = layer_class.allocate
         raise DNN_Error, "#{layer.class} is not an instance of #{self} class." unless layer.is_a?(self)
         layer.load_hash(hash)
-        layer.name = hash[:name]&.to_sym
         layer
       end
 
       def initialize
         @built = false
-        @name = nil
       end
 
       # Forward propagation and create a link.
@@ -70,7 +67,7 @@ module DNN
 
       # Layer to a hash.
       def to_hash(merge_hash = nil)
-        hash = { class: self.class.name, name: @name }
+        hash = { class: self.class.name }
         hash.merge!(merge_hash) if merge_hash
         hash
       end
@@ -78,10 +75,20 @@ module DNN
       def load_hash(hash)
         initialize
       end
+
+      def clean
+        input_shape = @input_shape
+        hash = to_hash
+        instance_variables.each do |ivar|
+          instance_variable_set(ivar, nil)
+        end
+        load_hash(hash)
+        build(input_shape)
+      end
     end
 
     # This class is a superclass of all classes with learning parameters.
-    class HasParamLayer < Layer
+    class TrainableLayer < Layer
       # @return [Boolean] Setting false prevents learning of parameters.
       attr_accessor :trainable
 
@@ -94,6 +101,22 @@ module DNN
       def get_params
         raise NotImplementedError, "Class '#{self.class.name}' has implement method 'get_params'"
       end
+
+      def clean
+        input_shape = @input_shape
+        hash = to_hash
+        params = get_params
+        instance_variables.each do |ivar|
+          instance_variable_set(ivar, nil)
+        end
+        load_hash(hash)
+        build(input_shape)
+        params.each do |(key, param)|
+          param.data = nil
+          param.grad = Xumo::SFloat[0] if param.grad
+          instance_variable_set("@#{key}", param)
+        end
+      end
     end
 
     class InputLayer < Layer
@@ -109,7 +132,7 @@ module DNN
       end
 
       def call(input)
-        build unless built?
+        build(@input_shape) unless built?
         if input.is_a?(Tensor)
           x = input.data
           prev_link = input&.link
@@ -120,7 +143,7 @@ module DNN
         Tensor.new(forward(x), Link.new(prev_link, self))
       end
 
-      def build
+      def build(input_shape)
         @built = true
       end
 
@@ -163,7 +186,7 @@ module DNN
     end
 
     # It is a superclass of all connection layers.
-    class Connection < HasParamLayer
+    class Connection < TrainableLayer
       attr_reader :weight
       attr_reader :bias
       attr_reader :weight_initializer

data/lib/dnn/core/{embedding.rb → layers/embedding.rb}

@@ -1,7 +1,7 @@
 module DNN
   module Layers
 
-    class Embedding < HasParamLayer
+    class Embedding < TrainableLayer
       attr_reader :input_length
       attr_reader :weight
       attr_reader :weight_initializer
@@ -19,16 +19,17 @@ module DNN
         @input_length = input_length
         @weight_initializer = weight_initializer
         @weight_regularizer = weight_regularizer
+        @weight = Param.new(nil, Xumo::SFloat[0])
       end
 
       def call(input_tensor)
-        build unless built?
+        build(@input_shape) unless built?
         Tensor.new(forward(input_tensor.data), Link.new(nil, self))
       end
 
-      def build
+      def build(input_shape)
         @built = true
-        @weight = Param.new(Xumo::SFloat.new(@input_length), Xumo::SFloat[0])
+        @weight.data = Xumo::SFloat.new(@input_length)
         @weight_initializer.init_param(self, @weight)
         @weight_regularizer.param = @weight if @weight_regularizer
       end

data/lib/dnn/core/{merge_layers.rb → layers/merge_layers.rb}

@@ -1,5 +1,5 @@
 module DNN
-  module MergeLayers
+  module Layers
 
     class MergeLayer < Layers::Layer
       def self.call(x1, x2, *args)

data/lib/dnn/core/{normalizations.rb → layers/normalizations.rb}

@@ -1,7 +1,7 @@
 module DNN
   module Layers
 
-    class BatchNormalization < HasParamLayer
+    class BatchNormalization < TrainableLayer
       attr_reader :gamma
       attr_reader :beta
       attr_reader :running_mean
@@ -18,14 +18,18 @@ module DNN
         @axis = axis
         @momentum = momentum
         @eps = eps
+        @gamma = Param.new(nil, Xumo::SFloat[0])
+        @beta = Param.new(nil, Xumo::SFloat[0])
+        @running_mean = Param.new
+        @running_var = Param.new
       end
 
       def build(input_shape)
         super
-        @gamma = Param.new(Xumo::SFloat.ones(*output_shape), Xumo::SFloat[0])
-        @beta = Param.new(Xumo::SFloat.zeros(*output_shape), Xumo::SFloat[0])
-        @running_mean = Param.new(Xumo::SFloat.zeros(*output_shape))
-        @running_var = Param.new(Xumo::SFloat.zeros(*output_shape))
+        @gamma.data = Xumo::SFloat.ones(*output_shape)
+        @beta.data = Xumo::SFloat.zeros(*output_shape)
+        @running_mean.data = Xumo::SFloat.zeros(*output_shape)
+        @running_var.data = Xumo::SFloat.zeros(*output_shape)
       end
 
       def forward(x)

data/lib/dnn/core/{rnn_layers.rb → layers/rnn_layers.rb}

@@ -31,7 +31,7 @@ module DNN
         @num_nodes = num_nodes
         @stateful = stateful
         @return_sequences = return_sequences
-        @layers = []
+        @hidden_layers = []
         @hidden = Param.new
         @recurrent_weight = Param.new(nil, Xumo::SFloat[0])
         @recurrent_weight_initializer = recurrent_weight_initializer
@@ -46,14 +46,19 @@ module DNN
         @time_length = @input_shape[0]
       end
 
+      private def create_hidden_layer
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'create_hidden_layer'"
+      end
+
       def forward(xs)
+        create_hidden_layer
         @xs_shape = xs.shape
         hs = Xumo::SFloat.zeros(xs.shape[0], @time_length, @num_nodes)
         h = @stateful && @hidden.data ? @hidden.data : Xumo::SFloat.zeros(xs.shape[0], @num_nodes)
         xs.shape[1].times do |t|
           x = xs[true, t, false]
-          @layers[t].trainable = @trainable
-          h = @layers[t].forward(x, h)
+          @hidden_layers[t].trainable = @trainable
+          h = @hidden_layers[t].forward(x, h)
           hs[true, t, false] = h
         end
         @hidden.data = h
@@ -70,7 +75,7 @@ module DNN
         dh = 0
         (dh2s.shape[1] - 1).downto(0) do |t|
           dh2 = dh2s[true, t, false]
-          dx, dh = @layers[t].backward(dh2 + dh)
+          dx, dh = @hidden_layers[t].backward(dh2 + dh)
           dxs[true, t, false] = dx
         end
         dxs
@@ -196,9 +201,10 @@ module DNN
         @recurrent_weight.data = Xumo::SFloat.new(@num_nodes, @num_nodes)
         @bias.data = Xumo::SFloat.new(@num_nodes) if @bias
         init_weight_and_bias
-        @time_length.times do
-          @layers << SimpleRNNDense.new(@weight, @recurrent_weight, @bias, @activation)
-        end
+      end
+
+      def create_hidden_layer
+        @hidden_layers = Array.new(@time_length) { SimpleRNNDense.new(@weight, @recurrent_weight, @bias, @activation) }
       end
 
       def to_hash
@@ -301,12 +307,14 @@ module DNN
         @recurrent_weight.data = Xumo::SFloat.new(@num_nodes, @num_nodes * 4)
         @bias.data = Xumo::SFloat.new(@num_nodes * 4) if @bias
         init_weight_and_bias
-        @time_length.times do
-          @layers << LSTMDense.new(@weight, @recurrent_weight, @bias)
-        end
+      end
+
+      def create_hidden_layer
+        @hidden_layers = Array.new(@time_length) { LSTMDense.new(@weight, @recurrent_weight, @bias) }
       end
 
       def forward(xs)
+        create_hidden_layer
         @xs_shape = xs.shape
         hs = Xumo::SFloat.zeros(xs.shape[0], @time_length, @num_nodes)
         h = nil
@@ -319,8 +327,8 @@ module DNN
         c ||= Xumo::SFloat.zeros(xs.shape[0], @num_nodes)
         xs.shape[1].times do |t|
           x = xs[true, t, false]
-          @layers[t].trainable = @trainable
-          h, c = @layers[t].forward(x, h, c)
+          @hidden_layers[t].trainable = @trainable
+          h, c = @hidden_layers[t].forward(x, h, c)
           hs[true, t, false] = h
         end
         @hidden.data = h
@@ -339,7 +347,7 @@ module DNN
         dc = 0
         (dh2s.shape[1] - 1).downto(0) do |t|
           dh2 = dh2s[true, t, false]
-          dx, dh, dc = @layers[t].backward(dh2 + dh, dc)
+          dx, dh, dc = @hidden_layers[t].backward(dh2 + dh, dc)
           dxs[true, t, false] = dx
         end
         dxs
@@ -444,9 +452,10 @@ module DNN
         @recurrent_weight.data = Xumo::SFloat.new(@num_nodes, @num_nodes * 3)
         @bias.data = Xumo::SFloat.new(@num_nodes * 3) if @bias
         init_weight_and_bias
-        @time_length.times do
-          @layers << GRUDense.new(@weight, @recurrent_weight, @bias)
-        end
+      end
+
+      def create_hidden_layer
+        @hidden_layers = Array.new(@time_length) { GRUDense.new(@weight, @recurrent_weight, @bias) }
       end
     end