ruby-dnn 1.2.2 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2aa12b717ef532b8afe44de7cb388c7d87cb271bf38a25adaad2c335c8817d4b
-  data.tar.gz: e1322f86b06c11ac3728e948e18469ccc6e454eeefb180108a0a9bfc2dbd8143
+  metadata.gz: df187618941592ff119bab49757f039544d806a6dab5b8c9962040b714a7873e
+  data.tar.gz: 263c5a9d1d366ad4782c0fa30b130a081edae1d6b744daf0b2f359560c83e0fd
 SHA512:
-  metadata.gz: 6c9c53ca73a5ab7fc53935f37e0804d761e36735aa06972f63e151ce76a44f01878a2c01dfe0622bf39baa5c98b16ee8d06db212356fcc329a9dd40ae2d78f1c
-  data.tar.gz: 5a195c0afd677127afad2433df2fda3a09c711464c46cb174713d00d695026d2b86faa64f82cc96f2b417b468fc372c04efee22761800c900e2f6b0ca05ac57d
+  metadata.gz: f2b53307f3a90d6fa3caaa01a9f57e7ea3d76a378fa90ecaba4c4e46f052fc1380718a89f066eba9df28a822b49236f10e03c65bb113004b523017412e5cbbf2
+  data.tar.gz: fc6d85c0f8de928f97fad6debf480fa683cad780d70855bedbe8ed989ad31efc92cb60d55a6e456107f76093d461ddfc328c806b51f69095ec08111632438191

data/Rakefile

@@ -1,5 +1,6 @@
 require "bundler/gem_tasks"
 require "rake/testtask"
+require "rake/extensiontask"
 require "yard"
 require "yard/rake/yardoc_task"
 
@@ -10,6 +11,10 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/*_test.rb", "test/layers_test/*_test.rb"]
 end
 
+Rake::ExtensionTask.new "rb_stb_image" do |ext|
+  ext.lib_dir = "lib/rb_stb_image"
+end
+
 task :build_rb_stb_image do
   sh "cd ext/rb_stb_image; ruby extconf.rb; make"
 end

data/examples/dcgan/dcgan.rb

@@ -121,7 +121,7 @@ class DCGAN < Model
     x
   end
 
-  def train_step(x_batch, y_batch)
+  def train_step(x_batch, y_batch, need_accuracy: false)
     batch_size = x_batch.shape[0]
     noise = Numo::SFloat.new(batch_size, 20).rand(-1, 1)
     images = @gen.predict(noise)

data/examples/iris_example.rb

@@ -3,6 +3,7 @@ require "dnn/datasets/iris"
 # If you use numo/linalg then please uncomment out.
 # require "numo/linalg/autoloader"
 
+include DNN::Models
 include DNN::Layers
 include DNN::Optimizers
 include DNN::Losses
@@ -14,44 +15,19 @@ x_test, y_test = x[100...150, true], y[100...150]
 y_train = DNN::Utils.to_categorical(y_train, 3, Numo::SFloat)
 y_test = DNN::Utils.to_categorical(y_test, 3, Numo::SFloat)
 
-epochs = 1000
-batch_size = 32
-
-opt = Adam.new
-lf = SoftmaxCrossEntropy.new
-
-train_iter = DNN::Iterator.new(x_train, y_train)
-test_iter = DNN::Iterator.new(x_test, y_test, random: false)
-
-w1 = DNN::Param.new(Numo::SFloat.new(4, 16).rand_norm)
-b1 = DNN::Param.new(Numo::SFloat.zeros(16))
-w2 = DNN::Param.new(Numo::SFloat.new(16, 3).rand_norm)
-b2 = DNN::Param.new(Numo::SFloat.zeros(3))
-
-net = -> x, y do
-  h = Dot.(x, w1) + b1
-  h = Sigmoid.(h)
-  out = Dot.(h, w2) + b2
-  out
-end
-
-(1..epochs).each do |epoch|
-  train_iter.foreach(batch_size) do |x_batch, y_batch, step|
-    x = DNN::Tensor.convert(x_batch)
-    y = DNN::Tensor.convert(y_batch)
-    out = net.(x, y)
-    loss = lf.(out, y)
-    loss.link.backward
-    puts "epoch: #{epoch}, step: #{step}, loss = #{loss.data.to_f}"
-    opt.update([w1, b1, w2, b2])
-  end
-end
-
-correct = 0
-test_iter.foreach(batch_size) do |x_batch, y_batch, step|
-  x = DNN::Tensor.convert(x_batch)
-  y = DNN::Tensor.convert(y_batch)
-  out = net.(x, y)
-  correct += out.data.max_index(axis: 1).eq(y_batch.max_index(axis: 1)).count
-end
-puts "correct = #{correct}"
+model = Sequential.new
+
+model << InputLayer.new(4)
+
+model << Dense.new(16)
+model << Sigmoid.new
+
+model << Dense.new(3)
+
+model.setup(Adam.new, SoftmaxCrossEntropy.new)
+
+model.train(x_train, y_train, 1000, batch_size: 32, test: [x_test, y_test])
+
+accuracy, loss = model.evaluate(x_test, y_test)
+puts "accuracy: #{accuracy}"
+puts "loss: #{loss}"

data/examples/iris_example_unused_model.rb

@@ -0,0 +1,57 @@
+require "dnn"
+require "dnn/datasets/iris"
+# If you use numo/linalg then please uncomment out.
+# require "numo/linalg/autoloader"
+
+include DNN::Layers
+include DNN::Optimizers
+include DNN::Losses
+
+x, y = DNN::Iris.load(true)
+x_train, y_train = x[0...100, true], y[0...100]
+x_test, y_test = x[100...150, true], y[100...150]
+
+y_train = DNN::Utils.to_categorical(y_train, 3, Numo::SFloat)
+y_test = DNN::Utils.to_categorical(y_test, 3, Numo::SFloat)
+
+epochs = 1000
+batch_size = 32
+
+opt = Adam.new
+lf = SoftmaxCrossEntropy.new
+
+train_iter = DNN::Iterator.new(x_train, y_train)
+test_iter = DNN::Iterator.new(x_test, y_test, random: false)
+
+w1 = DNN::Param.new(Numo::SFloat.new(4, 16).rand_norm)
+b1 = DNN::Param.new(Numo::SFloat.zeros(16))
+w2 = DNN::Param.new(Numo::SFloat.new(16, 3).rand_norm)
+b2 = DNN::Param.new(Numo::SFloat.zeros(3))
+
+net = -> x, y do
+  h = Dot.(x, w1) + b1
+  h = Sigmoid.(h)
+  out = Dot.(h, w2) + b2
+  out
+end
+
+(1..epochs).each do |epoch|
+  train_iter.foreach(batch_size) do |x_batch, y_batch, step|
+    x = DNN::Tensor.convert(x_batch)
+    y = DNN::Tensor.convert(y_batch)
+    out = net.(x, y)
+    loss = lf.(out, y)
+    loss.link.backward
+    puts "epoch: #{epoch}, step: #{step}, loss = #{loss.data.to_f}"
+    opt.update([w1, b1, w2, b2])
+  end
+end
+
+correct = 0
+test_iter.foreach(batch_size) do |x_batch, y_batch, step|
+  x = DNN::Tensor.convert(x_batch)
+  y = DNN::Tensor.convert(y_batch)
+  out = net.(x, y)
+  correct += out.data.max_index(axis: 1).eq(y_batch.max_index(axis: 1)).count
+end
+puts "correct = #{correct}"

data/examples/vae.rb

@@ -97,7 +97,7 @@ model = VAE.new
 dec = model.dec
 model.setup(Adam.new, VAELoss.new)
 
-model.train(x_train, x_train, 10, batch_size: 128)
+model.train(x_train, x_train, 10, batch_size: 128, need_accuracy: false)
 
 images = []
 10.times do |i|

data/lib/dnn/core/callbacks.rb

@@ -6,6 +6,12 @@ module DNN
 
       # Please implement the method used for callback event.
 
+      # Process performed before all training.
+      # def before_train; end
+
+      # Process performed after all training.
+      # def after_train; end
+
       # Process performed before one training.
       # def before_epoch; end
 
@@ -57,7 +63,7 @@ module DNN
 
     # A callback to stop training the model early after test on batch.
     # @param [Symbol] trigger A log that triggers early stopping.
-    #                         Specify one of train_loss, test_loss, test_accuracy.
+    #                         Specify one of :loss, :test_loss, :test_accuracy
     # @param [Float] tolerance Tolerance value for early stopping.
     class EarlyStopping < Callback
       def initialize(trigger, tolerance)
@@ -66,19 +72,21 @@ module DNN
       end
 
       def after_train_on_batch
-        throw :stop, "Early stopped." if judge_early_stopping_train
+        @model.request_early_stop if judge_early_stopping_train
       end
 
       def after_epoch
-        throw :stop, "Early stopped." if judge_early_stopping_test
+        @model.request_early_stop if judge_early_stopping_test
       end
 
       private
 
       def judge_early_stopping_train
         case @trigger
-        when :train_loss
+        when :loss
           return true if model.last_log[@trigger] <= @tolerance
+        when :accuracy
+          return true if model.last_log[@trigger] >= @tolerance
         end
         false
       end
@@ -97,7 +105,7 @@ module DNN
     # A callback to stop training the model if loss is NaN by after train on batch.
     class NaNStopping < Callback
       def after_train_on_batch
-        throw :stop, "loss is NaN." if model.last_log[:train_loss].nan?
+        throw :stop, "loss is NaN." if model.last_log[:loss].nan?
       end
     end
 
@@ -105,7 +113,8 @@ module DNN
     # The following logs will be recorded.
     # epoch:          Current epoch.
     # step:           Current step in epoch.
-    # train_loss:     Batch training loss.
+    # loss:           Batch training loss.
+    # accuracy:       Batch training accuracy.
     # test_loss:      Mean test loss.
     # test_accuracy:  Test accuracy.
     class Logger < Callback
@@ -113,7 +122,8 @@ module DNN
         @log = {
           epoch: [],
           step: [],
-          train_loss: [],
+          loss: [],
+          accuracy: [],
           test_loss: [],
           test_accuracy: [],
         }
@@ -124,7 +134,7 @@ module DNN
       end
 
       def after_train_on_batch
-        logging(:train_loss, :step)
+        logging(:loss, :step)
       end
 
       # Get a log.

data/lib/dnn/core/iterator.rb

@@ -4,11 +4,13 @@ module DNN
     attr_reader :num_datas
     attr_reader :last_round_down
 
-    # @param [Numo::SFloat | Array] x_datas input datas.
-    # @param [Numo::SFloat | Array] y_datas output datas.
+    # @param [Numo::NArray | Array] x_datas input datas.
+    # @param [Numo::NArray | Array] y_datas output datas.
     # @param [Boolean] random Set true to return batches randomly. Setting false returns batches in order of index.
     # @param [Boolean] last_round_down Set true to round down for last batch data when call foreach.
     def initialize(x_datas, y_datas, random: true, last_round_down: false)
+      Utils.check_input_data_type("x_datas", x_datas, Xumo::NArray)
+      Utils.check_input_data_type("y_datas", y_datas, Xumo::NArray)
       @x_datas = x_datas
       @y_datas = y_datas
       @random = random
@@ -64,12 +66,26 @@ module DNN
     # @param [Integer] batch_size Batch size.
     # @yield Executes block by receiving the specified arguments (x_batch, y_batch).
     def foreach(batch_size, &block)
-      steps = @last_round_down ? @num_datas / batch_size : (@num_datas.to_f / batch_size).ceil
-      steps.times do |step|
+      max_steps(batch_size).times do |step|
         x_batch, y_batch = next_batch(batch_size)
         block.call(x_batch, y_batch, step)
       end
       reset
     end
+
+    # Return the number of available data considering last_round_down.
+    def num_usable_datas(batch_size)
+      if @last_round_down
+        max_steps(batch_size) * batch_size
+      else
+        @num_datas
+      end
+    end
+
+    # Get max steps for iteration.
+    # @param [Integer] batch_size Batch size.
+    def max_steps(batch_size)
+      @last_round_down ? @num_datas / batch_size : (@num_datas.to_f / batch_size).ceil
+    end
   end
 end

data/lib/dnn/core/layers/basic_layers.rb

@@ -26,8 +26,8 @@ module DNN
       attr_reader :input_shape
       attr_reader :output_shape
 
-      def self.call(x, *args)
-        new(*args).(x)
+      def self.call(x, *args, **kwargs)
+        new(*args, **kwargs).(x)
       end
 
       def self.from_hash(hash)

data/lib/dnn/core/layers/merge_layers.rb

@@ -2,8 +2,8 @@ module DNN
   module Layers
 
     class MergeLayer < Layer
-      def self.call(x1, x2, *args)
-        new(*args).call(x1, x2)
+      def self.call(x1, x2, *args, **kwargs)
+        new(*args, **kwargs).call(x1, x2)
       end
 
       def call(input1, input2)

data/lib/dnn/core/layers/rnn_layers.rb

@@ -1,5 +1,16 @@
 module DNN
   module Layers
+    # Super class of all RNN cells.
+    class RNNCell
+      attr_accessor :trainable
+
+      def initialize(weight, recurrent_weight, bias)
+        @weight = weight
+        @recurrent_weight = recurrent_weight
+        @bias = bias
+        @trainable = true
+      end
+    end
 
     # Super class of all RNN classes.
     class RNN < Connection
@@ -136,15 +147,10 @@ module DNN
       end
     end
 
-    class SimpleRNNDense
-      attr_accessor :trainable
-
+    class SimpleRNNCell < RNNCell
       def initialize(weight, recurrent_weight, bias, activation)
-        @weight = weight
-        @recurrent_weight = recurrent_weight
-        @bias = bias
+        super(weight, recurrent_weight, bias)
         @activation = activation.clone
-        @trainable = true
       end
 
       def forward(x, h)
@@ -206,7 +212,7 @@ module DNN
       end
 
       def create_hidden_layer
-        @hidden_layers = Array.new(@time_length) { SimpleRNNDense.new(@weight, @recurrent_weight, @bias, @activation) }
+        @hidden_layers = Array.new(@time_length) { SimpleRNNCell.new(@weight, @recurrent_weight, @bias, @activation) }
       end
 
       def to_hash
@@ -228,19 +234,14 @@ module DNN
       end
     end
 
-    class LSTMDense
-      attr_accessor :trainable
-
+    class LSTMCell < RNNCell
       def initialize(weight, recurrent_weight, bias)
-        @weight = weight
-        @recurrent_weight = recurrent_weight
-        @bias = bias
+        super(weight, recurrent_weight, bias)
         @tanh = Layers::Tanh.new
         @g_tanh = Layers::Tanh.new
         @forget_sigmoid = Layers::Sigmoid.new
         @in_sigmoid = Layers::Sigmoid.new
         @out_sigmoid = Layers::Sigmoid.new
-        @trainable = true
       end
 
       def forward(x, h, c)
@@ -312,7 +313,7 @@ module DNN
       end
 
       def create_hidden_layer
-        @hidden_layers = Array.new(@time_length) { LSTMDense.new(@weight, @recurrent_weight, @bias) }
+        @hidden_layers = Array.new(@time_length) { LSTMCell.new(@weight, @recurrent_weight, @bias) }
       end
 
       def forward_node(xs)
@@ -365,17 +366,12 @@ module DNN
       end
     end
 
-    class GRUDense < Layer
-      attr_accessor :trainable
-
+    class GRUCell < RNNCell
       def initialize(weight, recurrent_weight, bias)
-        @weight = weight
-        @recurrent_weight = recurrent_weight
-        @bias = bias
+        super(weight, recurrent_weight, bias)
         @update_sigmoid = Layers::Sigmoid.new
         @reset_sigmoid = Layers::Sigmoid.new
         @tanh = Layers::Tanh.new
-        @trainable = true
       end
 
       def forward(x, h)
@@ -457,7 +453,7 @@ module DNN
       end
 
       def create_hidden_layer
-        @hidden_layers = Array.new(@time_length) { GRUDense.new(@weight, @recurrent_weight, @bias) }
+        @hidden_layers = Array.new(@time_length) { GRUCell.new(@weight, @recurrent_weight, @bias) }
       end
     end
 

data/lib/dnn/core/losses.rb

@@ -2,8 +2,8 @@ module DNN
   module Losses
 
     class Loss
-      def self.call(y, t, *args)
-        new(*args).(y, t)
+      def self.call(y, t, *args, **kwargs)
+        new(*args, **kwargs).(y, t)
       end
 
       def self.from_hash(hash)