ruby-dnn 0.13.4 → 0.14.0

data/lib/dnn/core/link.rb

@@ -14,7 +14,6 @@ module DNN
     end
   end
 
-
   class TwoInputLink
     attr_accessor :prev1
     attr_accessor :prev2
@@ -27,7 +26,7 @@ module DNN
     end
 
     def backward(dy)
-      dy1, dy2 =* @layer.backward(dy)
+      dy1, dy2 = *@layer.backward(dy)
       @prev1&.backward(dy1)
       @prev2&.backward(dy2)
     end

data/lib/dnn/core/losses.rb

@@ -6,14 +6,14 @@ module DNN
         return nil unless hash
         loss_class = DNN.const_get(hash[:class])
         loss = loss_class.allocate
-        raise DNN_Error.new("#{loss.class} is not an instance of #{self} class.") unless loss.is_a?(self)
+        raise DNN_Error, "#{loss.class} is not an instance of #{self} class." unless loss.is_a?(self)
         loss.load_hash(hash)
         loss
       end
 
       def loss(y, t, layers = nil)
         unless y.shape == t.shape
-          raise DNN_ShapeError.new("The shape of y does not match the t shape. y shape is #{y.shape}, but t shape is #{t.shape}.")
+          raise DNN_ShapeError, "The shape of y does not match the t shape. y shape is #{y.shape}, but t shape is #{t.shape}."
         end
         loss_value = forward(y, t)
         loss_value += regularizers_forward(layers) if layers
@@ -21,11 +21,11 @@ module DNN
       end
 
       def forward(y, t)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'forward'"
       end
 
       def backward(y, t)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'backward'")
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'backward'"
       end
 
       def regularizers_forward(layers)
@@ -58,7 +58,7 @@ module DNN
     class MeanSquaredError < Loss
       def forward(y, t)
         batch_size = t.shape[0]
-        0.5 * ((y - t) ** 2).sum / batch_size
+        0.5 * ((y - t)**2).sum / batch_size
       end
 
       def backward(y, t)
@@ -66,7 +66,6 @@ module DNN
       end
     end
 
-
     class MeanAbsoluteError < Loss
       def forward(y, t)
         batch_size = t.shape[0]
@@ -81,7 +80,6 @@ module DNN
       end
     end
 
-
     class Hinge < Loss
       def forward(y, t)
         @a = 1 - y * t
@@ -95,7 +93,6 @@ module DNN
       end
     end
 
-
     class HuberLoss < Loss
       def forward(y, t)
         loss_l1_value = loss_l1(y, t)
@@ -120,16 +117,19 @@ module DNN
 
       def loss_l2(y, t)
         batch_size = t.shape[0]
-        0.5 * ((y - t) ** 2).sum / batch_size
+        0.5 * ((y - t)**2).sum / batch_size
       end
     end
 
-
     class SoftmaxCrossEntropy < Loss
       attr_accessor :eps
 
-      def self.softmax(y)
-        Xumo::NMath.exp(y) / Xumo::NMath.exp(y).sum(1, keepdims: true)
+      class << self
+        def softmax(y)
+          Xumo::NMath.exp(y) / Xumo::NMath.exp(y).sum(1, keepdims: true)
+        end
+
+        alias activation softmax
       end
 
       # @param [Float] eps Value to avoid nan.
@@ -156,17 +156,24 @@ module DNN
       end
     end
 
-
     class SigmoidCrossEntropy < Loss
       attr_accessor :eps
 
+      class << self
+        def sigmoid(y)
+          Layers::Sigmoid.new.forward(y)
+        end
+
+        alias activation sigmoid
+      end
+
       # @param [Float] eps Value to avoid nan.
       def initialize(eps: 1e-7)
         @eps = eps
       end
 
       def forward(y, t)
-        @x = Activations::Sigmoid.new.forward(y)
+        @x = SigmoidCrossEntropy.sigmoid(y)
         -(t * Xumo::NMath.log(@x) + (1 - t) * Xumo::NMath.log(1 - @x))
       end
 

data/lib/dnn/core/merge_layers.rb

@@ -3,20 +3,21 @@ module DNN
 
     class MergeLayer < Layers::Layer
       def self.call(x1, x2, *args)
-        self.new(*args).call(x1, x2)
+        new(*args).call(x1, x2)
       end
 
-      def call(input1, input2)
-        x1, prev_link1 = *input1
-        x2, prev_link2 = *input2
+      def call(input_tensor1, input_tensor2)
+        x1 = input_tensor1.data
+        x2 = input_tensor2.data
+        prev_link1 = input_tensor1.link
+        prev_link2 = input_tensor2.link
         build(x1.shape[1..-1]) unless built?
         y = forward(x1, x2)
         link = TwoInputLink.new(prev_link1, prev_link2, self)
-        [y, link]
+        Tensor.new(y, link)
       end
     end
 
-
     class Add < MergeLayer
       def forward(x1, x2)
         x1 + x2
@@ -27,7 +28,6 @@ module DNN
       end
     end
 
-
     class Mul < MergeLayer
       def forward(x1, x2)
         @x1, @x2 = x1, x2
@@ -39,7 +39,6 @@ module DNN
       end
     end
 
-
     class Concatenate < MergeLayer
       attr_reader :axis
 

data/lib/dnn/core/models.rb

@@ -1,16 +1,16 @@
 module DNN
   module Models
-
     # This class deals with the model of the network.
     class Model
       attr_accessor :optimizer
       attr_accessor :loss_func
+      attr_reader :last_log
 
       # Load marshal model.
       # @param [String] file_name File name of marshal model to load.
       # @return [DNN::Models::Model] Return the loaded model.
       def self.load(file_name)
-        model = self.new
+        model = new
         loader = Loaders::MarshalLoader.new(model)
         loader.load(file_name)
         model
@@ -21,49 +21,9 @@ module DNN
         @loss_func = nil
         @last_link = nil
         @built = false
-        @callbacks = {
-          before_epoch: [],
-          after_epoch: [],
-          before_train_on_batch: [],
-          after_train_on_batch: [],
-          before_test_on_batch: [],
-          after_test_on_batch: [],
-        }
+        @callbacks = []
         @layers_cache = nil
-      end
-
-      # This method is provided for compatibility with v0.12.4.
-      # Load hash model parameters.
-      # @param [Hash] hash Hash to load model parameters.
-      def load_hash_params(hash)
-        has_param_layers_params = hash[:params]
-        has_param_layers_index = 0
-        has_param_layers.uniq.each do |layer|
-          hash_params = has_param_layers_params[has_param_layers_index]
-          hash_params.each do |key, (shape, bin)|
-            data = Xumo::SFloat.from_binary(bin).reshape(*shape)
-            layer.get_params[key].data = data
-          end
-          has_param_layers_index += 1
-        end
-      end
-
-      # This method is provided for compatibility with v0.12.4.
-      # Load json model parameters.
-      # @param [String] json_str JSON string to load model parameters.
-      def load_json_params(json_str)
-        hash = JSON.parse(json_str, symbolize_names: true)
-        has_param_layers_params = hash[:params]
-        has_param_layers_index = 0
-        has_param_layers.uniq.each do |layer|
-          hash_params = has_param_layers_params[has_param_layers_index]
-          hash_params.each do |key, (shape, base64_param)|
-            bin = Base64.decode64(base64_param)
-            data = Xumo::SFloat.from_binary(bin).reshape(*shape)
-            layer.get_params[key].data = data
-          end
-          has_param_layers_index += 1
-        end
+        @last_log = {}
       end
 
       # Set optimizer and loss_func to model.
@@ -71,10 +31,10 @@ module DNN
       # @param [DNN::Losses::Loss] loss_func Loss function to use for learning.
       def setup(optimizer, loss_func)
         unless optimizer.is_a?(Optimizers::Optimizer)
-          raise TypeError.new("optimizer:#{optimizer.class} is not an instance of DNN::Optimizers::Optimizer class.")
+          raise TypeError, "optimizer:#{optimizer.class} is not an instance of DNN::Optimizers::Optimizer class."
         end
         unless loss_func.is_a?(Losses::Loss)
-          raise TypeError.new("loss_func:#{loss_func.class} is not an instance of DNN::Losses::Loss class.")
+          raise TypeError, "loss_func:#{loss_func.class} is not an instance of DNN::Losses::Loss class."
         end
         @optimizer = optimizer
         @loss_func = loss_func
@@ -85,8 +45,8 @@ module DNN
       # @param [Numo::SFloat] x Input training data.
       # @param [Numo::SFloat] y Output training data.
       # @param [Integer] epochs Number of training.
-      # @param [Integer] initial_epoch Initial epoch.
       # @param [Integer] batch_size Batch size used for one training.
+      # @param [Integer] initial_epoch Initial epoch.
       # @param [Array | NilClass] test If you to test the model for every 1 epoch,
       #                                specify [x_test, y_test]. Don't test to the model, specify nil.
       # @param [Boolean] verbose Set true to display the log. If false is set, the log is not displayed.
@@ -95,57 +55,107 @@ module DNN
                 initial_epoch: 1,
                 test: nil,
                 verbose: true)
-        raise DNN_Error.new("The model is not optimizer setup complete.") unless @optimizer
-        raise DNN_Error.new("The model is not loss_func setup complete.") unless @loss_func
         check_xy_type(x, y)
-        iter = Iterator.new(x, y)
-        num_train_datas = x.is_a?(Array) ? x[0].shape[0] : x.shape[0]
-        (initial_epoch..epochs).each do |epoch|
-          call_callbacks(:before_epoch, epoch)
-          puts "【 epoch #{epoch}/#{epochs} 】" if verbose
-          iter.foreach(batch_size) do |x_batch, y_batch, index|
-            loss_value = train_on_batch(x_batch, y_batch)
-            if loss_value.is_a?(Xumo::SFloat)
-              loss_value = loss_value.mean
-            elsif loss_value.nan?
-              puts "\nloss is nan" if verbose
-              return
-            end
-            num_trained_datas = (index + 1) * batch_size
-            num_trained_datas = num_trained_datas > num_train_datas ? num_train_datas : num_trained_datas
-            log = "\r"
-            40.times do |i|
-              if i < num_trained_datas * 40 / num_train_datas
-                log << "="
-              elsif i == num_trained_datas * 40 / num_train_datas
-                log << ">"
-              else
-                log << "_"
+        train_iterator = Iterator.new(x, y)
+        train_by_iterator(train_iterator, epochs,
+                          batch_size: batch_size,
+                          initial_epoch: initial_epoch,
+                          test: test,
+                          verbose: verbose)
+      end
+
+      alias fit train
+
+      # Start training by iterator.
+      # Setup the model before use this method.
+      # @param [Iterator] train_iterator Iterator used for training.
+      # @param [Integer] epochs Number of training.
+      # @param [Integer] batch_size Batch size used for one training.
+      # @param [Integer] initial_epoch Initial epoch.
+      # @param [Array | NilClass] test If you to test the model for every 1 epoch,
+      #                                specify [x_test, y_test]. Don't test to the model, specify nil.
+      # @param [Boolean] verbose Set true to display the log. If false is set, the log is not displayed.
+      def train_by_iterator(train_iterator, epochs,
+                            batch_size: 1,
+                            initial_epoch: 1,
+                            test: nil,
+                            verbose: true)
+        raise DNN_Error, "The model is not optimizer setup complete." unless @optimizer
+        raise DNN_Error, "The model is not loss_func setup complete." unless @loss_func
+
+        num_train_datas = train_iterator.num_datas
+        num_train_datas = num_train_datas / batch_size * batch_size if train_iterator.last_round_down
+
+        stopped = catch(:stop) do
+          (initial_epoch..epochs).each do |epoch|
+            @last_log[:epoch] = epoch
+            call_callbacks(:before_epoch)
+            puts "【 epoch #{epoch}/#{epochs} 】" if verbose
+
+            train_iterator.foreach(batch_size) do |x_batch, y_batch, index|
+              train_step_met = train_step(x_batch, y_batch)
+              num_trained_datas = (index + 1) * batch_size
+              num_trained_datas = num_trained_datas > num_train_datas ? num_train_datas : num_trained_datas
+              log = "\r"
+              40.times do |i|
+                if i < num_trained_datas * 40 / num_train_datas
+                  log << "="
+                elsif i == num_trained_datas * 40 / num_train_datas
+                  log << ">"
+                else
+                  log << "_"
+                end
               end
+
+              log << "  #{num_trained_datas}/#{num_train_datas} "
+              log << metrics_to_str(train_step_met)
+              print log if verbose
             end
-            log << "  #{num_trained_datas}/#{num_train_datas} loss: #{sprintf('%.8f', loss_value)}"
-            print log if verbose
-          end
-          if test
-            acc, test_loss = accuracy(test[0], test[1], batch_size: batch_size)
-            print "  accuracy: #{acc}, test loss: #{sprintf('%.8f', test_loss)}" if verbose
+
+            if test
+              test_met = test(test[0], test[1], batch_size: batch_size)
+              print "  " + metrics_to_str(test_met) if verbose
+            end
+            puts "" if verbose
+            call_callbacks(:after_epoch)
           end
-          puts "" if verbose
-          call_callbacks(:after_epoch, epoch)
+          nil
+        end
+
+        if stopped
+          puts "\n#{stopped}" if verbose
         end
       end
 
-      alias fit train
+      alias fit_by_iterator train_by_iterator
+
+      # Implement the training process to be performed in one step.
+      # @param [Numo::SFloat] x Input training data.
+      # @param [Numo::SFloat] y Output training data.
+      # @return [Hash] Hash of contents to be output to log.
+      private def train_step(x, y)
+        loss_value = train_on_batch(x, y)
+        { loss: loss_value.mean }
+      end
+
+      # Implement the test process to be performed.
+      # @param [Numo::SFloat] x Input training data.
+      # @param [Numo::SFloat] y Output training data.
+      # @param [Integer] batch_size Batch size used for one test.
+      # @return [Hash] Hash of contents to be output to log.
+      private def test(x, y, batch_size: 100)
+        acc, test_loss = accuracy(x, y, batch_size: batch_size)
+        { accuracy: acc, test_loss: test_loss.mean }
+      end
 
       # Training once.
       # Setup the model before use this method.
       # @param [Numo::SFloat] x Input training data.
       # @param [Numo::SFloat] y Output training data.
-      # @param [Integer] batch_size Batch size used for one test.
       # @return [Float | Numo::SFloat] Return loss value in the form of Float or Numo::SFloat.
       def train_on_batch(x, y)
-        raise DNN_Error.new("The model is not optimizer setup complete.") unless @optimizer
-        raise DNN_Error.new("The model is not loss_func setup complete.") unless @loss_func
+        raise DNN_Error, "The model is not optimizer setup complete." unless @optimizer
+        raise DNN_Error, "The model is not loss_func setup complete." unless @loss_func
         check_xy_type(x, y)
         call_callbacks(:before_train_on_batch)
         x = forward(x, true)
@@ -154,7 +164,8 @@ module DNN
         backward(dy)
         @optimizer.update(layers)
         @loss_func.regularizers_backward(layers)
-        call_callbacks(:after_train_on_batch, loss_value)
+        @last_log[:train_loss] = loss_value
+        call_callbacks(:after_train_on_batch)
         loss_value
       end
 
@@ -169,15 +180,18 @@ module DNN
         batch_size = batch_size >= num_test_datas[0] ? num_test_datas : batch_size
         iter = Iterator.new(x, y, random: false)
         total_correct = 0
-        sum_loss = 0
+        sum_loss = Xumo::SFloat[0]
         max_steps = (num_test_datas.to_f / batch_size).ceil
         iter.foreach(batch_size) do |x_batch, y_batch|
           correct, loss_value = test_on_batch(x_batch, y_batch)
           total_correct += correct
-          sum_loss += loss_value.is_a?(Xumo::SFloat) ? loss_value.mean : loss_value
+          sum_loss += loss_value.mean
         end
         mean_loss = sum_loss / max_steps
-        [total_correct.to_f / num_test_datas, mean_loss]
+        acc = total_correct.to_f / num_test_datas
+        @last_log[:test_loss] = mean_loss
+        @last_log[:test_accuracy] = acc
+        [acc, mean_loss]
       end
 
       # Evaluate once.
@@ -189,7 +203,7 @@ module DNN
         x = forward(x, false)
         correct = evaluate(x, y)
         loss_value = @loss_func.loss(x, y, layers)
-        call_callbacks(:after_test_on_batch, loss_value)
+        call_callbacks(:after_test_on_batch)
         [correct, loss_value]
       end
 
@@ -214,42 +228,33 @@ module DNN
 
       # Predict data.
       # @param [Numo::SFloat] x Input data.
-      def predict(x)
+      # @param [Boolean] use_loss_activation Use loss activation when loss has an activation.
+      def predict(x, use_loss_activation: true)
         check_xy_type(x)
-        forward(x, false)
+        y = forward(x, false)
+        if use_loss_activation && @loss_func.class.respond_to?(:activation)
+          y = @loss_func.class.activation(y)
+        end
+        y
       end
 
       # Predict one data.
       # @param [Numo::SFloat] x Input data. However, x is single data.
-      def predict1(x)
+      def predict1(x, use_loss_activation: true)
         check_xy_type(x)
-        predict(x.reshape(1, *x.shape))[0, false]
+        predict(x.reshape(1, *x.shape), use_loss_activation: use_loss_activation)[0, false]
       end
 
       # Add callback function.
-      # @param [Symbol] event Callback event. The following can be used for event.
-      #                       before_epoch: Process: performed before one training.
-      #                       after_epoch: Process:  performed after one training.
-      #                       before_train_on_batch: Set the proc to be performed before train on batch processing.
-      #                       after_train_on_batch:  Set the proc to be performed after train on batch processing.
-      #                       before_test_on_batch:  Set the proc to be performed before test on batch processing.
-      #                       after_test_on_batch:   Set the proc to be performed after test on batch processing.
-      def add_callback(event, callback)
-        raise DNN_UnknownEventError.new("Unknown event #{event}.") unless @callbacks.has_key?(event)
-        @callbacks[event] << callback
+      # @param [Callback] callback Callback object.
+      def add_callback(callback)
+        callback.model = self
+        @callbacks << callback
       end
 
       # Clear the callback function registered for each event.
-      # @param [Symbol] event Callback event. The following can be used for event.
-      #                       before_epoch: Process: performed before one training.
-      #                       after_epoch: Process:  performed after one training.
-      #                       before_train_on_batch: Set the proc to be performed before train on batch processing.
-      #                       after_train_on_batch:  Set the proc to be performed after train on batch processing.
-      #                       before_test_on_batch:  Set the proc to be performed before test on batch processing.
-      #                       after_test_on_batch:   Set the proc to be performed after test on batch processing.
-      def clear_callbacks(event)
-        raise DNN_UnknownEventError.new("Unknown event #{event}.") unless @callbacks.has_key?(event)
-        @callbacks[event] = []
+      def clear_callbacks
+        @callbacks = []
       end
 
       # Save the model in marshal format.
@@ -267,7 +272,7 @@ module DNN
       # Get the all layers.
       # @return [Array] All layers array.
       def layers
-        raise DNN_Error.new("This model is not built. You need build this model using predict or train.") unless built?
+        raise DNN_Error, "This model is not built. You need build this model using predict or train." unless built?
         return @layers_cache if @layers_cache
         layers = []
         get_layers = -> link do
@@ -291,7 +296,7 @@ module DNN
       end
 
       # Get the layer that the model has.
-      # @param [Symbol] The name of the layer to get.
+      # @param [Symbol] name The name of the layer to get.
       # @return [DNN::Layers::Layer] Return the layer.
       def get_layer(name)
         layers.find { |layer| layer.name == name }
@@ -307,21 +312,22 @@ module DNN
       def forward(x, learning_phase)
         DNN.learning_phase = learning_phase
         @layers_cache = nil
-        y, @last_link = call(x)
+        output_tensor = call(Tensor.new(x, nil))
+        @last_link = output_tensor.link
         unless @built
           @built = true
           naming
         end
-        y
+        output_tensor.data
       end
 
       def backward(dy)
         @last_link.backward(dy)
       end
 
-      def call_callbacks(event, *args)
-        @callbacks[event].each do |callback|
-          callback.call(*args)
+      def call_callbacks(event)
+        @callbacks.each do |callback|
+          callback.send(event) if callback.respond_to?(event)
         end
       end
 
@@ -338,17 +344,20 @@ module DNN
         end
       end
 
+      def metrics_to_str(mertics)
+        mertics.map { |key, num| "#{key}: #{sprintf('%.4f', num)}" }.join(", ")
+      end
+
       def check_xy_type(x, y = nil)
         if !x.is_a?(Xumo::SFloat) && !x.is_a?(Array)
-          raise TypeError.new("x:#{x.class.name} is not an instance of #{Xumo::SFloat.name} class or Array class.")
+          raise TypeError, "x:#{x.class.name} is not an instance of #{Xumo::SFloat.name} class or Array class."
         end
         if y && !y.is_a?(Xumo::SFloat) && !x.is_a?(Array)
-          raise TypeError.new("y:#{y.class.name} is not an instance of #{Xumo::SFloat.name} class or Array class.")
+          raise TypeError, "y:#{y.class.name} is not an instance of #{Xumo::SFloat.name} class or Array class."
         end
       end
     end
 
-
     class Sequential < Model
       attr_reader :stack
 
@@ -363,7 +372,7 @@ module DNN
       # @return [DNN::Models::Model] Return self.
       def add(layer)
         unless layer.is_a?(Layers::Layer) || layer.is_a?(Model)
-          raise TypeError.new("layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Model class.")
+          raise TypeError, "layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Model class."
         end
         @stack << layer
         self
@@ -376,7 +385,7 @@ module DNN
       # @return [Boolean] Return true if success for remove layer.
       def remove(layer)
         unless layer.is_a?(Layers::Layer) || layer.is_a?(Model)
-          raise TypeError.new("layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Model class.")
+          raise TypeError, "layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Model class."
         end
         @stack.delete(layer) ? true : false
       end