ruby-dnn 0.12.4 → 0.13.0

data/lib/dnn/core/merge_layers.rb

@@ -7,12 +7,12 @@ module DNN
       end
 
       def call(input1, input2)
-        x1, prev_link1, learning_phase = *input1
-        x2, prev_link2, * = *input2
+        x1, prev_link1 = *input1
+        x2, prev_link2 = *input2
         build(x1.shape[1..-1]) unless built?
         y = forward(x1, x2)
         link = TwoInputLink.new(prev_link1, prev_link2, self)
-        [y, link, learning_phase]
+        [y, link]
       end
     end
 

data/lib/dnn/core/models.rb

@@ -1,25 +1,35 @@
-require "zlib"
-require "json"
-require "base64"
-
 module DNN
   module Models
 
     # This class deals with the model of the network.
     class Model
+      attr_accessor :optimizer
+      attr_accessor :loss_func
+
       # Load marshal model.
       # @param [String] file_name File name of marshal model to load.
       def self.load(file_name)
-        Marshal.load(Zlib::Inflate.inflate(File.binread(file_name)))
+        loader = Loaders::MarshalLoader.new(self.new)
+        loader.load(file_name)
       end
 
       def initialize
         @optimizer = nil
+        @loss_func = nil
         @last_link = nil
-        @setup_completed = false
         @built = false
+        @callbacks = {
+          before_epoch: [],
+          after_epoch: [],
+          before_train_on_batch: [],
+          after_train_on_batch: [],
+          before_test_on_batch: [],
+          after_test_on_batch: [],
+        }
+        @layers_cache = nil
       end
 
+      # This method is provided for compatibility with v0.12.4.
       # Load hash model parameters.
       # @param [Hash] hash Hash to load model parameters.
       def load_hash_params(hash)
@@ -35,6 +45,7 @@ module DNN
         end
       end
 
+      # This method is provided for compatibility with v0.12.4.
       # Load json model parameters.
       # @param [String] json_str JSON string to load model parameters.
       def load_json_params(json_str)
@@ -52,30 +63,6 @@ module DNN
         end
       end
 
-      # Convert model parameters to hash.
-      # @return [Hash] Return the hash of model parameters.
-      def params_to_hash
-        has_param_layers_params = has_param_layers.uniq.map do |layer|
-          layer.get_params.map { |key, param|
-            [key, [param.data.shape, param.data.to_binary]]
-          }.to_h
-        end
-        { version: VERSION, params: has_param_layers_params }
-      end
-
-      # Convert model parameters to JSON string.
-      # @return [String] Return the JSON string.
-      def params_to_json
-        has_param_layers_params = has_param_layers.uniq.map do |layer|
-          layer.get_params.map { |key, param|
-            base64_data = Base64.encode64(param.data.to_binary)
-            [key, [param.data.shape, base64_data]]
-          }.to_h
-        end
-        hash = { version: VERSION, params: has_param_layers_params }
-        JSON.dump(hash)
-      end
-
       # Set optimizer and loss_func to model.
       # @param [DNN::Optimizers::Optimizer] optimizer Optimizer to use for learning.
       # @param [DNN::Losses::Loss] loss_func Loss function to use for learning.
@@ -86,7 +73,6 @@ module DNN
         unless loss_func.is_a?(Losses::Loss)
           raise TypeError.new("loss_func:#{loss_func.class} is not an instance of DNN::Losses::Loss class.")
         end
-        @setup_completed = true
         @optimizer = optimizer
         @loss_func = loss_func
       end
@@ -100,32 +86,20 @@ module DNN
       # @param [Array | NilClass] test If you to test the model for every 1 epoch,
       #                                specify [x_test, y_test]. Don't test to the model, specify nil.
       # @param [Boolean] verbose Set true to display the log. If false is set, the log is not displayed.
-      # @param [Lambda] before_epoch_cbk Process performed before one training.
-      # @param [Lambda] after_epoch_cbk Process performed after one training.
-      # @param [Lambda] before_train_on_batch_cbk Set the proc to be performed before train on batch processing.
-      # @param [Lambda] after_train_on_batch_cbk Set the proc to be performed after train on batch processing.
-      # @param [Lambda] before_test_on_batch_cbk Set the proc to be performed before test on batch processing.
-      # @param [Lambda] after_test_on_batch_cbk Set the proc to be performed after test on batch processing.
       def train(x, y, epochs,
                 batch_size: 1,
                 test: nil,
-                verbose: true,
-                before_epoch_cbk: nil,
-                after_epoch_cbk: nil,
-                before_train_on_batch_cbk: nil,
-                after_train_on_batch_cbk: nil,
-                before_test_on_batch_cbk: nil,
-                after_test_on_batch_cbk: nil)
-        raise DNN_Error.new("The model is not setup complete.") unless setup_completed?
+                verbose: true)
+        raise DNN_Error.new("The model is not optimizer setup complete.") unless @optimizer
+        raise DNN_Error.new("The model is not loss_func setup complete.") unless @loss_func
         check_xy_type(x, y)
         iter = Iterator.new(x, y)
-        num_train_datas = x.shape[0]
+        num_train_datas = x.is_a?(Array) ? x[0].shape[0] : x.shape[0]
         (1..epochs).each do |epoch|
-          before_epoch_cbk&.call(epoch)
+          call_callbacks(:before_epoch, epoch)
           puts "【 epoch #{epoch}/#{epochs} 】" if verbose
           iter.foreach(batch_size) do |x_batch, y_batch, index|
-            loss_value = train_on_batch(x_batch, y_batch, before_train_on_batch_cbk: before_train_on_batch_cbk,
-                                        after_train_on_batch_cbk: after_train_on_batch_cbk)
+            loss_value = train_on_batch(x_batch, y_batch)
             if loss_value.is_a?(Xumo::SFloat)
               loss_value = loss_value.mean
             elsif loss_value.nan?
@@ -148,85 +122,82 @@ module DNN
             print log if verbose
           end
           if test
-            acc, test_loss = accurate(test[0], test[1], batch_size: batch_size, before_test_on_batch_cbk: before_test_on_batch_cbk,
-                                      after_test_on_batch_cbk: after_test_on_batch_cbk)
-            print "  accurate: #{acc}, test loss: #{sprintf('%.8f', test_loss)}" if verbose
+            acc, test_loss = accuracy(test[0], test[1], batch_size: batch_size)
+            print "  accuracy: #{acc}, test loss: #{sprintf('%.8f', test_loss)}" if verbose
           end
           puts "" if verbose
-          after_epoch_cbk&.call(epoch)
+          call_callbacks(:after_epoch, epoch)
         end
       end
 
+      alias fit train
+
       # Training once.
       # Setup the model before use this method.
       # @param [Numo::SFloat] x Input training data.
       # @param [Numo::SFloat] y Output training data.
-      # @param [Lambda] before_train_on_batch_cbk Set the proc to be performed before train on batch processing.
-      # @param [Lambda] after_train_on_batch_cbk Set the proc to be performed after train on batch processing.
       # @return [Float | Numo::SFloat] Return loss value in the form of Float or Numo::SFloat.
-      def train_on_batch(x, y, before_train_on_batch_cbk: nil, after_train_on_batch_cbk: nil)
-        raise DNN_Error.new("The model is not setup complete.") unless setup_completed?
+      def train_on_batch(x, y)
+        raise DNN_Error.new("The model is not optimizer setup complete.") unless @optimizer
+        raise DNN_Error.new("The model is not loss_func setup complete.") unless @loss_func
         check_xy_type(x, y)
-        before_train_on_batch_cbk&.call
+        call_callbacks(:before_train_on_batch)
         x = forward(x, true)
-        loss_value = @loss_func.forward(x, y, layers)
-        dy = @loss_func.backward(y, layers)
+        loss_value = @loss_func.loss(x, y, layers)
+        dy = @loss_func.backward(x, y)
         backward(dy)
         @optimizer.update(layers.uniq)
-        after_train_on_batch_cbk&.call(loss_value)
+        @loss_func.regularizers_backward(layers)
+        call_callbacks(:after_train_on_batch, loss_value)
         loss_value
       end
 
-      # Evaluate model and get accurate of test data.
+      # Evaluate model and get accuracy of test data.
       # @param [Numo::SFloat] x Input test data.
       # @param [Numo::SFloat] y Output test data.
-      # @param [Lambda] before_test_on_batch_cbk Set the proc to be performed before test on batch processing.
-      # @param [Lambda] after_test_on_batch_cbk Set the proc to be performed after test on batch processing.
-      # @return [Array] Returns the test data accurate and mean loss in the form [accurate, mean_loss].
-      def accurate(x, y, batch_size: 100, before_test_on_batch_cbk: nil, after_test_on_batch_cbk: nil)
+      # @return [Array] Returns the test data accuracy and mean loss in the form [accuracy, mean_loss].
+      def accuracy(x, y, batch_size: 100)
         check_xy_type(x, y)
-        batch_size = batch_size >= x.shape[0] ? x.shape[0] : batch_size
+        num_test_datas = x.is_a?(Array) ? x[0].shape[0] : x.shape[0]
+        batch_size = batch_size >= num_test_datas[0] ? num_test_datas : batch_size
         iter = Iterator.new(x, y, random: false)
         total_correct = 0
         sum_loss = 0
-        max_steps = (x.shape[0].to_f / batch_size).ceil
+        max_steps = (num_test_datas.to_f / batch_size).ceil
         iter.foreach(batch_size) do |x_batch, y_batch|
-          correct, loss_value = test_on_batch(x_batch, y_batch, before_test_on_batch_cbk: before_test_on_batch_cbk,
-                                              after_test_on_batch_cbk: after_test_on_batch_cbk)
+          correct, loss_value = test_on_batch(x_batch, y_batch)
           total_correct += correct
           sum_loss += loss_value.is_a?(Xumo::SFloat) ? loss_value.mean : loss_value
         end
         mean_loss = sum_loss / max_steps
-        [total_correct.to_f / x.shape[0], mean_loss]
+        [total_correct.to_f / num_test_datas, mean_loss]
       end
 
       # Evaluate once.
       # @param [Numo::SFloat] x Input test data.
       # @param [Numo::SFloat] y Output test data.
-      # @param [Lambda] before_test_on_batch_cbk Set the proc to be performed before test on batch processing.
-      # @param [Lambda] after_test_on_batch_cbk Set the proc to be performed after test on batch processing.
-      # @return [Array] Returns the test data accurate and mean loss in the form [accurate, mean_loss].
-      def test_on_batch(x, y, before_test_on_batch_cbk: nil, after_test_on_batch_cbk: nil)
-        before_test_on_batch_cbk&.call
+      # @return [Array] Returns the test data accuracy and mean loss in the form [accuracy, mean_loss].
+      def test_on_batch(x, y)
+        call_callbacks(:before_test_on_batch)
         x = forward(x, false)
         correct = evaluate(x, y)
-        loss_value = @loss_func.forward(x, y, layers)
-        after_test_on_batch_cbk&.call(loss_value)
+        loss_value = @loss_func.loss(x, y, layers)
+        call_callbacks(:after_test_on_batch, loss_value)
         [correct, loss_value]
       end
 
       private def evaluate(y, t)
-        correct = 0
-        y.shape[0].times do |i|
-          if y.shape[1..-1] == [1]
+        if y.shape[1..-1] == [1]
+          correct = 0
+          y.shape[0].times do |i|
             if @loss_func.is_a?(Losses::SigmoidCrossEntropy)
               correct += 1 if (y[i, 0] < 0 && t[i, 0] < 0.5) || (y[i, 0] >= 0 && t[i, 0] >= 0.5)
             else
               correct += 1 if (y[i, 0] < 0 && t[i, 0] < 0) || (y[i, 0] >= 0 && t[i, 0] >= 0)
             end
-          else
-            correct += 1 if y[i, true].max_index == t[i, true].max_index
           end
+        else
+          correct = y.max_index(axis: 1).eq(t.max_index(axis: 1)).count
         end
         correct
       end
@@ -245,17 +216,37 @@ module DNN
         predict(x.reshape(1, *x.shape))[0, false]
       end
 
+      # Add callback function.
+      # @param [Symbol] event Callback event. The following can be used for event.
+      #                       before_epoch: Process: performed before one training.
+      #                       after_epoch: Process:  performed after one training.
+      #                       before_train_on_batch: Set the proc to be performed before train on batch processing.
+      #                       after_train_on_batch:  Set the proc to be performed after train on batch processing.
+      #                       before_test_on_batch:  Set the proc to be performed before test on batch processing.
+      #                       after_test_on_batch:   Set the proc to be performed after test on batch processing.
+      def add_callback(event, callback)
+        raise DNN_UnknownEventError.new("Unknown event #{event}.") unless @callbacks.has_key?(event)
+        @callbacks[event] << callback
+      end
+
+      # Clear the callback function registered for each event.
+      # @param [Symbol] event Callback event. The following can be used for event.
+      #                       before_epoch: Process: performed before one training.
+      #                       after_epoch: Process:  performed after one training.
+      #                       before_train_on_batch: Set the proc to be performed before train on batch processing.
+      #                       after_train_on_batch:  Set the proc to be performed after train on batch processing.
+      #                       before_test_on_batch:  Set the proc to be performed before test on batch processing.
+      #                       after_test_on_batch:   Set the proc to be performed after test on batch processing.
+      def clear_callbacks(event)
+        raise DNN_UnknownEventError.new("Unknown event #{event}.") unless @callbacks.has_key?(event)
+        @callbacks[event] = []
+      end
+
       # Save the model in marshal format.
       # @param [String] file_name Name to save model.
       def save(file_name)
-        bin = Zlib::Deflate.deflate(Marshal.dump(self))
-        begin
-          File.binwrite(file_name, bin)
-        rescue Errno::ENOENT
-          dir_name = file_name.match(%r`(.*)/.+$`)[1]
-          Dir.mkdir(dir_name)
-          File.binwrite(file_name, bin)
-        end
+        saver = Savers::MarshalSaver.new(self)
+        saver.save(file_name)
       end
 
       # @return [DNN::Models::Model] Return the copy this model.
@@ -267,6 +258,7 @@ module DNN
       # @return [Array] All layers array.
       def layers
         raise DNN_Error.new("This model is not built. You need build this model using predict or train.") unless built?
+        return @layers_cache if @layers_cache
         layers = []
         get_layers = -> link do
           return unless link
@@ -279,7 +271,7 @@ module DNN
           end
         end
         get_layers.(@last_link)
-        layers
+        @layers_cache = layers
       end
 
       # Get the all has param layers.
@@ -289,38 +281,10 @@ module DNN
       end
 
       # Get the layer that the model has.
-      # @overload get_layer(index)
-      #   @param [Integer] The index of the layer to get.
-      #   @return [DNN::Layers::Layer] Return the layer.
-      # @overload get_layer(layer_class, index)
-      #   @param [Integer] The index of the layer to get.
-      #   @param [Class] The class of the layer to get.
-      #   @return [DNN::Layers::Layer] Return the layer.
-      def get_layer(*args)
-        if args.length == 1
-          index = args[0]
-          layers[index]
-        else
-          layer_class, index = args
-          layers.select { |layer| layer.is_a?(layer_class) }[index]
-        end
-      end
-
-      # @return [DNN::Optimizers::Optimizer] optimizer Return the optimizer to use for learning.
-      def optimizer
-        raise DNN_Error.new("The model is not setup complete.") unless setup_completed?
-        @optimizer
-      end
-
-      # @return [DNN::Losses::Loss] loss_func Return the loss function to use for learning.
-      def loss_func
-        raise DNN_Error.new("The model is not setup complete.") unless setup_completed?
-        @loss_func
-      end
-
-      # @return [Boolean] If model have already been setup completed then return true.
-      def setup_completed?
-        @setup_completed
+      # @param [Symbol] The name of the layer to get.
+      # @return [DNN::Layers::Layer] Return the layer.
+      def get_layer(name)
+        layers.find { |layer| layer.name == name }
       end
 
       # @return [Boolean] If model have already been built then return true.
@@ -331,31 +295,45 @@ module DNN
       private
 
       def forward(x, learning_phase)
-        @built = true
-        y, @last_link = call([x, nil, learning_phase])
+        DNN.learning_phase = learning_phase
+        @layers_cache = nil
+        y, @last_link = call(x)
+        unless @built
+          @built = true
+          naming
+        end
         y
       end
 
       def backward(dy)
-        bwd = -> link, dy do
-          return dy unless link
-          if link.is_a?(TwoInputLink)
-            dy1, dy2 = link.layer.backward(dy)
-            [bwd.(link.prev1, dy1), bwd.(link.prev2, dy2)]
-          else
-            dy = link.layer.backward(dy)
-            bwd.(link.prev, dy)
+        @last_link.backward(dy)
+      end
+
+      def call_callbacks(event, *args)
+        @callbacks[event].each do |callback|
+          callback.call(*args)
+        end
+      end
+
+      def naming
+        layers.uniq.each do |layer|
+          id = layers.uniq.select { |l| l.is_a?(layer.class) }.index(layer)
+          class_name = layer.class.name.split("::").last
+          layer.name = "#{class_name}_#{id}".to_sym unless layer.name
+          if layer.is_a?(Layers::HasParamLayer)
+            layer.get_params.each do |param_key, param|
+              param.name = "#{layer.name}__#{param_key}".to_sym unless param.name
+            end
           end
         end
-        bwd.(@last_link, dy)
       end
 
       def check_xy_type(x, y = nil)
-        unless x.is_a?(Xumo::SFloat)
-          raise TypeError.new("x:#{x.class.name} is not an instance of #{Xumo::SFloat.name} class.")
+        if !x.is_a?(Xumo::SFloat) && !x.is_a?(Array)
+          raise TypeError.new("x:#{x.class.name} is not an instance of #{Xumo::SFloat.name} class or Array class.")
         end
-        if y && !y.is_a?(Xumo::SFloat)
-          raise TypeError.new("y:#{y.class.name} is not an instance of #{Xumo::SFloat.name} class.")
+        if y && !y.is_a?(Xumo::SFloat) && !x.is_a?(Array)
+          raise TypeError.new("y:#{y.class.name} is not an instance of #{Xumo::SFloat.name} class or Array class.")
         end
       end
     end
@@ -373,7 +351,7 @@ module DNN
       # Add layer to the model.
       # @param [DNN::Layers::Layer] layer Layer to add to the model.
       # @return [DNN::Models::Model] Return self.
-      def <<(layer)
+      def add(layer)
         unless layer.is_a?(Layers::Layer) || layer.is_a?(Model)
           raise TypeError.new("layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Model class.")
         end
@@ -381,6 +359,18 @@ module DNN
         self
       end
 
+      alias << add
+
+      # Remove layer to the model.
+      # @param [DNN::Layers::Layer] layer Layer to remove to the model.
+      # @return [Boolean] Return true if success for remove layer.
+      def remove(layer)
+        unless layer.is_a?(Layers::Layer) || layer.is_a?(Model)
+          raise TypeError.new("layer: #{layer.class.name} is not an instance of the DNN::Layers::Layer class or DNN::Models::Model class.")
+        end
+        @stack.delete(layer) ? true : false
+      end
+
       def call(x)
         @stack.each do |layer|
           x = layer.(x)

data/lib/dnn/core/normalizations.rb

@@ -24,14 +24,6 @@ module DNN
         @eps = eps
       end
 
-      def call(input)
-        x, prev_link, learning_phase = *input
-        build(x.shape[1..-1]) unless built?
-        y = forward(x, learning_phase)
-        link = Link.new(prev_link, self)
-        [y, link, learning_phase]
-      end
-
       def build(input_shape)
         super
         @gamma = Param.new(Xumo::SFloat.ones(*output_shape), 0)
@@ -40,8 +32,8 @@ module DNN
         @running_var = Param.new(Xumo::SFloat.zeros(*output_shape))
       end
 
-      def forward(x, learning_phase)
-        if learning_phase
+      def forward(x)
+        if DNN.learning_phase
           mean = x.mean(axis: @axis, keepdims: true)
           @xc = x - mean
           var = (@xc ** 2).mean(axis: @axis, keepdims: true)