ruby-dnn 0.15.3 → 0.16.0

data/lib/dnn/core/models.rb

@@ -1,6 +1,7 @@
 module DNN
   module Models
 
+    # This class is used to hold multiple layers in an array.
     class LayersList < Array
       def self.from_hash_list(hash_list)
         layers_list = new
@@ -19,7 +20,7 @@ module DNN
       end
 
       def to_hash_list
-        map { |layer| layer.to_hash }
+        map(&:to_hash)
       end
 
       # Get the all layers.
@@ -38,8 +39,18 @@ module DNN
     end
 
     class Chain
-      def call(x)
-        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'call'"
+      # Forward propagation.
+      # @param [Tensor] input_tensor Input tensor.
+      # @return [Tensor] Output tensor.
+      def forward(input_tensor)
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'forward'"
+      end
+
+      # Forward propagation and create a link.
+      # @param [Tensor] input_tensor Input tensor.
+      # @return [Tensor] Output tensor.
+      def call(input_tensor)
+        forward(input_tensor)
       end
 
       # Get the all layers.
@@ -109,13 +120,19 @@ module DNN
       def initialize
         @optimizer = nil
         @loss_func = nil
-        @last_link = nil
         @built = false
         @callbacks = []
         @layers_cache = nil
         @last_log = {}
       end
 
+      def call(inputs)
+        @layers_cache = nil
+        output_tensor = forward(inputs)
+        @built = true unless @built
+        output_tensor
+      end
+
       # Set optimizer and loss_func to model.
       # @param [DNN::Optimizers::Optimizer] optimizer Optimizer to use for learning.
       # @param [DNN::Losses::Loss] loss_func Loss function to use for learning.
@@ -204,10 +221,10 @@ module DNN
 
             if test
               acc, loss = if test.is_a?(Array)
-                evaluate(test[0], test[1], batch_size: batch_size)
-              else
-                evaluate_by_iterator(test, batch_size: batch_size)
-              end
+                            evaluate(test[0], test[1], batch_size: batch_size)
+                          else
+                            evaluate_by_iterator(test, batch_size: batch_size)
+                          end
               print "  " + metrics_to_str({ accuracy: acc, test_loss: loss }) if verbose
             end
             puts "" if verbose
@@ -242,15 +259,14 @@ module DNN
         raise DNN_Error, "The model is not loss_func setup complete." unless @loss_func
         check_xy_type(x, y)
         call_callbacks(:before_train_on_batch)
-        x = forward(x, true)
-        loss_value = @loss_func.loss(x, y, layers)
-        dy = @loss_func.backward(x, y)
-        backward(dy)
-        @optimizer.update(layers)
-        @loss_func.regularizers_backward(layers)
-        @last_log[:train_loss] = loss_value
+        DNN.learning_phase = true
+        out = call(Tensor.convert(x))
+        loss = @loss_func.loss(out, Tensor.convert(y), layers)
+        loss.link.backward(Xumo::SFloat.zeros(y[0...1, false].shape))
+        @optimizer.update(get_all_trainable_params)
+        @last_log[:train_loss] = loss.data
         call_callbacks(:after_train_on_batch)
-        loss_value
+        loss.data
       end
 
       # Evaluate model and get accuracy and loss of test data.
@@ -291,11 +307,12 @@ module DNN
       # @return [Array] Returns the test data accuracy and mean loss in the form [accuracy, mean_loss].
       def test_on_batch(x, y)
         call_callbacks(:before_test_on_batch)
-        x = forward(x, false)
-        correct = accuracy(x, y)
-        loss_value = @loss_func.loss(x, y)
+        DNN.learning_phase = false
+        out = call(Tensor.convert(x))
+        correct = accuracy(out.data, y)
+        loss = @loss_func.(out, Tensor.convert(y))
         call_callbacks(:after_test_on_batch)
-        [correct, loss_value]
+        [correct, loss.data]
       end
 
       # Implement the process to accuracy this model.
@@ -323,7 +340,9 @@ module DNN
       # @param [Boolean] use_loss_activation Use loss activation when loss has an activation.
       def predict(x, use_loss_activation: true)
         check_xy_type(x)
-        y = forward(x, false)
+        DNN.learning_phase = false
+        out = call(Tensor.convert(x))
+        y = out.data
         if use_loss_activation && @loss_func.class.respond_to?(:activation)
           y = @loss_func.class.activation(y)
         end
@@ -386,9 +405,7 @@ module DNN
       # @return [DNN::Layers::Layer] Return the layer.
       def get_layer(name)
         layer = instance_variable_get("@#{name}")
-        if layer.is_a?(Layers::Layer) || layer.is_a?(Chain) || layer.is_a?(LayersList)
-          return layer
-        end
+        return layer if layer.is_a?(Layers::Layer) || layer.is_a?(Chain) || layer.is_a?(LayersList)
         nil
       end
 
@@ -398,11 +415,8 @@ module DNN
       end
 
       def clean_layers
-        layers.each do |layer|
-          layer.clean
-        end
+        layers.each(&:clean)
         @loss_func.clean
-        @last_link = nil
         @layers_cache = nil
       end
 
@@ -424,24 +438,10 @@ module DNN
 
       private
 
-      def forward(x, learning_phase)
-        DNN.learning_phase = learning_phase
-        @layers_cache = nil
-        inputs = if x.is_a?(Array)
-                   x.map { |a| Tensor.new(a, nil) }
-                 else
-                   Tensor.new(x, nil)
-                 end
-        output_tensor = call(inputs)
-        @last_link = output_tensor.link
-        unless @built
-          @built = true
-        end
-        output_tensor.data
-      end
-
-      def backward(dy)
-        @last_link.backward(dy)
+      def get_all_trainable_params
+        layers.select { |layer| layer.is_a?(Layers::TrainableLayer) && layer.trainable }
+              .map { |layer| layer.get_params.values }.flatten.compact
+              .select(&:grad)
       end
 
       def call_callbacks(event)
@@ -512,7 +512,7 @@ module DNN
         @stack.delete(layer) ? true : false
       end
 
-      def call(x)
+      def forward(x)
         @stack.each do |layer|
           x = layer.(x)
         end

data/lib/dnn/core/monkey_patch.rb

@@ -0,0 +1,75 @@
+class Integer
+  alias dnn__add +
+  def +(other)
+    if other.is_a?(DNN::Tensor)
+      DNN::Layers::Add.(self, other)
+    else
+      dnn__add(other)
+    end
+  end
+
+  alias dnn__sub -
+  def -(other)
+    if other.is_a?(DNN::Tensor)
+      DNN::Layers::Sub.(self, other)
+    else
+      dnn__sub(other)
+    end
+  end
+
+  alias dnn__mul *
+  def *(other)
+    if other.is_a?(DNN::Tensor)
+      DNN::Layers::Mul.(self, other)
+    else
+      dnn__mul(other)
+    end
+  end
+
+  alias dnn__div /
+  def /(other)
+    if other.is_a?(DNN::Tensor)
+      DNN::Layers::Div.(self, other)
+    else
+      dnn__div(other)
+    end
+  end
+end
+
+class Float
+  alias dnn__add +
+  def +(other)
+    if other.is_a?(DNN::Tensor)
+      DNN::Layers::Add.(self, other)
+    else
+      dnn__add(other)
+    end
+  end
+
+  alias dnn__sub -
+  def -(other)
+    if other.is_a?(DNN::Tensor)
+      DNN::Layers::Sub.(self, other)
+    else
+      dnn__sub(other)
+    end
+  end
+
+  alias dnn__mul *
+  def *(other)
+    if other.is_a?(DNN::Tensor)
+      DNN::Layers::Mul.(self, other)
+    else
+      dnn__mul(other)
+    end
+  end
+
+  alias dnn__div /
+  def /(other)
+    if other.is_a?(DNN::Tensor)
+      DNN::Layers::Div.(self, other)
+    else
+      dnn__div(other)
+    end
+  end
+end

data/lib/dnn/core/optimizers.rb

@@ -19,16 +19,20 @@ module DNN
         @clip_norm = clip_norm
       end
 
+      def update(params)
+        clip_grads(params) if @clip_norm
+        update_params(params)
+        params.each do |param|
+          param.grad = Xumo::SFloat[0]
+        end
+      end
+
       # Update layers has params.
-      def update(layers)
+      def update_layers(layers)
         target_params = layers.select { |layer| layer.is_a?(Layers::TrainableLayer) && layer.trainable }
                               .map { |layer| layer.get_params.values }.flatten.compact
                               .select(&:grad)
-        clip_grads(target_params) if @clip_norm
-        update_params(target_params)
-        target_params.each do |param|
-          param.grad = Xumo::SFloat[0]
-        end
+        update(target_params)
       end
 
       def to_hash(merge_hash = nil)

data/lib/dnn/core/param.rb

@@ -1,11 +1,28 @@
 module DNN
   class Param
+    attr_accessor :trainable
     attr_accessor :data
     attr_accessor :grad
 
     def initialize(data = nil, grad = nil)
       @data = data
       @grad = grad
+      @trainable = true
+    end
+
+    def backward(grad)
+      if @trainable
+        @grad ||= Xumo::SFloat[0]
+        if @data.shape == grad.shape
+          @grad += grad
+        elsif @data.shape == grad.shape[1..-1]
+          @grad += grad.sum(0)
+        else
+          raise DNN_Error, "Shape is missmatch."
+        end
+      else
+        @grad = Xumo::SFloat[0]
+      end
     end
   end
 end

data/lib/dnn/core/regularizers.rb

@@ -17,10 +17,6 @@ module DNN
         raise NotImplementedError, "Class '#{self.class.name}' has implement method 'forward'"
       end
 
-      def backward
-        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'backward'"
-      end
-
       def to_hash(merge_hash)
         hash = { class: self.class.name }
         hash.merge!(merge_hash)
@@ -33,25 +29,25 @@ module DNN
     end
 
     class L1 < Regularizer
-      attr_accessor :l1_lambda
-
       # @param [Float] l1_lambda L1 regularizer coefficient.
       def initialize(l1_lambda = 0.01)
-        @l1_lambda = l1_lambda
+        @l1 = Layers::Lasso.new(l1_lambda)
       end
 
       def forward(x)
-        x + @l1_lambda * @param.data.abs.sum
+        x + @l1.(@param)
+      end
+
+      def l1_lambda
+        @l1.l1_lambda
       end
 
-      def backward
-        dparam = Xumo::SFloat.ones(*@param.data.shape)
-        dparam[@param.data < 0] = -1
-        @param.grad += @l1_lambda * dparam
+      def l1_lambda=(lam)
+        @l1.l1_lambda = lam
       end
 
       def to_hash
-        super(l1_lambda: @l1_lambda)
+        super(l1_lambda: l1_lambda)
       end
 
       def load_hash(hash)
@@ -60,23 +56,25 @@ module DNN
     end
 
     class L2 < Regularizer
-      attr_accessor :l2_lambda
-
       # @param [Float] l2_lambda L2 regularizer coefficient.
       def initialize(l2_lambda = 0.01)
-        @l2_lambda = l2_lambda
+        @l2 = Layers::Ridge.new(l2_lambda)
       end
 
       def forward(x)
-        x + 0.5 * @l2_lambda * (@param.data**2).sum
+        x + @l2.(@param)
+      end
+
+      def l2_lambda
+        @l2.l2_lambda
       end
 
-      def backward
-        @param.grad += @l2_lambda * @param.data
+      def l2_lambda=(lam)
+        @l2.l2_lambda = lam
       end
 
       def to_hash
-        super(l2_lambda: @l2_lambda)
+        super(l2_lambda: l2_lambda)
       end
 
       def load_hash(hash)
@@ -85,27 +83,31 @@ module DNN
     end
 
     class L1L2 < Regularizer
-      attr_accessor :l1_lambda
-      attr_accessor :l2_lambda
-
       # @param [Float] l1_lambda L1 regularizer coefficient.
       # @param [Float] l2_lambda L2 regularizer coefficient.
       def initialize(l1_lambda = 0.01, l2_lambda = 0.01)
-        @l1_lambda = l1_lambda
-        @l2_lambda = l2_lambda
+        @l1 = Layers::Lasso.new(l1_lambda)
+        @l2 = Layers::Ridge.new(l2_lambda)
       end
 
       def forward(x)
-        l1 = @l1_lambda * @param.data.abs.sum
-        l2 = 0.5 * @l2_lambda * (@param.data**2).sum
-        x + l1 + l2
+        x + @l1.(@param) + @l2.(@param)
+      end
+
+      def l1_lambda
+        @l1.l1_lambda
+      end
+
+      def l1_lambda=(lam)
+        @l1.l1_lambda = lam
+      end
+
+      def l2_lambda
+        @l2.l2_lambda
       end
 
-      def backward
-        dparam = Xumo::SFloat.ones(*@param.data.shape)
-        dparam[@param.data < 0] = -1
-        @param.grad += @l1_lambda * dparam
-        @param.grad += @l2_lambda * @param.data
+      def l2_lambda=(lam)
+        @l2.l2_lambda = lam
       end
 
       def to_hash

data/lib/dnn/core/tensor.rb

@@ -3,9 +3,49 @@ module DNN
     attr_reader :data
     attr_accessor :link
 
+    def self.convert(inputs)
+      if inputs.is_a?(Array)
+        inputs.map { |input| Tensor.new(input) }
+      else
+        Tensor.new(inputs)
+      end
+    end
+
     def initialize(data, link = nil)
       @data = data
       @link = link
     end
+
+    def shape
+      @data.shape
+    end
+
+    def +@
+      self
+    end
+
+    def -@
+      self * -1
+    end
+
+    def +(other)
+      Layers::Add.(self, other)
+    end
+
+    def -(other)
+      Layers::Sub.(self, other)
+    end
+
+    def *(other)
+      Layers::Mul.(self, other)
+    end
+
+    def /(other)
+      Layers::Div.(self, other)
+    end
+
+    def **(index)
+      Layers::Pow.new(index).(self)
+    end
   end
 end