ruby-dnn 0.15.3 → 0.16.0

data/lib/dnn/core/layers/basic_layers.rb

@@ -1,6 +1,28 @@
 module DNN
   module Layers
 
+    module LayerNode
+      def forward(input_tensor)
+        x = input_tensor.data
+        prev_link = (input_tensor.is_a?(Tensor) ? input_tensor.link : input_tensor)
+        y = forward_node(x)
+        link = Link.new(prev_link, self)
+        Tensor.new(y, link)
+      end
+
+      def backward(dy)
+        backward_node(dy)
+      end
+
+      def forward_node(x)
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'forward_node'"
+      end
+
+      def backward_node(dy)
+        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'backward_node'"
+      end
+    end
+
     # Super class of all layer classes.
     class Layer
       attr_reader :input_shape
@@ -26,12 +48,9 @@ module DNN
       # @param [Tensor] input_tensor Input tensor.
       # @return [Tensor] Output tensor.
       def call(input_tensor)
-        x = input_tensor.data
-        prev_link = input_tensor.link
-        build(x.shape[1..-1]) unless built?
-        y = forward(x)
-        link = Link.new(prev_link, self)
-        Tensor.new(y, link)
+        input_tensor = Tensor.new(input_tensor) if !input_tensor.is_a?(Tensor) && !input_tensor.is_a?(Param)
+        build(input_tensor.data.shape[1..-1]) unless built?
+        forward(input_tensor)
       end
 
       # Build the layer.
@@ -47,17 +66,12 @@ module DNN
       end
 
       # Forward propagation.
-      # @param [Numo::SFloat] x Input data.
-      def forward(x)
+      # @param [Tensor] input_tensor Input tensor.
+      # @return [Tensor] Output tensor.
+      def forward(input_tensor)
         raise NotImplementedError, "Class '#{self.class.name}' has implement method 'forward'"
       end
 
-      # Backward propagation.
-      # @param [Numo::SFloat] dy Differential value of output data.
-      def backward(dy)
-        raise NotImplementedError, "Class '#{self.class.name}' has implement method 'backward'"
-      end
-
       # Please reimplement this method as needed.
       # The default implementation return input_shape.
       # @return [Array] Return the shape of the output data.
@@ -76,6 +90,7 @@ module DNN
         initialize
       end
 
+      # Clean the layer state.
       def clean
         input_shape = @input_shape
         hash = to_hash
@@ -120,6 +135,8 @@ module DNN
     end
 
     class InputLayer < Layer
+      include LayerNode
+
       def self.call(input)
         shape = input.is_a?(Tensor) ? input.data.shape : input.shape
         new(shape[1..-1]).(input)
@@ -140,21 +157,21 @@ module DNN
           x = input
           prev_link = nil
         end
-        Tensor.new(forward(x), Link.new(prev_link, self))
+        Tensor.new(forward_node(x), Link.new(prev_link, self))
       end
 
       def build(input_shape)
         @built = true
       end
 
-      def forward(x)
+      def forward_node(x)
         unless x.shape[1..-1] == @input_shape
           raise DNN_ShapeError, "The shape of x does not match the input shape. input shape is #{@input_shape}, but x shape is #{x.shape[1..-1]}."
         end
         x
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dy
       end
 
@@ -248,6 +265,8 @@ module DNN
     end
 
     class Dense < Connection
+      include LayerNode
+
       attr_reader :num_nodes
 
       # @param [Integer] num_nodes Number of nodes.
@@ -273,14 +292,14 @@ module DNN
         init_weight_and_bias
       end
 
-      def forward(x)
+      def forward_node(x)
         @x = x
         y = x.dot(@weight.data)
         y += @bias.data if @bias
         y
       end
 
-      def backward(dy)
+      def backward_node(dy)
         if @trainable
           @weight.grad += @x.transpose.dot(dy)
           @bias.grad += dy.sum(0) if @bias
@@ -307,11 +326,13 @@ module DNN
     end
 
     class Flatten < Layer
-      def forward(x)
+      include LayerNode
+
+      def forward_node(x)
         x.reshape(x.shape[0], *output_shape)
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dy.reshape(dy.shape[0], *@input_shape)
       end
 
@@ -321,6 +342,8 @@ module DNN
     end
 
     class Reshape < Layer
+      include LayerNode
+
       attr_reader :output_shape
 
       def initialize(output_shape)
@@ -328,11 +351,11 @@ module DNN
         @output_shape = output_shape
       end
 
-      def forward(x)
+      def forward_node(x)
         x.reshape(x.shape[0], *@output_shape)
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dy.reshape(dy.shape[0], *@input_shape)
       end
 
@@ -345,7 +368,69 @@ module DNN
       end
     end
 
+    class Lasso < Layer
+      include LayerNode
+
+      attr_accessor :l1_lambda
+
+      # @param [Float] l1_lambda L1 regularizer coefficient.
+      def initialize(l1_lambda = 0.01)
+        super()
+        @l1_lambda = l1_lambda
+      end
+
+      def forward_node(x)
+        @x = x
+        @l1_lambda * x.abs.sum
+      end
+
+      def backward_node(dy)
+        dx = Xumo::SFloat.ones(*@x.shape)
+        dx[@x < 0] = -1
+        @l1_lambda * dx
+      end
+
+      def to_hash
+        super(l1_lambda: @l1_lambda)
+      end
+
+      def load_hash(hash)
+        initialize(hash[:l1_lambda])
+      end
+    end
+
+    class Ridge < Layer
+      include LayerNode
+
+      attr_accessor :l2_lambda
+
+      # @param [Float] l2_lambda L2 regularizer coefficient.
+      def initialize(l2_lambda = 0.01)
+        super()
+        @l2_lambda = l2_lambda
+      end
+
+      def forward_node(x)
+        @x = x
+        0.5 * @l2_lambda * (x**2).sum
+      end
+
+      def backward_node(dy)
+        @l2_lambda * @x
+      end
+
+      def to_hash
+        super(l2_lambda: @l2_lambda)
+      end
+
+      def load_hash(hash)
+        initialize(hash[:l2_lambda])
+      end
+    end
+
     class Dropout < Layer
+      include LayerNode
+
       attr_accessor :dropout_ratio
       attr_reader :use_scale
 
@@ -361,7 +446,7 @@ module DNN
         @rnd = Random.new(@seed)
       end
 
-      def forward(x)
+      def forward_node(x)
         if DNN.learning_phase
           Xumo::SFloat.srand(@rnd.rand(1 << 31))
           @mask = Xumo::SFloat.new(*x.shape).rand < @dropout_ratio
@@ -372,7 +457,7 @@ module DNN
         x
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dy[@mask] = 0
         dy
       end

data/lib/dnn/core/layers/cnn_layers.rb

@@ -84,6 +84,7 @@ module DNN
     end
 
     class Conv2D < Connection
+      include LayerNode
       include Conv2DUtils
 
       attr_reader :num_filters
@@ -130,7 +131,7 @@ module DNN
         @out_size = calc_conv2d_out_size(prev_h, prev_w, *@filter_size, *@pad_size, @strides)
       end
 
-      def forward(x)
+      def forward_node(x)
         x = zero_padding(x, @pad_size) if @padding
         @x_shape = x.shape
         @col = im2col(x, *@out_size, *@filter_size, @strides)
@@ -139,7 +140,7 @@ module DNN
         y.reshape(x.shape[0], *@out_size, y.shape[3])
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dy = dy.reshape(dy.shape[0..2].reduce(:*), dy.shape[3])
         if @trainable
           @weight.grad += @col.transpose.dot(dy)
@@ -186,6 +187,7 @@ module DNN
     end
 
     class Conv2DTranspose < Connection
+      include LayerNode
       include Conv2DUtils
 
       attr_reader :num_filters
@@ -232,7 +234,7 @@ module DNN
         @out_size = calc_conv2d_transpose_out_size(prev_h, prev_w, *@filter_size, *@pad_size, @strides)
       end
 
-      def forward(x)
+      def forward_node(x)
         bsize = x.shape[0]
         x = x.reshape(x.shape[0..2].reduce(:*), x.shape[3])
         @x = x
@@ -243,7 +245,7 @@ module DNN
         @padding ? zero_padding_bwd(y, @pad_size) : y
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dy = zero_padding(dy, @pad_size) if @padding
         col = im2col(dy, *input_shape[0..1], *@filter_size, @strides)
         if @trainable
@@ -291,6 +293,7 @@ module DNN
 
     # Super class of all pooling2D class.
     class Pool2D < Layer
+      include LayerNode
       include Conv2DUtils
 
       attr_reader :pool_size
@@ -345,7 +348,9 @@ module DNN
     end
 
     class MaxPool2D < Pool2D
-      def forward(x)
+      include LayerNode
+
+      def forward_node(x)
         x = zero_padding(x, @pad_size) if @padding
         @x_shape = x.shape
         col = im2col(x, *@out_size, *@pool_size, @strides)
@@ -354,7 +359,7 @@ module DNN
         col.max(1).reshape(x.shape[0], *@out_size, x.shape[3])
       end
 
-      def backward(dy)
+      def backward_node(dy)
         dmax = Xumo::SFloat.zeros(dy.size * @pool_size.reduce(:*))
         dmax[@max_index.flatten] = dy.flatten
         dcol = dmax.reshape(dy.shape[0..2].reduce(:*), @pool_size.reduce(:*) * dy.shape[3])
@@ -364,7 +369,9 @@ module DNN
     end
 
     class AvgPool2D < Pool2D
-      def forward(x)
+      include LayerNode
+
+      def forward_node(x)
         x = zero_padding(x, @pad_size) if @padding
         @x_shape = x.shape
         col = im2col(x, *@out_size, *@pool_size, @strides)
@@ -372,7 +379,7 @@ module DNN
         col.mean(1).reshape(x.shape[0], *@out_size, x.shape[3])
       end
 
-      def backward(dy)
+      def backward_node(dy)
         row_length = @pool_size.reduce(:*)
         dy /= row_length
         davg = Xumo::SFloat.zeros(dy.size, row_length)
@@ -391,24 +398,15 @@ module DNN
           raise DNN_ShapeError, "Input shape is #{input_shape}. But input shape must be 3 dimensional."
         end
         super
-        @avg_pool2d = AvgPool2D.new(input_shape[0..1])
-        @avg_pool2d.build(input_shape)
-        @flatten = Flatten.new
-        @flatten.build([1, 1, input_shape[2]])
       end
 
       def forward(x)
-        y = @avg_pool2d.forward(x)
-        @flatten.forward(y)
-      end
-
-      def backward(dy)
-        dy = @flatten.backward(dy)
-        @avg_pool2d.backward(dy)
+        Flatten.(AvgPool2D.(x, input_shape[0..1]))
       end
     end
 
     class UnPool2D < Layer
+      include LayerNode
       include Conv2DUtils
 
       attr_reader :unpool_size
@@ -432,7 +430,7 @@ module DNN
         @num_channel = input_shape[2]
       end
 
-      def forward(x)
+      def forward_node(x)
         @x_shape = x.shape
         unpool_h, unpool_w = @unpool_size
         x2 = Xumo::SFloat.zeros(x.shape[0], x.shape[1], unpool_h, x.shape[2], unpool_w, @num_channel)
@@ -444,7 +442,7 @@ module DNN
         x2.reshape(x.shape[0], *@out_size, x.shape[3])
       end
 
-      def backward(dy)
+      def backward_node(dy)
         in_size = input_shape[0..1]
         col = im2col(dy, *in_size, *@unpool_size, @unpool_size)
         col = col.reshape(dy.shape[0] * in_size.reduce(:*), @unpool_size.reduce(:*), dy.shape[3])

data/lib/dnn/core/layers/embedding.rb

@@ -24,7 +24,7 @@ module DNN
 
       def call(input_tensor)
         build(@input_shape) unless built?
-        Tensor.new(forward(input_tensor.data), Link.new(nil, self))
+        Tensor.new(forward_node(input_tensor.data), Link.new(nil, self))
       end
 
       def build(input_shape)
@@ -34,7 +34,7 @@ module DNN
         @weight_regularizer.param = @weight if @weight_regularizer
       end
 
-      def forward(x)
+      def forward_node(x)
         @x = x
         y = Xumo::SFloat.zeros(*x.shape)
         x.shape[0].times do |i|
@@ -43,7 +43,7 @@ module DNN
         y
       end
 
-      def backward(dy)
+      def backward_node(dy)
         @weight.grad += Xumo::SFloat.zeros(*@weight.data.shape)
         @x.shape[0].times do |i|
           @x.shape[1].times do |j|

data/lib/dnn/core/layers/math_layers.rb

@@ -0,0 +1,169 @@
+module DNN
+  module Layers
+
+    class Add < MergeLayer
+      def forward_node(x1, x2)
+        x1 + x2
+      end
+
+      def backward_node(dy)
+        [dy, dy]
+      end
+    end
+
+    class Sub < MergeLayer
+      def forward_node(x1, x2)
+        x1 - x2
+      end
+
+      def backward_node(dy)
+        [dy, -dy]
+      end
+    end
+
+    class Mul < MergeLayer
+      def forward_node(x1, x2)
+        @x1, @x2 = x1, x2
+        x1 * x2
+      end
+
+      def backward_node(dy)
+        [dy * @x2, dy * @x1]
+      end
+    end
+
+    class Div < MergeLayer
+      def forward_node(x1, x2)
+        @x1, @x2 = x1, x2
+        x1 / x2
+      end
+
+      def backward_node(dy)
+        dx1 = dy / @x2
+        dx2 = dy * -(@x1 / @x2**2)
+        [dx1, dx2]
+      end
+    end
+
+    class Dot < MergeLayer
+      def forward_node(x1, x2)
+        @x1, @x2 = x1, x2
+        x1.dot(x2)
+      end
+
+      def backward_node(dy)
+        [dy.dot(@x2.transpose), @x1.transpose.dot(dy)]
+      end
+    end
+
+    class Exp < Layer
+      include LayerNode
+
+      def forward_node(x)
+        @x = x
+        Xumo::NMath.exp(x)
+      end
+
+      def backward_node(dy)
+        dy * Xumo::NMath.exp(@x)
+      end
+    end
+
+    class Log < Layer
+      include LayerNode
+
+      def forward_node(x)
+        @x = x
+        Xumo::NMath.log(x)
+      end
+
+      def backward_node(dy)
+        dy / @x
+      end
+    end
+
+    class Pow < Layer
+      include LayerNode
+
+      def initialize(index)
+        super()
+        @index = index
+      end
+
+      def forward_node(x)
+        @x = x
+        x**@index
+      end
+
+      def backward_node(dy)
+        @index * @x**(@index - 1)
+      end
+    end
+
+    class Sqrt < Layer
+      include LayerNode
+
+      def forward_node(x)
+        @x = x
+        Xumo::NMath.sqrt(x)
+      end
+
+      def backward_node(dy)
+        dy * (1.0 / 2 * Xumo::NMath.sqrt(@x))
+      end
+    end
+
+    class Sum < Layer
+      include LayerNode
+
+      def initialize(axis: 0)
+        super()
+        @axis = axis
+      end
+
+      def forward_node(x)
+        if @axis
+          @dim = x.shape[@axis]
+          x.sum(axis: @axis, keepdims: true)
+        else
+          x.sum
+        end
+      end
+
+      def backward_node(dy)
+        dx = dy.clone
+        if @axis
+          (@dim - 1).times do
+            dx = dx.concatenate(dy, axis: @axis)
+          end
+        end
+        dx
+      end
+    end
+
+    class Mean < Layer
+      include LayerNode
+
+      def initialize(axis: 0)
+        super()
+        @axis = axis
+      end
+
+      def forward_node(x)
+        @dim = @axis ? x.shape[@axis] : x.size
+        x.mean(axis: @axis, keepdims: true)
+      end
+
+      def backward_node(dy)
+        dx = dy
+        if @axis
+          (@dim - 1).times do
+            dx = dx.concatenate(dy, axis: @axis)
+          end
+        end
+        dx / @dim
+      end
+    end
+
+  end
+end