red-chainer 0.3.2 → 0.4.0

data/lib/chainer/functions/activation/leaky_relu.rb

@@ -2,7 +2,7 @@ module Chainer
   module Functions
     module Activation
       # Leaky rectifier unit.
-      class LeakyReLU < Function
+      class LeakyReLU < FunctionNode
         # Leaky Rectified Linear Unit function.
         #
         # This function is expressed as
@@ -13,7 +13,7 @@ module Chainer
         # 
         # where $a$ is a configurable slope value.
         # 
-        # @param [Chainer::Variable or Numo::NArray] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
+        # @param [Chainer::Variable or Numo::NArray or Cumo::NArray] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @param [float] slope Slope value $a$.
         # @return [Chainer::Variable] Output variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @example
@@ -31,32 +31,57 @@ module Chainer
         #    [-0.4, 1]]
         #
         def self.leaky_relu(x, slope: 0.2)
-          self.new(slope: slope).(x)
+          self.new(slope: slope).apply([x])[0]
         end
 
         def initialize(slope:0.2)
           @slope = slope
         end
 
-        def forward_cpu(x)
-          y = x[0].dup()
-          y[x[0] < 0] *= @slope
+        def forward(inputs)
+          x, = inputs
+          y = x.dup
+          y[x < 0] *= @slope
           if @slope >= 0
-            retain_inputs([])
             retain_outputs([0])
+          else
+            retain_inputs([0])
           end
           [y]
         end
 
-        def backward_cpu(x, gy)
-          gx = gy[0].dup()
+        def backward(indexes, grad_outputs)
+          if @slope >= 0
+            x = nil
+            y = get_retained_outputs.first.data
+          else
+            x = get_retained_inputs.first.data
+            y = nil
+          end
+          LeakyReLUGrad.new(x, y, @slope).apply(grad_outputs)
+        end
+      end
+
+      class LeakyReLUGrad < FunctionNode
+        def initialize(x, y, slope)
+          @x = x
+          @y = y
+          @slope = slope
+        end
+
+        def forward(inputs)
+          gy, = inputs
+          gy = gy.dup
           if @slope >= 0
-            y = @output_data
-            gx[y[0] < 0] *= @slope
+            gy[@y < 0] *= @slope
           else
-            gx[x[0] < 0] *= @slope
+            gy[@x < 0] *= @slope
           end
-          [gx]
+          [gy]
+        end
+
+        def backward(indexes, grad_outputs)
+          LeakyReLUGrad.new(@x, @y, @slope).apply(grad_outputs)
         end
       end
     end

data/lib/chainer/functions/activation/log_softmax.rb

@@ -2,11 +2,12 @@ module Chainer
   module Functions
     module Activation
       def self.logsumexp(x)
+        xm = Chainer.get_array_module(x)
         m = x.max(axis: 1, keepdims: true)
         y = x - m
-        y = Numo::NMath.exp(y)
+        y = xm::NMath.exp(y)
         s = y.sum(axis: 1, keepdims: true)
-        s = Numo::NMath.log(s)
+        s = xm::NMath.log(s)
         m + s
       end
 
@@ -16,7 +17,7 @@ module Chainer
       end
 
       # Log-softmax activation function.
-      class LogSoftmax < Function
+      class LogSoftmax < FunctionNode
         # Channel-wise log-softmax function.
         #
         # This function computes its logarithm of softmax along the second axis.
@@ -36,7 +37,7 @@ module Chainer
         #   because +softmax(x)+ may returns +0+.
         #   +log_softmax+ method is more stable.
         #
-        # @param [Chainer::Variable or Numo::NArray] x Input variable. A $n$-dimensional ($n \\geq 2$) float array.
+        # @param [Chainer::Variable or Numo::NArray or Cumo::NArray] x Input variable. A $n$-dimensional ($n \\geq 2$) float array.
         # @return [Chainer::Variable] Output variable. A $n$-dimensional ($n \\geq 2$) float array, which is the same shape with x.
         #
         # @see Chainer::Functions::Softmax
@@ -56,23 +57,59 @@ module Chainer
         #   => true
         #
         def self.log_softmax(x)
-          self.new.(x)
+          self.new.apply([x]).first
         end
 
         def forward(xs)
           y = Chainer::Functions::Activation._log_softmax(xs[0])
           @x_shape = xs[0].shape
           @x_dtype = xs[0].class
-          retain_inputs([])
           retain_outputs([0])
           [y]
         end
 
-        def backward(x, gy)
-          y = @output_data[0]
-          gx = gy[0] - Numo::NMath.exp(y) * gy[0].sum(axis: 1, keepdims: true)
+        def backward(indexes, gy)
+          y = get_retained_outputs.first
+          LogSoftmaxGrad.new(@x_shape, @x_dtype).apply([y, gy[0]])
+        end
+      end
+
+      class LogSoftmaxGrad < FunctionNode
+        def initialize(x_shape, x_dtype)
+          @x_shape = x_shape
+          @x_dtype = x_dtype
+        end
+
+        def forward(inputs)
+          retain_inputs([0, 1])
+          y, gy = inputs
+
+          xm = Chainer.get_array_module(y)
+          gx = gy - xm::NMath.exp(y) * gy.sum(axis: 1, keepdims: true)
           [gx]
         end
+
+        def backward(indexes, ggx)
+          y, gy = get_retained_inputs
+          ret = []
+          exp_y = Chainer::Functions::Math::Exp.exp(y)
+
+          if indexes.include?(0)
+            gy_sum = Chainer::Functions::Math::Sum.sum(gy, axis: 1, keepdims: true)
+            gy_sum = Chainer::Functions::Array::BroadcastTo.broadcast_to(gy_sum, gy.shape)
+
+            g0 = -ggx.first * exp_y * gy_sum
+            ret << g0
+          end
+          if indexes.include?(1)
+            a = Chainer::Functions::Math::Sum.sum(ggx.first * exp_y, axis: 1, keepdims: true)
+            a = Chainer::Functions::Array::BroadcastTo.broadcast_to(a, gy.shape)
+            g1 = ggx.first - a
+            ret << g1
+          end
+
+          ret
+        end
       end
     end
   end

data/lib/chainer/functions/activation/relu.rb

@@ -2,14 +2,14 @@ module Chainer
   module Functions
     module Activation
       # Rectified Linear Unit.
-      class Relu < Function
+      class Relu < FunctionNode
         # Rectified Linear Unit function.
         #
         # $$
         # f(x)=\\max(0, x).
         # $$
         #
-        # @param [Chainer::Variable or Numo::NArray] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
+        # @param [Chainer::Variable or Numo::NArray or Cumo::NArray] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @return [Chainer::Variable] Output variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @example
         #   > x = Numo::SFloat[[-1, 0], [2, -3], [-2, 1]]
@@ -23,18 +23,18 @@ module Chainer
         #   => [3, 2]
         #
         def self.relu(x)
-          self.new.(x)
+          y, = self.new.apply([x])
+          y
         end
 
-        def forward_cpu(x)
-          retain_inputs([])
+        def forward(x)
           retain_outputs([0])
           [Utils::Array.force_array(x[0].class.maximum(x[0], 0))]
         end
 
-        def backward_cpu(x, gy)
-          y = @output_data[0]
-          [Utils::Array.force_array(gy[0] * (y > 0))]
+        def backward(indexes, gy)
+          y = get_retained_outputs.first
+          ReLUGrad2.new(y).apply([gy[0]])
         end
       end
     end

data/lib/chainer/functions/activation/relu_grad2.rb

@@ -0,0 +1,34 @@
+module Chainer
+  module Functions
+    module Activation
+      # Computes the gradient of the ReLU function.
+      #
+      # This function takes 2 variables b and c, and
+      # computes f(b, c) = sign(b) * c with backpropagation
+      # where operations are dones in elementwise manner
+      # and sign(x) = 1 when x > 0 is positive and 0 otherwise.
+      # As the gradient of f with respect to b is 0,
+      # we do not backpropagate errors toward b for computational efficiency.<Paste>
+      class ReLUGrad2 < FunctionNode
+        def initialize(b)
+          @b = b.data
+        end
+
+        def forward(inputs)
+          y = inputs[0] * (@b > 0)
+          [Utils::Array.force_array(y, y.class)]
+        end
+
+        def backward(indexes, gy)
+          [gy[0] * heaviside(@b)]
+        end
+
+        private
+
+        def heaviside(x)
+          (x > 0).cast_to(x.class)
+        end
+      end
+    end
+  end
+end

data/lib/chainer/functions/activation/sigmoid.rb

@@ -2,14 +2,14 @@ module Chainer
   module Functions
     module Activation
       # Logistic sigmoid function.
-      class Sigmoid < Function
+      class Sigmoid < FunctionNode
         # Element-wise sigmoid logistic function.
         #
         # $$
         # f(x)=(1 + \\exp(-x))^ { -1 }.
         # $$
         #
-        # @param [Chainer::Variable or Numo::NArray] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
+        # @param [Chainer::Variable or Numo::NArray or Cumo::NArray] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @return [Chainer::Variable] Output variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @example  It maps the input values into the range of $`[0, 1]`$.
         #   > x = Numo::SFloat.new(3).seq(-2, 2)
@@ -21,21 +21,23 @@ module Chainer
         #   [0.119203, 0.5, 0.880797]
         #
         def self.sigmoid(x)
-          self.new.(x)
+          self.new.apply([x]).first
         end
 
-        def forward_cpu(x)
+        def forward(inputs)
+          x, = inputs
           half = 0.5
-          y = Utils::Array.force_array((Numo::NMath.tanh(x[0] * half) * half)+ half)
-          retain_inputs([])
+          xm = Chainer.get_array_module(x)
+          y = Utils::Array.force_array((xm::NMath.tanh(x * half) * half)+ half)
           retain_outputs([0])
-          return [y]
+          [y]
         end
 
-        def backward_cpu(x, gy)
-          one = 1
-          y = @output_data[0]
-          [Utils::Array.force_array((gy[0] * y) * (one - y))]
+        def backward(indexes, grad_outputs)
+          x = nil
+          y = get_retained_outputs.first
+          gy, = grad_outputs
+          Chainer::Functions::Activation::SigmoidGrad.new([x]).apply([y, gy])
         end
       end
     end

data/lib/chainer/functions/activation/sigmoid_grad.rb

@@ -0,0 +1,25 @@
+module Chainer
+  module Functions
+    module Activation
+      # Logistic sigmoid gradient function.
+      class SigmoidGrad < FunctionNode
+        def initialize(inputs)
+          @x, = inputs
+        end
+
+        def forward(inputs)
+          retain_inputs([0, 1])
+          y, gy = inputs
+          one = 1
+          [Utils::Array.force_array(gy * y * (one - y))]
+        end
+
+        def backward(indexes, grad_outputs)
+          y, gy = get_retained_inputs
+          g, = grad_outputs
+          [g * gy * ( 1 -2 * y), g * y * (1 - y)]
+        end
+      end
+    end
+  end
+end

data/lib/chainer/functions/activation/tanh.rb

@@ -2,14 +2,14 @@ module Chainer
   module Functions
     module Activation
       # Hyperbolic tangent function.
-      class Tanh < Function
+      class Tanh < FunctionNode
         # Elementwise hyperbolic tangent function.
         #
         # $$
         # f(x)=\\tanh(x).
         # $$
         #
-        # @param [Chainer::Variable or Numo::NArray] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
+        # @param [Chainer::Variable or Numo::NArray or Cumo::NArray] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @return [Chainer::Variable] Output variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @example
         #   > x = Numo::SFloat.new(3).seq(-1, 2)
@@ -21,20 +21,57 @@ module Chainer
         #   [-0.761594, 0.761594, 0.995055]
         #
         def self.tanh(x)
-          self.new.(x)
+          self.new.apply([x]).first
         end
 
-        def forward_cpu(x)
-          y = Utils::Array.force_array(Numo::NMath.tanh(x[0]))
-          retain_inputs([])
+        def forward(x)
+          xm = Chainer.get_array_module(x[0])
+          y = Utils::Array.force_array(xm::NMath.tanh(x[0]))
           retain_outputs([0])
-          return [y]
+          @use_cudnn = false
+          [y]
         end
 
-        def backward_cpu(x, gy)
-          y = @output_data[0]
-          one = y.class.cast(1)
-          [Utils::Array.force_array(gy[0] * (one - y * y))]
+        def backward(indexes, grad_outputs)
+          if @use_cudnn
+            x = get_retained_inputs.first.data
+          else
+            x = nil
+          end
+
+          y = get_retained_outputs.first
+          gy = grad_outputs.first
+          TanhGrad.new(x).apply([y, gy])
+        end
+      end
+
+      class TanhGrad < FunctionNode
+        def initialize(x)
+          super()
+
+          # The original input `x` is only required for cuDNN.
+          # If it is None, this class does not use cuDNN.
+          # Note that x must be c-contiguous and it is checked
+          # in Tanh.forward_gpu.
+          @x = x
+        end
+
+        def forward(inputs)
+          retain_inputs([0, 1])
+          y, gy = inputs
+
+          one = y.class.new.fill(1)
+          [Utils::Array.force_array(gy * (one - y * y))]
+        end
+
+        def backward(indexes, grad_outputs)
+          y, gy = get_retained_inputs
+          g = grad_outputs[0]
+
+          y_mul_g = y * g
+          grad_y = -2 * gy * y_mul_g
+          ggy = g - y * y_mul_g
+          [grad_y, ggy]
         end
       end
     end

data/lib/chainer/functions/array/broadcast_to.rb

@@ -0,0 +1,56 @@
+module Chainer
+  module Functions
+    module Array
+      # Function that broadcasts an array to a new shape.
+      class BroadcastTo < FunctionNode
+        def initialize(shape)
+            @shape = shape
+        end
+
+        def self.broadcast_to(x, shape)
+          return Chainer::Variable.as_variable(x) if x.shape == shape
+          self.new(shape).apply([x]).first
+        end
+
+        def forward(inputs)
+          x = inputs.first
+          [Chainer::Utils::Array.broadcast_to(x, @shape)]
+        end
+
+        def backward(indexes, grad_outputs)
+          gx = grad_outputs.first
+          shape = @inputs.first.shape
+          ndim = shape.size
+          lead = gx.ndim - ndim
+          lead_axis = lead.times.to_a
+          axis = shape.each_with_object([]).with_index do |(sx, res), i|
+            next unless sx == 1
+            res << i + lead
+          end
+          gx = Chainer::Functions::Math::Sum.sum(gx, axis: lead_axis + axis, keepdims: true)
+          return [Chainer::Functions::Array::Squeeze.squeeze(gx, axis: lead_axis)] if lead > 0
+          [gx]
+        end
+
+        private
+
+        def backward_one(shape, dtype, g)
+          return dtype.zeros(shape) unless g
+
+          ndim = shape.size
+          if g.ndim != ndim
+            g = g.sum(axis: 0...(g.ndim - ndim))
+          end
+
+          axis = shape.each_with_index.select{|sx, i| sx == 1 }.map{|sx, i| i }
+          if axis.size > 0
+            g.sum(keepdims: true, axis: axis)
+          else
+            g
+          end
+        end
+      end
+    end
+  end
+end
+