red-chainer 0.2.1 → 0.3.0

data/lib/chainer/functions/math/identity.rb

@@ -0,0 +1,26 @@
+module Chainer
+  module Functions
+    module Math
+      # Identity function.
+      class Identity < Chainer::Function
+        def check_type_forward(in_types)
+          # pass
+        end
+
+        def forward(xs)
+          retain_inputs([])
+          return xs
+        end
+
+        def backward(xs, gys)
+          return gys
+        end
+
+        # Just returns input variables.
+        def self.identity(*inputs)
+          self.new.(*inputs)
+        end
+      end
+    end
+  end
+end

data/lib/chainer/functions/noise/dropout.rb

@@ -0,0 +1,45 @@
+module Chainer
+  module Functions
+    module Noise
+      class Dropout < Chainer::Function
+        # Drops elements of input variable randomly.
+        #
+        # This function drops input elements randomly with probability `ratio` and
+        # scales the remaining elements by factor `1 / (1 - ratio)`.
+        # In testing mode, it does nothing and just returns `x`.
+        #
+        # @param [Chainer::Variable] x Input variable.
+        # @param [float] ratio Dropout ratio. The ``ratio`` must be `0.0 <= ratio < 1.0`.
+        # @return [Chainer::Variable] Output variable.
+        def self.dropout(x, ratio: 0.5)
+          Chainer.configuration.train ? self.new(ratio).(x) : x
+        end
+
+        def initialize(dropout_ratio)
+          if dropout_ratio < 0 || dropout_ratio >= 1.0
+            raise 'dropout_ratio must be in the range [0, 1)'
+          end
+          @dropout_ratio = dropout_ratio
+        end
+
+        def forward(x)
+          retain_inputs([])
+          unless self.instance_variable_defined?(:@mask)
+            scale = x[0].class[*[1.0 / (1 - @dropout_ratio)]][0]
+            flag = x[0].class.new(*x[0].shape).rand >= @dropout_ratio
+
+            @mask = x[0].class.zeros(*x[0].shape)
+            @mask[flag] = 1
+            @mask *= scale
+          end
+          [x[0] * @mask]
+        end
+
+        def backward(x, gy)
+          [gy[0] * @mask]
+        end
+      end
+    end
+  end
+end
+

data/lib/chainer/functions/normalization/batch_normalization.rb

@@ -0,0 +1,136 @@
+module Chainer
+  module Functions
+    module Normalization
+      class BatchNormalizationFunction < Chainer::Function
+        attr_reader :running_mean, :running_var
+        # Batch normalization function with fixed statistics.
+        # This is a variant of batch normalization, where the mean and variance
+        # statistics are given by the caller as fixed variables. This is
+        # used on testing mode of the batch normalization layer, where batch
+        # statistics cannot be used for prediction consistency.
+        #
+        # @param [Chainer::Variable] x Input variable.
+        # @param [Chainer::Variable] gamma Scaling parameter of normalized data.
+        # @param [Chainer::Variable] beta Shifting parameter of scaled normalized data.
+        # @param [Chainer::Variable] mean Shifting parameter of input.
+        # @param [Chainer::Variable] var Square of scaling parameter of input.
+        # @param [float] eps Epsilon value for numerical stability.
+        def self.fixed_batch_normalization(x, gamma, beta, mean, var, eps: 2e-5)
+          old_train = Chainer.configuration.train
+          Chainer.configuration.train = false
+          norm = self.new(eps: eps, mean: nil, var: nil, decay: 0.0).(x, gamma, beta, mean, var)
+          Chainer.configuration.train = old_train
+          norm
+        end
+      
+        def initialize(eps: 2e-5, mean: nil, var: nil, decay: 0.9) 
+          @running_mean = mean
+          @running_var = var
+          @eps = eps
+          @mean_cache = nil
+          @decay = decay
+        end
+
+        def forward(inputs)
+          x, gamma, beta = inputs[0], inputs[1], inputs[2]
+          if Chainer.configuration.train
+            if @running_mean.nil?
+              @running_mean = Numo::NArray[*gamma].new_zeros
+              @running_var = Numo::NArray[*gamma].new_zeros
+            else
+              @running_mean = Numo::NArray[*@running_mean]
+              @running_var = Numo::NArray[*@running_var]
+            end
+          elsif inputs.size == 5
+            @fixed_mean = inputs[3]
+            @fixed_var = inputs[4]
+          end
+
+          head_ndim = gamma.ndim + 1
+          gamma_expander = [1] + gamma.shape + [1] * (x.ndim - head_ndim)
+          gamma = gamma.reshape(*gamma_expander)
+          beta_expander = [1] + beta.shape + [1] * (x.ndim - head_ndim)
+          beta = beta.reshape(*beta_expander)
+        
+          if Chainer.configuration.train
+            axis = [0] + (head_ndim...(x.ndim)).to_a
+            mean = x.mean(axis: axis)
+            # FIXME: numpy.var
+            var = x.var(axis: axis)
+            var += @eps
+          else
+            mean = @fixed_mean
+            var = @fixed_var + @eps
+          end
+
+          @std = Numo::NMath.sqrt(var)
+
+          mean_expander = [1] + mean.shape + [1] * (x.ndim - head_ndim)
+          x_mu = x - mean.reshape(*mean_expander)
+          std_expander = [1] + @std.shape + [1] * (x.ndim - head_ndim)
+          x_mu /= @std.reshape(*std_expander)
+          @x_hat = x_mu
+          y = gamma * @x_hat
+          y += beta
+
+          if Chainer.configuration.train
+            m = x.size.div(gamma.size)
+            adjust = m / [m - 1.0, 1.0].max
+            @running_mean *= @decay
+            temp_ar = Numo::NArray[*mean]
+            temp_ar *= (1 - @decay)
+            @running_mean += temp_ar
+            
+            @running_var *= @decay
+            temp_ar = Numo::NArray[*var]
+            temp_ar *= ((1 - @decay) * adjust)
+            @running_var += temp_ar
+          end
+
+          [y,]
+        end
+
+        def backward(inputs, grad_outputs)
+          x, gamma = inputs[0], inputs[1]
+          gy = grad_outputs[0]
+          head_ndim = gamma.ndim + 1
+          m = gamma.class[x.size.div(gamma.size)][0]
+          axis = [0] + (head_ndim...(x.ndim)).to_a
+
+          if inputs.size == 5
+            mean = inputs[3]
+            var = inputs[4]
+            std = Numo::NMath.sqrt(var)
+            gs = gamma / std
+            gbeta = gy.sum(axis: axis)
+
+            mean_expander = [1] + mean.shape + [1] * (x.ndim - head_ndim)
+            x_mu = x - mean.reshape(*mean_expander)
+            std_expander = [1] + std.shape + [1] * (x.ndim - head_ndim)
+            x_mu /= std.reshape(*std_expander)
+            x_hat = x_mu
+            ggamma = (gy * x_hat).sum(axis: axis)
+            gmean = -gs * gbeta
+            gvar = -0.5 * gamma / var * ggamma
+            gs_expander = [1] + gs.shape + [1] * (x.ndim - head_ndim)
+            gx = gs.reshape(*gs_expander)
+            return [gx, ggamma, gbeta, gmean, gvar]
+          end
+
+          gbeta = gy.sum(axis: axis)
+          ggamma = (gy * @x_hat).sum(axis: axis)
+          tmp = (gamma / @std)
+          tmp_expander = [1] + tmp.shape + [1] * (x.ndim - head_ndim)
+          tmp = tmp.reshape(*tmp_expander)
+
+          ggamma_expander = [1] + ggamma.shape + [1] * (x.ndim - head_ndim)
+          gbeta_expander = [1] + gbeta.shape + [1] * (x.ndim - head_ndim)
+          
+          gx = tmp * (gy - (@x_hat * ggamma.reshape(*ggamma_expander) + gbeta.reshape(*gbeta_expander)) / m )
+
+          [gx, ggamma, gbeta]
+        end
+      end
+    end
+  end
+end

data/lib/chainer/functions/pooling/max_pooling_2d.rb

@@ -0,0 +1,57 @@
+module Chainer
+  module Functions
+    module Pooling
+      class MaxPooling2D < Pooling2D
+        # Spatial max pooling function
+        #
+        # @param [Chainer::Variable] x Input variable
+        # @param [integer || 2D integer array] Size of pooling window
+        # @param [integer || 2D integer array] Stride of pooling applications
+        # @param [integer || 2D integer array] Spatial padding width for the input array
+        # @param [boolean] If `true`, all spatial locations are pooled int some output pixels
+        # @return [Chainer::Variable] Output variable
+        def self.max_pooling_2d(x, ksize, stride: nil, pad: 0, cover_all: true)
+          self.new(ksize, stride: stride, pad: pad, cover_all: cover_all).(x)
+        end
+
+        def forward_cpu(x)
+          retain_inputs([])
+          @in_shape = x[0].shape
+          @in_dtype = x[0].class
+
+          col = Chainer::Utils::Conv.im2col_cpu(x[0], @kh, @kw, @sy, @sx, @ph, @pw, pval: -Float::INFINITY, cover_all: @cover_all)
+          n, c, kh, kw, out_h, out_w = col.shape
+          col = col.reshape(n , c, kh * kw, out_h, out_w)
+
+          # TODO: numpy.argmax(axis=2)
+          d = col.shape[3..-1].reduce(:*) || 1
+          dx = col.shape[2..-1].reduce(:*) || 1
+          max_index = col.max_index(2)
+          @indexes = max_index.flatten.map_with_index { |val, idx| (val - (dx * (idx / d))) / d }.reshape(*max_index.shape)
+
+          y = col.max(axis: 2)
+          [y]
+        end
+
+        def backward_cpu(x, gy)
+          n, c, out_h, out_w = gy[0].shape
+          h, w  = @in_shape[2..-1]
+          kh, kw = @kh, @kw
+
+          gcol = @in_dtype.zeros(n * c * out_h * out_w * kh * kw)
+
+          indexes = @indexes.flatten
+          indexes += Numo::Int64.new((indexes.size * kh * kw) / (kh * kw)).seq(0, kh * kw)
+         
+          gcol[indexes] = gy[0].flatten.dup
+          gcol = gcol.reshape(n, c, out_h, out_w, kh, kw)
+          gcol = gcol.swapaxes(2, 4)
+          gcol = gcol.swapaxes(3, 5)
+
+          gx = Chainer::Utils::Conv.col2im_cpu(gcol, @sy, @sx, @ph, @pw, h, w)
+          [gx]
+        end
+      end
+    end
+  end
+end

data/lib/chainer/functions/pooling/pooling_2d.rb

@@ -0,0 +1,20 @@
+module Chainer
+  module Functions
+    module Pooling
+      # Base class of pooling function over a set of 2d planes
+      class Pooling2D < Chainer::Function
+        def initialize(ksize, stride: nil, pad: 0, cover_all: true)
+          if stride.nil?
+            stride = ksize
+          end
+
+          @kh, @kw = ksize.is_a?(Array) ? ksize : [ksize, ksize]
+          @sy, @sx = stride.is_a?(Array) ? stride : [stride, stride]
+          @ph, @pw = pad.is_a?(Array) ? pad: [pad, pad]
+
+          @cover_all = cover_all
+        end
+      end
+    end
+  end
+end

data/lib/chainer/gradient_check.rb

@@ -0,0 +1,240 @@
+module Chainer
+  def _copy_arrays(xs)
+    xp = Chainer::get_array_module(*xs)
+    xs.map{|x| (x.is_a? Numo::NArray) ? x.dup : x}
+  end
+
+  # Computes numerical gradient by finite differences.
+  #
+  # This function is used to implement gradient check. For usage example, see
+  # unit tests of Chainer::Functions.
+  #
+  # @param [function] f Ruby function with no arguments that runs forward
+  #   computation and returns the result.
+  # @param [Array<Arrays>] inputs Array of arrays that should be treated as
+  #   inputs. Each element of them is slightly modified to realize numerical
+  #   gradient by finite differences.
+  # @param [Array<Arrays>] grad_outputs Array of arrays that are treated as
+  #   output gradients.
+  # @param [Float] eps Epsilon value of finite differences.
+  # @return [Array] Numerical gradient arrays corresponding to +inputs+.
+  #
+  def numerical_grad(f, inputs, grad_outputs, eps=0.001)
+    raise unless eps > 0
+    inputs = inputs.to_a
+    grad_outputs = grad_outputs.to_a
+    xp = Numo::NArray
+    grads = inputs.map{|x| x.new_zeros()}
+
+    if inputs[0].ndim < 2
+      tmp = [[inputs[0], grads[0]]]
+    else
+      tmp = (0...inputs[0].shape[0]).map{|i|[inputs[0][i, false], grads[0][i, false]]}
+    end
+
+    tmp.each do |x, gx|
+      x.each_with_index{|xx, *i|
+        orig = x[*i]   # hold original value
+        x[*i] = orig + eps
+        ys1 = _copy_arrays(f.call(x))
+        x[*i] = orig - eps
+        ys2 = _copy_arrays(f.call(x))
+        x[*i] = orig
+
+        ys1.zip(ys2, grad_outputs).each do |y1, y2, gy|
+          if !gy.nil?
+            if  ((y1 - y2) * gy).is_a? Numo::NArray
+              dot = ((y1 - y2) * gy).sum()
+            else
+              dot = ((y1 - y2) * gy).inject(:+)
+            end
+            gx[*i] += dot / (2*eps).to_f
+          end
+        end
+      }
+    end
+
+    return grads
+  end
+
+  def _as_tuple(x)
+    if x.is_a? Array
+      return x
+    else
+      return [x]
+    end
+  end
+
+  # Test backward procedure of a given function.
+  #
+  # This function automatically check backward-process of given function.
+  # For example, when you have a +Chainer::Function+ class +MyFunc+,
+  # that gets two arguments and returns one value, you can make its test like this:
+  #
+  #   def test_my_func(self):
+  #     func = MyFunc()
+  #     x1_data = Numo::NArray[...]
+  #     x2_data = Numo::NArray[...]
+  #     gy_data = Numo::NArray[...]
+  #     check_backward(func, [x1_data, x2_data], gy_data)
+  #
+  # This method creates +Chainer::Variable+ objects with +x_data+
+  # and calls +func+ with the +Chainer::Variable+ s to get its result
+  # as +Chainer::Variable+.
+  # Then, it sets +y_grad+ array to +grad+ attribute of the result and
+  # calls +backward+ method to get gradients of the inputs.
+  # To check correctness of the gradients, the function calls
+  # +numerical_grad+ to calculate numerically the gradients and compares
+  # the types of gradients with +Chainer::Testing.assert_allclose+.
+  # If input objects (+x1_data+ or/and +x2_data+ in this example) represent
+  # integer variables, their gradients are ignored.
+  #
+  # You can simplify a test when +MyFunc+ gets only one argument:
+  #
+  #   check_backward(func, x1_data, gy_data)
+  #
+  # If +MyFunc+ is a loss function which returns a zero-dimensional
+  # array, pass +nil+ to +gy_data+. In this case, it sets +1+ to
+  # +grad+ attribute of the result:
+  #
+  #   check_backward(my_loss_func, [x1_data, x2_data], nil)
+  #
+  # If +MyFunc+ returns multiple outputs, pass all gradients for outputs as a Array:
+  #
+  #   gy1_data = Numo::NArray[...]
+  #   gy2_data = Numo::NArray[...]
+  #   check_backward(func, x1_data, [gy1_data, gy2_data])
+  #
+  # You can also test a +Chainer::Link+.
+  # To check gradients of parameters of the link, set a Array of the parameters
+  # to +params+ arguments:
+  #
+  #   check_backward(my_link, [x1_data, x2_data], gy_data, [my_link.W, my_link.b])
+  #
+  # Note that +params+ are not +Numo::NArray+ s,
+  # but +Chainer::Variables+ s.
+  #
+  # Function objects are acceptable as +func+ argument:
+  #
+  #   check_backward(lambda{|x1, x1| f(x1, x2)}, [x1_data, x2_data], gy_data)
+  #
+  # @note
+  #  +func+ is called many times to get numerical gradients for all inputs.
+  #  This function doesn't work correctly when +func+ behaves randomly as
+  #  it gets different gradients.
+  # @param [Method, Proc] func A function which gets +Chainer::Variable+ s
+  #   and returns +Chainer::Variable+ s. +func+ must returns
+  #   a Array of +Chainer::Variable+ s or one
+  #   +Chainer::Variable+. You can use +Chainer::Function+
+  #   object, +Chainer::Link+ object or a function satisfying the
+  #   condition.
+  # @param [Numo::NArray or Array<Numo::NArray>] x_data A set of +Numo::NArray+ s to be
+  #   passed to +func+. If +x_data+ is one +Numo::NArray+ object, it is
+  #   treated as +(x_data,)+.
+  # @param [Numo::NArray or Array<Numo::NArray> or nil] y_grad A set of +Numo::NArray+ s representing gradients of return-values of
+  #   +func+. If +y_grad+ is one +Numo::NArray+ object, it is
+  #   treated as +(y_grad,)+. If +func+ is a loss-function,
+  #   +y_grad+ should be set to +nil+.
+  # @param [Chainer::Variable or Array<Chainder::Variable>] params  A set of +Chainer::Variable+ s whose gradients are checked.
+  #   When +func+ is a +Chainer::Link+ object,
+  #   set its parameters as +params+.
+  #   If +params+ is one +Chainer::Variable+ object,
+  #   it is treated as +(params,)+.
+  # @param [Float] eps Epsilon value to be passed to +numerical_grad+.
+  # @param [Float] atol Absolute tolerance to be passed to +Chainer::Testing.assert_allclose+.
+  # @param [Float] rtol Relative tolerance to be passed to +Chainer::Testing.assert_allclose+.
+  # @param [Array<Boolean>] no_grads Flag to skip variable for gradient assertion.
+  #   It should be same length as +x_data+.
+  # @param [Numo::NArray.class] dtype +x_data+ and +y_grad+ are casted to this
+  #   dtype when calculating numerical gradients. Only float types and
+  #   +nil+ are allowed.
+  # @see
+  #   .numerical_grad
+  #
+  def check_backward(func, x_data, y_grad, params=[], eps: 0.001, atol: 1e-5, rtol: 1e-4, no_grads: nil, dtype: nil)
+    x_data = _as_tuple(x_data)
+    if !y_grad.nil?
+      y_grad = _as_tuple(y_grad)
+    end
+
+    params = _as_tuple(params)
+    xs = x_data.map{|x| Chainer::Variable.new(x)}
+    y = func.(*xs)
+    y = _as_tuple(y)
+    y = Chainer::Functions::Math::Identity.identity(*y)
+    y = _as_tuple(y)
+
+    if !y_grad.nil?
+      if (y).size != (y_grad).size
+        raise TypeError, "`y_grad` must have the same length of output values"
+      end
+
+      y.zip(y_grad).each do |iy, igy|
+        iy.grad = igy
+      end
+    else
+      if (y).size != 1
+        raise TypeError, "When `y_grad` is `nil`, the function must return azero-dimentional array"
+      end
+      y_grad = [1]
+    end
+    # We only need to call `backward` for one result `Chainer::Variable`.
+    # `Chainer::Variable.backward` method calls `Chainer::Function.backward` of its creator.
+    y[0].backward()
+
+    if dtype.nil?
+      casted_xs = x_data.map{|x| Chainer::Variable.new(x)}
+    else
+      if (dtype != Numo::DFloat) and (dtype != Numo::SFloat)
+        raise TypeError, "`dtype` is allowed only float type"
+      end
+      if (params).size > 0
+        raise TypeError, "`dtype` is available only if `params` is empty"
+      end
+      casted_xs = x_data.map{|x|
+                    if x.class == Numo::DFloat or x.class == Numo::SFloat
+                      Chainer::Variable.new(dtype.cast(x))
+                    else
+                      Chainer::Variable.new(x)
+                    end
+                  }
+    end
+
+    f = lambda do |_|
+      ys = func.(*casted_xs)
+      ys = _as_tuple(ys)
+      return ys.map{|y| y.data}.to_a
+    end
+
+    if no_grads.nil?
+      no_grads = xs.map{|x| (x.dtype != Numo::DFloat) and (x.dtype != Numo::SFloat)}
+    else
+      if no_grads.size != xs.size
+        raise TypeError, "Length of no_grads param and xs should be same."
+      end
+    end
+
+    no_grads.zip(xs, casted_xs).each do |skip, x, cx|
+      if skip
+        raise unless x.grad.nil?
+        next
+      end
+      gx, = numerical_grad(f, [cx.data], y_grad, eps)
+      Chainer::Testing.assert_allclose(x.grad, gx, atol: atol, rtol: rtol)
+      if dtype.nil?
+        raise unless gx.class == x.grad.class
+      else
+        if ((gx.class != Numo::DFloat) and (gx.class != Numo::SFloat)) and (gx.class != dtype)
+           raise
+        end
+      end
+    end
+
+    params.each do |p|
+      gp, = numerical_grad(f, [p.data], y_grad, eps)
+      Chainer::Testing.assert_allclose(p.grad, gp, atol: atol, rtol: rtol)
+      raise unless gp.dtype === p.grad.dtype
+    end
+  end
+  module_function :_copy_arrays, :numerical_grad, :_as_tuple, :check_backward
+end