red-chainer 0.2.1 → 0.3.0

data/lib/chainer/datasets/mnist.rb

@@ -3,7 +3,7 @@ require 'zlib'
 module Chainer
   module Datasets
     module Mnist
-      def self.get_mnist(withlabel: true, ndim: 1, scale: 1.0, dtype: Numo::DFloat, label_dtype: Numo::Int32)
+      def self.get_mnist(withlabel: true, ndim: 1, scale: 1.0, dtype: Numo::SFloat, label_dtype: Numo::Int32)
         train_raw = retrieve_mnist_training
         train = preprocess_mnist(train_raw, withlabel, ndim, scale, dtype, label_dtype)
 
@@ -15,9 +15,9 @@ module Chainer
       def self.preprocess_mnist(raw, withlabel, ndim, scale, image_dtype, label_dtype)
         images = raw[:x]
         if ndim == 2
-          images = images.reshape(-1, 28, 28)
+          images = images.reshape(true, 28, 28)
         elsif ndim == 3
-          images = images.reshape(-1, 1, 28, 28)
+          images = images.reshape(true, 1, 28, 28)
         elsif ndim != 1
           raise "invalid ndim for MNIST dataset"
         end

data/lib/chainer/datasets/tuple_dataset.rb

@@ -16,7 +16,9 @@ module Chainer
       end
 
       def [](index)
-        batches = @datasets.map { |dataset| dataset.ndim > 1 ? dataset[index, 0...dataset.shape[1]] : dataset[index] }
+        batches = @datasets.map do |dataset|
+          dataset.ndim > 1 ? dataset[index, false] : dataset[index]
+        end
         if index.kind_of?(Enumerable)
           length = batches[0].shape[0]
           length.times.map {|i| batches.map { |m| m[i] } }

data/lib/chainer/function.rb

@@ -23,6 +23,7 @@ module Chainer
       @input_indexes_to_retain = nil
       @output_indexes_to_retain = nil
       outputs = forward(in_data)
+      raise if !outputs.is_a? Array
 
       ret = outputs.map do |y|
         Variable.new(y, requires_grad: requires_grad)

data/lib/chainer/functions/activation/leaky_relu.rb

@@ -13,19 +13,19 @@ module Chainer
         # 
         # where $a$ is a configurable slope value.
         # 
-        # @param [Chainer::Variable or Numo::DFloat] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
+        # @param [Chainer::Variable or Numo::NArray] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @param [float] slope Slope value $a$.
         # @return [Chainer::Variable] Output variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @example
-        #   > x = Numo::DFloat[[-1, 0], [2, -3], [-2, 1]]
+        #   > x = Numo::SFloat[[-1, 0], [2, -3], [-2, 1]]
         #   > x
-        #   => Numo::DFloat#shape=[3,2]
+        #   => Numo::SFloat#shape=[3,2]
         #   [[-1, 0], 
         #    [2, -3], 
         #    [-2, 1]]
         #   > F = Chainer::Functions::Activation::LeakyReLU
         #   > F.leaky_relu(x, slope:0.2).data
-        #   => Numo::DFloat#shape=[3,2]
+        #   => Numo::SFloat#shape=[3,2]
         #   [[-0.2, 0], 
         #    [2, -0.6], 
         #    [-0.4, 1]]

data/lib/chainer/functions/activation/log_softmax.rb

@@ -36,19 +36,19 @@ module Chainer
         #   because +softmax(x)+ may returns +0+.
         #   +log_softmax+ method is more stable.
         #
-        # @param [Chainer::Variable or Numo::DFloat] x Input variable. A $n$-dimensional ($n \\geq 2$) float array.
+        # @param [Chainer::Variable or Numo::NArray] x Input variable. A $n$-dimensional ($n \\geq 2$) float array.
         # @return [Chainer::Variable] Output variable. A $n$-dimensional ($n \\geq 2$) float array, which is the same shape with x.
         #
         # @see Chainer::Functions::Softmax
         #
         # @example
-        #   > x = Numo::DFloat[[0, 1, 2], [0, 2, 4]]
-        #   => Numo::DFloat#shape=[2,3]
+        #   > x = Numo::SFloat[[0, 1, 2], [0, 2, 4]]
+        #   => Numo::SFloat#shape=[2,3]
         #   [[0, 1, 2],
         #    [0, 2, 4]]
         #   > F = Chainer::Functions::Activation::LogSoftmax
         #   > F.log_softmax(x).data
-        #   => Numo::DFloat#shape=[2,3]
+        #   => Numo::SFloat#shape=[2,3]
         #   [[-2.40761, -1.40761, -0.407606],
         #    [-4.14293, -2.14293, -0.142932]]
         # @example (T.B.I : F.log, F.softmax)

data/lib/chainer/functions/activation/relu.rb

@@ -9,10 +9,10 @@ module Chainer
         # f(x)=\\max(0, x).
         # $$
         #
-        # @param [Chainer::Variable or Numo::DFloat] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
+        # @param [Chainer::Variable or Numo::NArray] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @return [Chainer::Variable] Output variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @example
-        #   > x = Numo::DFloat[[-1, 0], [2, -3], [-2, 1]]
+        #   > x = Numo::SFloat[[-1, 0], [2, -3], [-2, 1]]
         #   > (x < 0).any?
         #   => true
         #   > F = Chainer::Functions::Activation::Relu
@@ -29,8 +29,7 @@ module Chainer
         def forward_cpu(x)
           retain_inputs([])
           retain_outputs([0])
-          x[0][x[0]<=0] = 0
-          [Utils::Array.force_array(x[0])] 
+          [Utils::Array.force_array(x[0].class.maximum(x[0], 0))]
         end
 
         def backward_cpu(x, gy)

data/lib/chainer/functions/activation/sigmoid.rb

@@ -9,15 +9,15 @@ module Chainer
         # f(x)=(1 + \\exp(-x))^ { -1 }.
         # $$
         #
-        # @param [Chainer::Variable or Numo::DFloat] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
+        # @param [Chainer::Variable or Numo::NArray] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @return [Chainer::Variable] Output variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @example  It maps the input values into the range of $`[0, 1]`$.
-        #   > x = Numo::DFloat.new(3).seq(-2, 2)
-        #   => Numo::DFloat#shape=[3]
+        #   > x = Numo::SFloat.new(3).seq(-2, 2)
+        #   => Numo::SFloat#shape=[3]
         #   [-2, 0, 2]
         #   > F = Chainer::Functions::Activation::Sigmoid
         #   > F.sigmoid(x).data
-        #   => Numo::DFloat#shape=[3]
+        #   => Numo::SFloat#shape=[3]
         #   [0.119203, 0.5, 0.880797]
         #
         def self.sigmoid(x)

data/lib/chainer/functions/activation/tanh.rb

@@ -9,15 +9,15 @@ module Chainer
         # f(x)=\\tanh(x).
         # $$
         #
-        # @param [Chainer::Variable or Numo::DFloat] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
+        # @param [Chainer::Variable or Numo::NArray] x Input variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @return [Chainer::Variable] Output variable. A $(s_1, s_2, ..., s_N)$-shaped float array.
         # @example
-        #   > x = Numo::DFloat.new(3).seq(-1, 2)
-        #   => Numo::DFloat#shape=[3]
+        #   > x = Numo::SFloat.new(3).seq(-1, 2)
+        #   => Numo::SFloat#shape=[3]
         #   [-1, 1, 3]
         #   > F = Chainer::Functions::Activation::Tanh
         #   > F.tanh(x).data
-        #   => Numo::DFloat#shape=[3]
+        #   => Numo::SFloat#shape=[3]
         #   [-0.761594, 0.761594, 0.995055]
         #
         def self.tanh(x)
@@ -33,7 +33,7 @@ module Chainer
 
         def backward_cpu(x, gy)
           y = @output_data[0]
-          one = y.dtype.type(1)
+          one = y.class.cast(1)
           [Utils::Array.force_array(gy[0] * (one - y * y))]
         end
       end

data/lib/chainer/functions/connection/convolution_2d.rb

@@ -0,0 +1,92 @@
+module Chainer
+  module Functions
+    module Connection
+      class Convolution2DFunction < Chainer::Function
+        # Two-dimensional convolution function.
+        # This is an implementation of two-dimensional convolution in ConvNets.
+        # It takes three variables: the input image `x`, the filter weight `w`, and the bias vector `b`.
+        #
+        # a notation for dimensionalities.
+        #
+        # - :math:`n` is the batch size.
+        # - :math:`c_I` and :math:`c_O` are the number of the input and output channels, respectively.
+        # - :math:`h_I` and :math:`w_I` are the height and width of the input image, respectively.
+        # - :math:`h_K` and :math:`w_K` are the height and width of the filters, respectively.
+        # - :math:`h_P` and :math:`w_P` are the height and width of the spatial padding size, respectively.
+        #
+        # Then the `Convolution2D` function computes correlations between filters and patches of size :math:`(h_K, w_K)` in `x`.
+        # Patches are extracted at positions shifted by multiples of `stride` from the first position `(-h_P, -w_P)` for each spatial axis.
+        # The right-most (or bottom-most) patches do not run over the padded spatial size.
+        # Let :math:`(s_Y, s_X)` be the stride of filter application.
+        # Then, the output size :math:`(h_O, w_O)` is determined by the following equations:
+        # 
+        # math:
+        #  h_O &= (h_I + 2h_P - h_K) / s_Y + 1,\\\\
+        #   w_O &= (w_I + 2w_P - w_K) / s_X + 1.
+        # If `cover_all` option is `true`, the filter will cover the all spatial locations.
+        # So, if the last stride of filter does not cover the end of spatial locations,
+        # an addtional stride will be applied to the end part of spatial locations.
+        # In this case, the output size :math:`(h_O, w_O)` is determined by the following equations:
+        # 
+        # math:
+        #  h_O &= (h_I + 2h_P - h_K + s_Y - 1) / s_Y + 1,\\\\
+        #  w_O &= (w_I + 2w_P - w_K + s_X - 1) / s_X + 1.
+        # If the bias vector is given, then it is added to all spatial locations of the output of convolution.
+        #
+        # @param [Chainer::Variable or Numo::NArray] x Input variable of shape :math:`(n, c_I, h_I, w_I)`.
+        # @param [Chainer::Variable or Numo::NArray] w Weight variable of shape :math:`(c_O, c_I, h_K, w_K)`.
+        # @param [Chainer::Variable or Numo::NArray] b Bias variable of length :math:`c_O`
+        # @param [Int or 2-D Array] stride Stride of filter applications. `stride=s` and `stride=(s, s)` are equivalent.
+        # @param [Int or 2-D Array] pad Spatial padding width for input arrays.
+        # @param [Boolean] cover_all If `true`, all spatial locations are convoluted into some output pixels.
+        # @return [Chainer::Variable] Output variable of shape :math:`(n, c_O, h_O, w_O)`.
+        def self.convolution_2d(x, w, b: nil, stride: 1, pad: 0, cover_all: false)
+          func = self.new(stride: stride, pad: pad, cover_all: cover_all)
+          if b.nil?
+              func.(x, w)
+          else
+              func.(x, w, b)
+          end
+        end
+
+        def initialize(stride: 1, pad: 0, cover_all: false)
+          @sy, @sx = stride.is_a?(Array) ? stride : [stride, stride]
+          @ph, @pw = pad.is_a?(Array) ? pad : [pad, pad]
+          @cover_all = cover_all
+        end
+
+        def forward_cpu(inputs)
+          x = inputs[0]
+          w = inputs[1]
+          b = inputs.size == 3 ? inputs[2] : nil
+
+          kh, kw = w.shape[2], w.shape[3]
+
+          @col = Chainer::Utils::Conv.im2col_cpu(x, kh, kw, @sy, @sx, @ph, @pw, cover_all: @cover_all)
+          y = Chainer::Utils::Math.tensordot(@col, w, [[1, 2, 3], [1, 2, 3]])
+          y += b if b
+          
+          [y.transpose(0, 3, 1, 2)]
+        end
+
+        def backward_cpu(inputs, grad_outputs)
+          x, w, b = inputs[0], inputs[1], inputs[2]
+          gy = grad_outputs[0]
+          height, width = x.shape[2..-1]
+
+          gw = Chainer::Utils::Math.tensordot(gy, @col, [[0, 2, 3], [0, 4, 5]])
+          gcol = Chainer::Utils::Math.tensordot(w, gy, [0, 1])
+          gcol = gcol.transpose(3, 0, 1, 2)
+          gx = Chainer::Utils::Conv.col2im_cpu(gcol, @sy, @sx, @ph, @pw, height, width)
+
+          if b.nil?
+            [gx, gw]
+          else
+            gb = gy.sum(axis: [0, 2, 3])
+            [gx, gw, gb]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chainer/functions/connection/linear.rb

@@ -40,7 +40,7 @@ module Chainer
 
         def as_mat(x)
           return x if x.ndim == 2
-          x.reshape(x.size, -1)
+          x.reshape(x.shape.first, true)
         end
       end
     end

data/lib/chainer/functions/loss/mean_squared_error.rb

@@ -0,0 +1,34 @@
+module Chainer
+  module Functions
+    module Loss
+      # Mean squared error (a.k.a. Euclidean loss) function.
+      class MeanSquaredError < Function
+        # Mean squared error function.
+        #
+        # This function computes mean squared error between two variables. The mean
+        # is taken over the minibatch. Note that the error is not scaled by 1/2.
+        #
+        # @param [Chainer::Variable or Numo::NArray] x0 Input variable.
+        # @param [Chainer::Variable or Numo::NArray] x1 Input variable.
+        # @return [Chainer::Variable] A variable holding an array representing the mean squared error of two inputs.
+        #
+        def self.mean_squared_error(x0, x1)
+          self.new.(x0, x1)
+        end
+
+        def forward_cpu(inputs)
+          x0, x1 = inputs
+          @diff = x0 - x1
+          diff = @diff.flatten.dup()
+          [diff.class.cast(diff.dot(diff) / diff.size)]
+        end
+
+        def backward(inputs, gy)
+          coeff = gy[0] * gy[0].class.cast(2.0 / @diff.size)
+          gx0 = coeff * @diff
+          [gx0, -(gx0)]
+        end
+      end
+    end
+  end
+end

data/lib/chainer/functions/loss/softmax_cross_entropy.rb

@@ -13,17 +13,17 @@ module Chainer
 
           unless class_weight.nil?
             if @class_weight.ndim != 1
-              raise ArgumentError 'class_weight.ndim should be 1'
-            elsif @class_weight.dtype != Numo::DFloat
-              raise ArgumentError 'The dtype of class_weight should be \'Numo::DFloat\''
+              raise ArgumentError, 'class_weight.ndim should be 1'
+            elsif (@class_weight.class != Numo::DFloat) and (@class_weight.class != Numo::SFloat)
+              raise ArgumentError, "The dtype of class_weight should be 'Numo::DFloat' or 'Numo::SFloat'"
             elsif @class_weight.kind_of?(Chainer::Variable)
-              raise ArgumentError 'class_weight should be a Numo::NArray, not a chainer.Variable'
+              raise ArgumentError, 'class_weight should be a Numo::NArray, not a chainer.Variable'
             end
           end
 
           @ignore_label = ignore_label
           unless ['mean', 'no'].include?(reduce)
-            raise ArgumentError "only 'mean' and 'no' are valid for 'reduce', but #{reduce} is given"
+            raise ArgumentError, "only 'mean' and 'no' are valid for 'reduce', but #{reduce} is given"
           end
 
           @reduce = reduce
@@ -37,40 +37,37 @@ module Chainer
             @y = Numo::NMath.exp(log_y)
           end
           if @class_weight
-            shape = x.ndim.times.map { |e| e == 1 ? -1 : 1 } 
-            log_y += broadcast_to(@class_weight.reshape(*shape), x.shape)
+            shape = x.ndim.times.map { |e| e == 1 ? true : 1 }
+            log_y *= Chainer::Functions::Loss.broadcast_to(@class_weight.reshape(*shape), x.shape)
           end
-          log_yd = rollaxis(log_y, 1)
+          log_yd = Chainer::Functions::Loss.rollaxis(log_y, 1)
           begin
-            log_yd = log_yd.reshape(log_yd.size, -1)
+            log_yd = log_yd.reshape(log_yd.shape[0], true)
           rescue ArgumentError
           end
-
           ravel_arr = t.dup.flatten.dup
           ravel_arr[ravel_arr<0] = 0
           arange_arr = t.class.new(t.size).seq
 
           # https://github.com/chainer/chainer/blob/v2.0.2/chainer/functions/loss/softmax_cross_entropy.py#L79
           log_p = []
-          arange_arr.each do |col_idx|
-            log_p << log_yd[ravel_arr, col_idx][col_idx]
+          ravel_arr.each_with_index do |r, i|
+            log_p << log_yd[r, i]
           end
-          log_p = Numo::NArray.[](*log_p)
-
-          log_p[log_p.eq(@ignore_label)] = 0
+          log_p = log_yd.class.[](*log_p)
+          log_p[t.flatten.dup.eq(@ignore_label)] = 0
 
           if @reduce == 'mean'
             if @normalize
               count = t.ne(@ignore_label).count
             else
-              count = x.size
+              count = x.shape[0]
             end
             @coeff = 1.0 / [count, 1].max
-
             y = log_p.sum(keepdims: true) * (-@coeff)
-            [y.reshape(())]
+            [y.class.cast(y[0])]
           else
-            [-log_p.reshape(t.shape)]
+            [-log_p.reshape(*t.shape)]
           end
         end
 
@@ -87,48 +84,78 @@ module Chainer
 
           if y.ndim == 2
             gx = y
-            t[t<0] = 0
-            t.each_with_index do |v, idx|
-              gx[(idx * 10)...(idx * 10 + 10)][v] -= 1
-            end
+            t.class.new(t.shape[0]).seq(0).to_a.zip(t.class.maximum(t, 0).to_a).each{|v| gx[*v] -= 1}
 
             if @class_weight
-              shape = x.ndim.times.map { |d| d == 1 ? -1 : 1 }
-              c = broadcast_to(@class_weight.reshape(shape), x.shape)
-              c = c[Numo::DFloat.new(t.size).seq, t]
-              gx *= broadcast_to(t.expand_dims(1), gx.shape)
+              shape = x.ndim.times.map { |d| d == 1 ? true : 1 }
+              c = Chainer::Functions::Loss.broadcast_to(@class_weight.reshape(*shape), x.shape)
+              c = c.class.cast(t.class.new(t.shape[0]).seq.to_a.zip(t.class.maximum(t, 0).to_a).map{|v| c[*v]})
+              gx *= Chainer::Functions::Loss.broadcast_to(c.expand_dims(1), gx.shape)
             end
 
             bit = t.flatten.dup
             bit[t.ne(@ignore_label)] = 1
             bit[bit.ne(1)] = 0
-            gx *= bit.reshape(t.size, 1)
+            gx *= bit.reshape(t.shape[0], 1)
           else
-            raise 'TODO: ndim > 2 backward'
+            # in the case where y.ndim is higher than 2,
+            # we think that a current implementation is inefficient
+            # because it yields two provisional arrays for indexing.
+
+            n_unit = t.size / t.shape[0]
+            gx = y.reshape(y.shape[0], y.shape[1], true)
+            fst_index = Numo::Int32.new(t.size).seq(0) / n_unit
+            trd_index = Numo::Int32.new(t.size).seq(0) % n_unit
+            fst_index.to_a.zip(t.class.maximum(t.flatten.dup, 0).to_a, trd_index.to_a).each{|v| gx[*v] -= 1}
+            if @class_weight
+              shape = x.ndim.times.map{|d| d == 1 ? true : 1}
+              c = Chainer::Functions::Loss.broadcast_to(@class_weight.reshape(*shape), x.shape)
+              c = c.reshape(*gx.shape)
+              c = c.class.cast(fst_index.to_a.zip(t.class.maximum(t.flatten.dup, 0).to_a, trd_index.to_a).map{|v| c[*v]})
+              c = c.reshape(y.shape[0], 1, true)
+              gx *= Chainer::Functions::Loss.broadcast_to(c, gx.shape)
+            end
+            gx *= (t.ne @ignore_label).reshape(t.shape[0], 1, true)
+            gx = gx.reshape(*y.shape)
           end
 
           if @reduce == 'mean'
             gx *= gloss * @coeff
           else
-            raise 'TODO: reduce'
+            gx *= gloss[true,:- , false]
           end
           return [gx, nil]
         end
+      end
 
+      def rollaxis(y, axis, start: 0)
+        axes = (0...y.ndim).to_a
+        axes.delete_at(axis)
+        axes.insert(start <= axes.size ? start : -1, axis)
+        y.transpose(*axes)
+      end
 
-        private
-
-        def broadcast_to(array, shape)
-          array.class.tile(array, shape[0]).reshape(*shape)
+      def broadcast_to(array, shape)
+        if array.shape.size > shape.size
+           raise TypeError, "Shape of data  mismatch\n array.shape.size(#{array.shape.size}) > shape.size(#{shape.size})"
         end
 
-        def rollaxis(y, axis, start: 0)
-          axes = (0...y.ndim).to_a
-          axes.delete_at(axis)
-          axes.insert(start, axis)
-          y.transpose(*axes)
-        end
+        tile_shape = []
+        shape_check = shape[-array.shape.size..-1]
+        shape_check.each_with_index{|s, i|
+          if array.shape[i] == 1
+            tile_shape << s
+          elsif array.shape[i] == s
+            tile_shape << 1
+          else
+            raise TypeError, "Shape of data  mismatch\n#{array.shape} != #{shape}"
+          end
+        }
+
+        array.tile(*shape[0...-array.shape.size], *tile_shape)
       end
+
+      module_function :rollaxis, :broadcast_to
     end
   end
 end