red-chainer 0.3.2 → 0.4.0

data/lib/chainer/testing/array.rb

@@ -22,7 +22,7 @@ module Chainer
       end
 
       actual.each_with_index{|actual_val, *i|
-        if (expect[*i].to_f - actual_val.to_f).abs > atol + rtol * expect[*i].abs
+        if (expect[*i].to_f - actual_val.to_f).abs > atol + rtol * expect[*i].to_f.abs
           raise "assert_allclose Error\n  expect: #{expect.inspect}\n  actual : #{actual.inspect}\n    (#{i})=> #{(expect - actual).abs.max()} > #{atol + rtol * expect[*i].abs}"
         end
       }

data/lib/chainer/training/extensions/evaluator.rb

@@ -34,7 +34,7 @@ module Chainer
         #                               If this is just a link object, the link is registered by the name 'main'.
         # @param [Dataset::Convert] converter Converter function to build input arrays.
         #             												`Chainer::Dataset.concat_examples` is used by default.
-        # @param [integer] device Device to which the training data is sent. Negative value indicates the host memory (CPU).
+        # @param [Chainer::Device] device Device to which the training data is sent.
         # @param [Function] eval_hook Function to prepare for each evaluation process.
         #															It is called at the beginning of the evaluation.
         # 														The evaluator extension object is passed at each call.
@@ -97,8 +97,7 @@ module Chainer
         # @return dict Result dictionary. This dictionary is further reported via `Chainer.save_report` without specifying any observer.
         def evaluate
           iterator = @iterators[:main]
-          target = @targets[:main]
-          eval_func = @eval_func || target
+          eval_func = @eval_func || @targets[:main]
 
           @eval_hook.(self) if @eval_hook
 

data/lib/chainer/training/extensions/exponential_shift.rb

@@ -57,7 +57,7 @@ module Chainer
         def serialize(serializer)
           @t = serializer.('t', @t)
           @last_value = serializer.('last_value', @last_value)
-          if @last_value.is_a?(Numo::NArray)
+          if Chainer.array?(@last_value)
             @last_value = @last_value[0]
           end
         end

data/lib/chainer/training/extensions/progress_bar.rb

@@ -1,4 +1,5 @@
 require 'erb'
+require 'time'
 
 module Chainer
   module Training

data/lib/chainer/training/trainer.rb

@@ -1,12 +1,11 @@
 module Chainer
   module Training
     class ExtensionEntry
-      attr_accessor :extension, :trigger, :invoke_before_training, :priority
+      attr_accessor :extension, :trigger, :priority
 
-      def initialize(extension, priority, trigger, invoke_before_training)
+      def initialize(extension, priority, trigger)
         @extension = extension
         @trigger = trigger
-        @invoke_before_training = invoke_before_training
         @priority = priority
       end
     end
@@ -47,7 +46,7 @@ module Chainer
         Time.now.to_f - @start_at + @snapshot_elapsed_time.to_f
       end
 
-      def extend(extension, name: nil, trigger: nil, priority: nil, invoke_before_training: nil)
+      def extend(extension, name: nil, trigger: nil, priority: nil)
         if name.nil?
           name = if extension.name
                    extension.name
@@ -69,10 +68,6 @@ module Chainer
           priority = extension.methods.include?(:priority) ? extension.priority : Extension::PRIORITY_READER
         end
 
-        if invoke_before_training.nil?
-          invoke_before_training = extension.methods.include?(:invoke_before_training) ? extension.invoke_before_training : false
-        end
-
         modified_name = name
         ordinal = 0
 
@@ -82,7 +77,7 @@ module Chainer
         end
 
         extension.name = modified_name
-        @extensions[modified_name] = ExtensionEntry.new(extension, priority, trigger, invoke_before_training)
+        @extensions[modified_name] = ExtensionEntry.new(extension, priority, trigger)
       end
 
       def get_extension(name)

data/lib/chainer/training/triggers/interval.rb

@@ -13,6 +13,11 @@ module Chainer
           @previous_epoch_detail = 0.0
         end
 
+        # Decides whether the extension should be called on this iteration.
+        #
+        # @param [Chainer::Trainer] trainer Trainer object that this trigger is associated with.
+        #   The updater associated with this trainer is used to determine if the trigger should fire.
+        # @return [boolean]  True if the corresponding extension should be invoked in this iteration.
         def call(trainer)
           updater = trainer.updater
           if @unit == 'epoch'
@@ -30,7 +35,7 @@ module Chainer
             iteration = updater.iteration
             previous_iteration = @previous_iteration
             if previous_iteration < 0
-                previous_iteration = iteration - 1
+              previous_iteration = iteration - 1
             end
             fire = previous_iteration.div(@period).floor != iteration.div(@period).floor
           end
@@ -38,7 +43,7 @@ module Chainer
           # save current values
           @previous_iteration = updater.iteration
           @previous_epoch_detail = updater.epoch_detail
-        
+
           fire
         end
 

data/lib/chainer/utils/array.rb

@@ -4,7 +4,8 @@ module Chainer
       def self.force_array(x, dtype=nil)
         if x.is_a? Integer or x.is_a? Float
           if dtype.nil?
-            Numo::NArray.cast(x)
+            xm = Chainer::Device.default.xm
+            xm::NArray.cast(x)
           else
             dtype.cast(x.dup)
           end
@@ -16,6 +17,84 @@ module Chainer
           end
         end
       end
+
+      def self.take(x, indices, axis: nil)
+        if axis
+          indices = make_indecies_with_axis(x.shape, indices, axis)
+        end
+        x[indices]
+      end
+
+      def self.make_indecies_with_axis(shape, indices, axis, values = [])
+        target_axis = values.size
+        if shape.size == values.size
+          values.zip(shape.drop(1) + [1]).reduce(0) do |sum, (x, ndim)|
+            (sum + x) * ndim
+          end
+        else
+          enum = (axis == target_axis) ? indices : (0...shape[target_axis])
+          if enum.is_a?(Integer)
+            make_indecies_with_axis(shape, indices, axis, values + [indices])
+          else
+            enum.map do |x|
+              make_indecies_with_axis(shape, indices, axis, values + [x])
+            end
+          end
+        end
+      end
+
+      def self.rollaxis(y, axis, start: 0)
+        n = y.ndim
+        # normalize axis
+        axis = axis < 0 ? n + axis : axis
+        if axis >= n
+          raise ArgumentError, "axis #{axis} is out of bounds for array of dimension #{n}"
+        end
+
+        if start < 0
+          start += n
+        end
+
+        unless 0 <= start && start < n + 1
+          raise ArgumentError, "start arg requires #{-n} <= start < #{n}, but #{start} was passed in"
+        end
+
+        if axis < start
+          start -= 1
+        end
+
+        if axis == start
+          return y
+        end
+
+        axes = (0...n).to_a
+        axes.delete_at(axis)
+        axes.insert(start <= axes.size ? start : -1, axis)
+        y.transpose(*axes)
+      end
+
+      def self.broadcast_to(x, shape)
+        if x.shape.size > shape.size
+           raise TypeError, "Shape of data  mismatch\n x.shape.size(#{x.shape.size}) > shape.size(#{shape.size})"
+        end
+
+        tile_shape = []
+        if x.shape.size > 0
+          shape[-x.shape.size..-1].each_with_index do |s, i|
+            if  x.shape[i] == 1
+              tile_shape << s
+            elsif x.shape[i] == s
+              tile_shape << 1
+            else
+              raise TypeError, "Shape of data  mismatch\n#{x.shape} != #{shape}"
+            end
+          end
+        else
+          tile_shape = shape
+        end
+
+        x.tile(*shape[0...-x.shape.size], *tile_shape)
+      end
     end
   end
 end

data/lib/chainer/utils/conv.rb

@@ -10,7 +10,15 @@ module Chainer
         end
       end
 
-      def self.im2col_cpu(img, kh, kw, sy, sx, ph, pw, pval: 0, cover_all: false, dy: 1, dx: 1)
+      def self.get_deconv_outsize(size, k, s, p, cover_all: false)
+        if cover_all
+          s * (size - 1) + k -s + 1 - 2 * p
+        else
+          s * (size - 1) + k - 2 * p
+        end
+      end
+
+      def self.im2col(img, kh, kw, sy, sx, ph, pw, pval: 0, cover_all: false, dy: 1, dx: 1)
         n, c, h, w = img.shape
 
         out_h = self.get_conv_outsize(h, kh, sy, ph, cover_all: cover_all, d: dy)
@@ -40,7 +48,7 @@ module Chainer
         col
       end
 
-      def self.col2im_cpu(col, sy, sx, ph, pw, h, w, dy: 1, dx: 1)
+      def self.col2im(col, sy, sx, ph, pw, h, w, dy: 1, dx: 1)
         n, c, kh, kw, out_h, out_w = col.shape
         img = col.class.zeros(n, c, h + 2 * ph + sy - 1, w + 2 * pw + sx - 1)
         kh.times do |j|

data/lib/chainer/utils/initializer.rb

@@ -1,10 +1,10 @@
 module Chainer
   module Utils
     module Initializer
-      def self.get_fans(shape)
+      def self.get_fans(shape, device: Chainer::Device.default)
         raise 'shape must be of length >= 2: shape={}' if shape.size < 2
         slice_arr = shape.slice(2, shape.size)
-        receptive_field_size = slice_arr.empty? ? 1 : Numo::Int32[slice_arr].prod
+        receptive_field_size = slice_arr.empty? ? 1 : device.xm::Int32[slice_arr].prod
         fan_in = shape[1] * receptive_field_size
         fan_out = shape[0] * receptive_field_size
         [fan_in, fan_out]

data/lib/chainer/variable.rb

@@ -2,25 +2,47 @@ module Chainer
   class Variable
     attr_accessor :data, :grad, :requires_grad, :node
 
+    # Converts an array or a variable into +Chainer::Variable+.
+    # This is a convenient function to get a +Chainer::Variable+ object
+    # transparently from a raw array or a variable.
+    # Note: that this function should only be used for type consistency
+    # (i.e. to enforce the return value of an API having type +Chainer::Variable+).
+    # The +Chianer::Variable.requires_grad+ flag is kept as is; if +obj+ is a raw array,
+    # the newly created variable has +requires_grad = false+.
+    # In order to make a variable w.r.t. which you want to compute the gradient,
+    # you should use $Chainer::Variable$ directly.
+    #
+    # @param [Numo::NArray or Chainer::Variable] obj An array or a variable that you want to convert to $Chainer::Variable$.
+    # @return [Chainer::Variable] A variable converted from +obj+. If +obj+ is a raw array,
+    #   this is a new +Chianer::Variable+ object that wraps the array. If +obj+ is already a +Chainer::Variable+ object, this function returns +obj+ as is.
+    def self.as_variable(obj)
+      return obj if obj.kind_of?(Chainer::Variable)
+      # TODO if obj is_backprop_required is true, set requires_grad = true
+      self.new(obj, requires_grad: false)
+    end
+
     def initialize(data=nil, name: nil, grad: nil, requires_grad: true)
-      unless data.nil? || data.is_a?(Numo::NArray)
-        raise TypeError, "Numo::NArray are expected."
+      unless data.nil? || Chainer.array?(data)
+        raise TypeError, "Numo::NArray or Cumo::NArray are expected."
       end
 
       @data = [data]
       @grad = grad
       @requires_grad = requires_grad
-      @node = VariableNode.new(variable: self, name: name, grad: grad)
+      @node = VariableNode.new(variable: self, name: name)
+      @grad_var = grad.nil? ? nil : Chainer::Variable.new(grad)
     end
 
     def data
       return @data[0]
     end
+    alias_method :array, :data
 
     def data=(d)
       @data[0] = d
       @node.set_data_type(d)
     end
+    alias_method :array=, :data=
 
     def name
       return @node.name
@@ -34,6 +56,7 @@ module Chainer
       @node.label
     end
 
+    # deprecated FunctionNode
     def creator
       @node.creator
     end
@@ -42,12 +65,30 @@ module Chainer
       @node.creator = func
     end
 
+    def creator_node
+      @node.creator_node
+    end
+
+    def creator_node=(func)
+      @node.creator_node = func
+    end
+
     def grad
-      @node.grad
+      gv = @grad_var
+      gv.nil? ? nil : gv.data
     end
 
     def grad=(g)
-      @node.set_grad_with_check(g, nil, self)
+      self.grad_var = g.nil? ? nil : Chainer::Variable.new(g)
+    end
+
+    def grad_var
+      @grad_var
+    end
+
+    def grad_var=(g)
+      Utils::Variable.check_grad_type(nil, self, g.data) unless g.nil?
+      @grad_var = g
     end
 
     def shape
@@ -70,123 +111,172 @@ module Chainer
       @node.rank
     end
 
+    def transpose
+      Chainer::Functions::Array::Transpose.transpose(self)
+    end
+
+    def reshape(*shape)
+      if shape.size == 1 && shape[0].kind_of?(::Aray)
+        shape = shape[0]
+      end
+      Chainer::Functions::Array::Reshape.reshape(self, shape)
+    end
+
+    # Clears the gradient array.
     def cleargrad
-      @node.grad = nil
+      @grad_var = nil
+    end
+
+    # Notifies the variable that the given node is its creator.
+    #
+    # @param [Chainer::FunctionNode] fnode node that has this variable as an output.
+    def set_creator_node(fnode)
+      @node.set_creator_node(fnode)
     end
 
-    def backward(retain_grad: false)
-      return if self.creator.nil?
+    def backward(retain_grad: false, enable_double_backprop: true)
+      old_enable_backprop = Chainer.configuration.enable_backprop
+      Chainer.configuration.enable_backprop = enable_double_backprop
+      _backward_main(retain_grad)
+      Chainer.configuration.enable_backprop = old_enable_backprop
+    end
+
+    def _backward_main(retain_grad)
+      node.check_old_style_gradient
+      return if self.creator_node.nil?
 
-      if self.data.size == 1 && self.grad.nil?
+      seen_set = Set.new
+      grads = {}
+      if self.data.size == 1 && self.grad_var.nil?
         self.grad = self.data.new_ones
       end
+      grads[self.node] = self.grad_var
 
-      funcs = [self.creator]
+      funcs = [self.creator_node]
+      seen_set.add(self.creator_node)
 
-      while func = funcs.pop
+      while func = funcs.shift
+        inputs = func.inputs
+        target_input_indexes = inputs.each_with_index.map { |x, i| i if x.requires_grad }.compact
+        next if target_input_indexes.empty?
         outputs = func.outputs.map(&:__getobj__)
-        in_data = func.inputs.map(&:data)
-        out_grad = outputs.map { |y| y.nil? ? nil : y.grad }
-
-        func.output_data = outputs.map { |y| y.nil? ? nil : y.data }
-        gxs = func.backward(in_data, out_grad)
 
-        raise unless gxs.size == in_data.size
-
-        unless func.retain_after_backward
-          func.output_data = nil
+        in_data = inputs.map(&:data)
+        out_grad = outputs.map do |y|
+          next nil if y.nil?
+          next grads[y] unless grads[y].nil?
+          y.grad_var
         end
+        out_grad_data = out_grad.map { |g| g.nil? ? g : g.data }
+
+        # Collect the current input gradients.
+        #
+        # When the same variable is passed to multiple input slots (e.g. an expression like +f(x, x)+),
+        # it makes the gradient accumulation complicated since the back-propagated gradients w.r.t.
+        # the first and second argument should be accumulated to the current gradient w.r.t. the same variable.
+        # In this case, the current implementation passes the current gradient only to the first occurrence of the variable
+        # in the input tuple and passes +nil+ to the rest of the occurrences.
+        # For example, when the input variables are +(x, x)+,
+        # the input gradient passed to the +backward_accumulate+ method is +(gx, nil)+ where +gx+ is the current gradient of ++x++.
+        # See also the docstring of +FunctionNode.backward_accumulate+.
+        target_inputs = target_input_indexes.map { |i| inputs[i] }
+        in_grad = []
+        target_input_indexes.each_with_index do |index_i, i|
+          x = inputs[index_i]
+          if target_inputs[0...i].include?(x)
+            gx = nil
+          elsif grads[x]
+						gx = grads[x]
+          elsif x.creator_node.nil?
+            gx = x.grad_var
+          else
+            gx = nil
+          end
+          in_grad << gx
+        end
+
+        gxs = func.backward_accumulate(target_input_indexes, out_grad, in_grad)
+        raise "Unmatched matries size: gxs.size(#{gxs.size}) != in_grad.size(#{in_grad.size})" unless gxs.size == in_grad.size
 
         unless retain_grad
           outputs.each do |y|
             unless y.nil? || y == @node
-              y.grad = nil
+              grads[y] = nil
+              y_var = y.get_variable
+              y_var.grad_var = nil unless y_var.nil?
             end
           end
         end
 
-        seen_vars = []
-        need_copy = []
-
-        func.inputs.zip(gxs).each do |x, gx|
+        gxs.each_with_index do |gx, i|
           next if gx.nil?
+          x = target_inputs[i]
           next unless x.requires_grad
 
-          Utils::Variable.check_grad_type(func, x, gx)
-
-          id_x = x.object_id
-          if x.creator.nil? # leaf
-            if x.grad.nil?
-              x.grad = gx
-              need_copy << id_x
-            else
-              if need_copy.include?(id_x)
-                x.grad = Utils::Array.force_array(x.grad + gx)
-                need_copy.delete(id_x)
-              else
-                x.grad += gx
-              end
-            end
-          else # not leaf
-            funcs << x.creator
-            if seen_vars.include?(id_x)
-              if need_copy.include?(id_x)
-                x.grad = Utils::Array.force_array(gx + x.grad)
-                need_copy.delete(id_x)
-              else
-                x.grad += gx
-              end
-            else
-              x.grad = gx
-              seen_vars << id_x
-              need_copy << id_x
-            end
+          Utils::Variable.check_grad_type(func, x, gx.data)
+
+          if target_inputs[0...i].include?(x)
+            cur_gx = grads[x]
+            grads[x] = cur_gx.nil? ? gx : gx + cur_gx
+          else
+            grads[x] = gx
+          end
+
+          x_var = x.get_variable
+          x_var.grad_var = grads[x] if x_var
+
+          if x.creator_node && !seen_set.include?(x.creator_node)
+            funcs << x.creator_node
+            seen_set.add(x.creator_node)
           end
         end
-      end 
+
+        funcs.sort_by! { |f| -f.rank }
+
+      end
     end
 
     def -@
-      Functions::Math::Neg.new.(self) 
+      Functions::Math::Neg.new.apply([self]).first
     end
 
     def +(other)
       if other.instance_of?(Chainer::Variable)
-        Functions::Math::Add.new.(*[self, other])
+        Functions::Math::Add.new.apply([self, other])[0]
       else
-        Functions::Math::AddConstant.new(other).(self)
+        Functions::Math::AddConstant.new(other).apply([self])[0]
       end
     end
 
-    def -(other) 
+    def -(other)
       if other.instance_of?(Chainer::Variable)
-        Functions::Math::Sub.new.(*[self, other])
+        Functions::Math::Sub.new.apply([self, other])[0]
       else
-        Functions::Math::AddConstant.new(-other).(self)
+        Functions::Math::AddConstant.new(-other).apply([self])[0]
       end
     end
 
     def *(other)
       if other.instance_of?(Chainer::Variable)
-        Functions::Math::Mul.new.(*[self, other])
+        Functions::Math::Mul.new.apply([self, other])[0]
       else
-        Functions::Math::MulConstant.new(other).(self)
+        Functions::Math::MulConstant.new(other).apply([self])[0]
       end
     end
 
     def /(other)
       if other.instance_of?(Chainer::Variable)
-        Functions::Math::Div.new.(*[self, other])
+        Functions::Math::Div.new.apply([self, other])[0]
       else
-        Functions::Math::MulConstant.new(1 / other).(self)
+        Functions::Math::MulConstant.new(1 / other).apply([self])[0]
       end
     end
 
-    def **(other) 
+    def **(other)
       if other.instance_of?(Chainer::Variable)
-        Functions::Math::PowVarVar.new.(*[self, other])
+        Functions::Math::PowVarVar.new.apply([self, other])[0]
       else
-        Functions::Math::PowVarConst.new(other).(self)
+        Functions::Math::PowVarConst.new(other).apply([self])[0]
       end
     end
 
@@ -196,7 +286,7 @@ module Chainer
 
     # when left side is Numeric value and right side is Chainer::Value, call this method.
     def coerce(other)
-      other = self.data.class[*other] if other.kind_of?(Numeric)
+      other = self.data.class.new.fill(other) if other.kind_of?(Numeric)
       [Chainer::Variable.new(other, requires_grad: false), self]
     end
   end