red-chainer 0.2.1 → 0.3.0

data/lib/chainer/initializer.rb

@@ -1,5 +1,7 @@
 module Chainer
   class Initializer
+    attr_accessor :dtype
+
     def initialize(dtype: nil)
       @dtype = dtype
     end

data/lib/chainer/initializers/constant.rb

@@ -8,7 +8,7 @@ module Chainer
 
       def call(array)
         if @dtype
-          raise ArgumentError unless array.dtype == @dtype
+          raise ArgumentError unless array.class == @dtype
         end
         array.store(@fill_value)
         array

data/lib/chainer/initializers/init.rb

@@ -1,7 +1,11 @@
 module Chainer
   module Initializers
     def self.generate_array(initializer, shape)
-      array = Numo::DFloat.new(shape).rand
+      klass = Numo::SFloat
+      if initializer.respond_to?(:dtype) && initializer.dtype
+        klass = initializer.dtype
+      end
+      array = klass.new(shape).rand
       initializer.(array)
     end
 

data/lib/chainer/initializers/normal.rb

@@ -8,7 +8,7 @@ module Chainer
 
       def call(array)
         args = { loc: 0.0, scale: @scale, size: array.shape}
-        Numo::DFloat.new(array.shape).rand_norm(0.0, @scale)
+        array.class.new(array.shape).rand_norm(0.0, @scale)
       end
     end
 

data/lib/chainer/iterators/serial_iterator.rb

@@ -18,8 +18,8 @@ module Chainer
         @previous_epoch_detail = epoch_detail
 
         i = @current_position
-        i_end = i + @batch_size
         n = @dataset.size
+        i_end = [i + @batch_size, n].min
 
         batch = @order[i...i_end].to_a.map { |index| @dataset[index] }
 

data/lib/chainer/link.rb

@@ -39,6 +39,17 @@ module Chainer
       end
     end
 
+    # Registers an attribute of a given name as a persistent value.
+    # This is a convenient method to register an existing attribute as a persistent value.
+    # If `name` has been already registered as a parameter,
+    # this method removes it from the list of parameter names and re-registers it as a persistent value.
+    #
+    # @param [string] name Name of the attribute to be registered.
+    def register_persistent(name)
+      @persistent << name
+      @params.delete(name)
+    end
+
     def params(include_uninit: true)
       @params.map do |name|
         data = self.instance_variable_get(name).data

data/lib/chainer/links/connection/convolution_2d.rb

@@ -0,0 +1,98 @@
+module Chainer
+  module Links
+    module Connection
+      class Convolution2D < ::Chainer::Link
+        # Two-dimensional convolutional layer.
+        # 
+        # This link wraps the :func:`chainer.functions.convolution_2d` function
+        # and holds the filter weight and bias vector as parameters.
+        # 
+        # @param [integer or nil] in_channels Number of channels of input arrays.
+        #     If `nil`, parameter initialization will be deferred until the first forward data pass at which time the size will be determined.
+        # @param [integer] out_channels Number of channels of output arrays.
+        # @param [integer or 2-d int array] ksize Size of filters (a.k.a. kernels).
+        # @param [integer or 2-d int array] stride Stride of filter applications.
+        # @param [integer or 2-d int array] pad Spatial padding width for input arrays.
+        # @param [boolean] nobias If `true`, then this link does not use the bias term.
+        # @param [Numo::NArray] initialW Initial weight value. If `nil`, the default initializer is used.
+        # @param [Numo::NArray] initial_bias Initial bias value. If `nil`, the bias is set to 0.
+        #
+        # Example
+        # There are several ways to make a Convolution2D link.
+        # Let an input vector `x` be:
+        # > x = Numo::DFloat.new(1, 3, 10, 10).seq
+        #
+        # 1. Give the first three arguments explicitly:
+        # > l = Chainer::Links::Connection::Convolution2D.new(3, 7, 5)
+        # > y = l.(x)
+        # > y.shape
+        # [1, 7, 6, 6]
+        #
+        # 2. Omit `in_channels` or fill it with `nil`:
+        #    The below two cases are the same.
+        #
+        # > l = Chainer::Links::Connection::Convolution2D.new(7, 5)
+        # > y = l.(x)
+        # > y.shape
+        # [1, 7, 6, 6]
+        #
+        # > l = Chainer::Links::Connection::Convolution2D.new(nil, 7, 5)
+        # > y = l.(x)
+        # > y.shape
+        # [1, 7, 6, 6]
+        #
+        # When you omit the first argument, you need to specify the other subsequent arguments from `stride` as keyword auguments.
+        #
+        # > l = Chainer::Links::Connection::Convolution2D.new(7, 5, stride: 1, pad: 0)
+        # > y = l.(x)
+        # > y.shape
+        # [1, 7, 6, 6]
+        def initialize(in_channels, out_channels, ksize=nil, stride: 1, pad: 0, nobias: false, initial_w: nil, initial_bias: nil)
+          super()
+          if ksize.nil?
+            out_channels, ksize, in_channels = in_channels, out_channels, nil
+          end
+
+          @ksize = ksize
+          @stride = stride.is_a?(Array) ? stride : [stride, stride]
+          @pad = pad.is_a?(Array) ? pad : [pad, pad]
+          @out_channels = out_channels
+          
+          init_scope do
+            w_initializer = Chainer::Initializers.get_initializer(initial_w)
+            @w = Chainer::Parameter.new(initializer: w_initializer)
+            if in_channels
+              initialize_params(in_channels)
+            end
+
+            if nobias
+              @b = nil
+            else
+              initial_bias = 0 if initial_bias.nil?
+              bias_initializer = Chainer::Initializers.get_initializer(initial_bias)
+              @b = Chainer::Parameter.new(initializer: bias_initializer, shape: out_channels)
+            end
+          end
+        end
+
+        # Applies the convolution layer.
+        # @param [Chainer::Variable] x Input image.
+        # @return [Chainer::Variable] Output of the convolution.
+        def call(x)
+          initialize_params(x.shape[1]) if @w.data.nil?
+          Chainer::Functions::Connection::Convolution2DFunction.convolution_2d(x, @w, b: @b, stride: @stride, pad: @pad)
+        end
+
+        private
+
+        def initialize_params(in_channels)
+          kh, kw = @ksize.is_a?(Array) ? @ksize : [@ksize, @ksize]
+          w_shape = [@out_channels, in_channels, kh, kw]
+          @w.init(w_shape)
+        end
+      end
+    end
+  end
+end
+
+

data/lib/chainer/links/normalization/batch_normalization.rb

@@ -0,0 +1,106 @@
+module Chainer
+  module Links
+    module Normalization 
+      class BatchNormalization < Chainer::Link
+        # Batch normalization layer on outputs of linear or convolution functions.
+        # 
+        # It runs in three modes: training mode, fine-tuning mode, and testing mode.
+        # In training mode, it normalizes the input by *batch statistics*. It also
+        # maintains approximated population statistics by moving averages, which can
+        # be used for instant evaluation in testing mode.
+        # 
+        # In fine-tuning mode, it accumulates the input to compute *population
+        # statistics*. In order to correctly compute the population statistics, a
+        # user must use this mode to feed mini-batches running through whole training dataset.
+        # 
+        # In testing mode, it uses pre-computed population statistics to normalize the input variable.
+        # The population statistics is approximated if it is computed by training mode,
+        # or accurate if it is correctly computed by fine-tuning mode.
+        #
+        # @param [integer or int array] size Size (or shape) of channel dimensions.
+        # @param [float] decay Decay rate of moving average. It is used on training.
+        # @param [float] eps Epsilon value for numerical stability.
+        # @param [Numo::NArray.dtype] dtype Type to use in computing.
+        # @param [boolean] use_gamma If `true`, use scaling parameter. Otherwise, use unit(1) which makes no effect.
+        # @param [boolean] use_beta If `true`, use shifting parameter. Otherwise, use unit(0) which makes no effect.
+        def initialize(size, decay: 0.9, eps: 2e-5, dtype: Numo::DFloat, use_gamma: true, use_beta: true, initial_gamma: nil, initial_beta: nil)
+          super()
+          @avg_mean = dtype.zeros(size)
+          register_persistent('avg_mean')
+          @avg_var = dtype.zeros(size)
+          register_persistent('avg_var')
+          @n = 0
+          register_persistent('n')
+          @decay = decay
+          @eps = eps
+
+          init_scope do
+            if use_gamma
+              initial_gamma = 1 if initial_gamma.nil?
+              initial_gamma = Chainer::Initializers.get_initializer(initial_gamma)
+              initial_gamma.dtype = dtype
+              @gamma = Chainer::Parameter.new(initializer: initial_gamma, shape: size)
+            end
+            if use_beta
+              initial_beta = 0 if initial_beta.nil?
+              initial_beta = Chainer::Initializers.get_initializer(initial_beta)
+              initial_beta.dtype = dtype
+              @beta = Chainer::Parameter.new(initializer: initial_beta, shape: size)
+            end
+          end
+        end
+
+        # Invokes the forward propagation of BatchNormalization.
+        # In training mode, the BatchNormalization computes moving averages of
+        # mean and variance for evaluatino during training, and normalizes the input using batch statistics.
+        # @param [Chainer::Variable] x Input variable.
+        # @param [boolean] finetune If it is in the training mode and `finetune` is `True`,
+        # BatchNormalization runs in fine-tuning mode;
+        # it accumulates the input array to compute population statistics for normalization,
+        # and normalizes the input using batch statistics.
+        def call(x, finetune: false)
+          if self.instance_variable_defined?(:@gamma)
+            gamma = @gamma
+          else
+            gamma = Chainer::Variable.new(x.data.class.ones(@avg_mean.shape))
+          end
+
+          if self.instance_variable_defined?(:@beta)
+            beta = @beta
+          else
+            beta = Chainer::Variable.new(x.data.class.zeros(*@avg_mean.shape))
+          end
+          
+          if Chainer.configuration.train
+            if finetune
+              @n += 1
+              decay = 1.0 - 1.0 / @n
+            else
+              decay = @decay
+            end
+
+            func = Chainer::Functions::Normalization::BatchNormalizationFunction.new(eps: @eps, mean: @avg_mean, var: @avg_var, decay: decay)
+            ret = func.(x, gamma, beta)
+
+            @avg_mean[false] = func.running_mean
+            @avg_var[false] = func.running_var
+          else
+            mean = Chainer::Variable(@avg_mean)
+            var = Chainer::Variable(@avg_var)
+            ret = Chainer::Functions::Normalization::BatchNormalizationFunction.fixed_batch_normalization(x, gamma, beta, mean, var, eps: @eps)
+          end
+
+          ret
+        end
+
+        # Resets the population count for collecting population statistics.
+        # This method can be skipped if it is the first time to use the fine-tuning mode.
+        # Otherwise, this method should be called before starting the fine-tuning mode again.
+        def start_finetuning
+          @n = 0
+        end
+      end
+    end
+  end
+end
+

data/lib/chainer/optimizer.rb

@@ -81,7 +81,7 @@ module Chainer
           # try to initialize the state to retrieve state entries
           @state = {}
           self_copy = self.dup
-          arr = Numo::DFloat.new(1)
+          arr = Numo::SFloat.new(1)
           self_copy.init_state(Chainer::Variable.new(arr, grad: arr))
           @state.keys.each do |key|
             @state[key] = serializer.(key.to_s, nil)
@@ -104,4 +104,43 @@ module Chainer
       @state.select! { |_, v| v.kind_of?(Numo::NArray) }
     end
   end
+
+  class HyperparameterProxy  
+    def initialize(obj, attr_name)
+      obj.class.class_eval do
+        obj.class.send(:define_method, attr_name) do
+          self.instance_variable_get(:@hyperparam).instance_variable_get("@#{attr_name}")
+        end
+
+        obj.class.send(:define_method, "#{attr_name}=") do |val|
+          self.instance_variable_get(:@hyperparam).instance_variable_set("@#{attr_name}", val)
+        end
+      end
+    end
+  end
+
+  # Optimizer/UpdateRule hook function for weight decay regularization
+  #
+  # This hook function adds a scaled parameter to the correspondeing gradient
+  # It can be used as a regularization
+  #
+  # @param [Float] rate Coefficient for the weight decay
+  class WeightDecay
+    def self.name
+      "WeightDecay"
+    end
+
+    def self.call_for_each_param
+      true
+    end
+
+    def initialize(rate)
+      @rate = rate
+    end
+
+    def call(rule, param)
+      return if param.data.nil? || param.grad.nil?
+      param.grad += @rate * param.data
+    end
+  end
 end

data/lib/chainer/optimizers/momentum_sgd.rb

@@ -0,0 +1,49 @@
+module Chainer
+  module Optimizers
+    # Update rule for the classical momentum SGD
+    class MomentumSGDRule < UpdateRule
+      def initialize(parent_hyperparam: nil, lr: nil, mementum: nil)
+        hyperparam = Hyperparameter.new
+        hyperparam.instance_variable_set('@lr', 0.01)
+        hyperparam.instance_variable_set('@momentum', 0.9)
+
+        super(parent_hyperparam: parent_hyperparam || hyperparam)
+        
+        @hyperparam.instance_variable_set('@lr', lr) if lr
+        @hyperparam.instance_variable_set('@mementum', mementum) if mementum
+      end
+   
+      def init_state(param)
+        @state[:v] = param.data.new_zeros
+      end
+
+      def update_core_cpu(param)
+        grad = param.grad
+        return if grad.nil?
+          
+        v = @state[:v]
+        v *= @hyperparam.momentum
+        v -= @hyperparam.lr * grad
+        param.data += v
+      end 
+    end
+    
+    # Momentum SGD optimizer
+    class MomentumSGD < GradientMethod
+      attr_accessor :lr, :momentum
+      # @param [Float] lr Learning rate
+      # @param [Float] momentum Exponential decay rate of the first order moment
+      def initialize(lr: nil, momentum: nil)
+        super()
+        @hyperparam.instance_variable_set('@lr', lr || 0.01)
+        @hyperparam.instance_variable_set('@momentum', momentum || 0.9)
+        Chainer::HyperparameterProxy.new(self, "lr")
+        Chainer::HyperparameterProxy.new(self, "momentum")
+      end
+     
+      def create_update_rule
+        MomentumSGDRule.new(parent_hyperparam: @hyperparam)
+      end
+    end
+  end
+end

data/lib/chainer/parameter.rb

@@ -15,7 +15,7 @@ module Chainer
         else
           super(name: name)
           @initializer = initializer
-          dtype = initializer.respond_to?(:dtype) ? initializer.dtype : 'DFloat'
+          dtype = initializer.respond_to?(:dtype) ? initializer.dtype : 'SFloat'
           @grad_initializer = Chainer::Initializers.nan()
         end
       else

data/lib/chainer/serializers/marshal.rb

@@ -3,10 +3,14 @@ module Chainer
     class MarshalSerializer < Chainer::Serializer      
       attr_accessor :target, :path
 
-      def self.save_file(filename, obj)
+      # @param [string] file_path Target file path
+      # @param [Object] obj Object to be serialized
+      def self.save_file(file_path, obj)
         s = self.new
         s.save(obj)
-        Marshal.dump(s.target, filename)
+        File.open(file_path, 'wb') do |f|
+          Marshal.dump(s.target, f)
+        end
       end
 
       def initialize(target: nil, path: "")
@@ -24,7 +28,7 @@ module Chainer
           arr = Numo::Bit[1]
         elsif value.is_a?(FalseClass)
           arr = Numo::Bit[0]
-        elsif value.instance_of?(String)
+        elsif value.instance_of?(String) || value.nil?
           arr = value
         else
           arr = Numo::NArray.cast(value)

data/lib/chainer/testing/array.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Chainer
+  module Testing
+    def assert_allclose(expect, actual, atol: 1e-5, rtol: 1e-4)
+      # Asserts if some corresponding element of x and y differs too much.
+      #
+      #   This function can handle both CPU and GPU arrays simultaneously.
+      #
+      #   Args:
+      #       expect: Left-hand-side array.
+      #       actual: Right-hand-side array.
+      #       atol (float): Absolute tolerance.
+      #       rtol (float): Relative tolerance.
+      #
+      expect = Utils::Array.force_array(expect)
+      actual = Utils::Array.force_array(actual)
+
+      # If the expected is 0-dim arrary, extend the dimension to the actual.
+      if (expect.shape != actual.shape) and (expect.ndim == 0)
+         expect = actual.class.new(actual.shape).fill(expect.to_f)
+      end
+
+      actual.each_with_index{|actual_val, *i|
+        if (expect[*i].to_f - actual_val.to_f).abs > atol + rtol * expect[*i].abs
+          raise "assert_allclose Error\n  expect: #{expect.inspect}\n  actual : #{actual.inspect}\n    (#{i})=> #{(expect - actual).abs.max()} > #{atol + rtol * expect[*i].abs}"
+        end
+      }
+    end
+    module_function :assert_allclose
+  end
+end