red-chainer 0.1.0

data/lib/chainer/links/connection/linear.rb

@@ -0,0 +1,43 @@
+module Chainer
+  module Links
+    module Connection
+      class Linear < ::Chainer::Link
+        attr_reader :w, :b
+
+        def initialize(in_size, out_size: nil, nobias: false, initial_w: nil, initial_bias: nil)
+          super()
+          in_size, out_size = nil, in_size if out_size.nil?
+          @out_size = out_size
+
+          init_scope do
+            w_initializer = Chainer::Initializers.get_initializer(initial_w)
+            @w = Chainer::Parameter.new(initializer: w_initializer)
+            
+            initialize_params(in_size) unless in_size.nil?
+            
+            if nobias
+              @b = nil
+            else
+              initial_bias = 0 if initial_bias.nil?
+              bias_initializer = Chainer::Initializers.get_initializer(initial_bias)
+              @b = Chainer::Parameter.new(initializer: bias_initializer, shape: out_size)
+            end
+          end
+        end
+
+        def call(x)
+          if @w.data.nil?
+            initialize_params(x.size.div(x.shape[0]))
+          end
+          Chainer::Functions::Connection::LinearFunction.linear(x, @w, @b)
+        end
+
+        private
+
+        def initialize_params(in_size)
+          @w.init([@out_size, in_size])
+        end
+      end
+    end
+  end
+end

data/lib/chainer/links/model/classifier.rb

@@ -0,0 +1,39 @@
+module Chainer
+  module Links
+    module Model
+      class Classifier < Chain
+        attr_accessor :compute_accuracy
+
+        def initialize(predictor, lossfun=Functions::Loss::SoftmaxCrossEntropy.method(:softmax_cross_entropy), accfun=Functions::Evaluation::Accuracy.method(:accuracy))
+          super()
+          @lossfun = lossfun
+          @accfun = accfun
+          @y = nil
+          @loss = nil
+          @accuracy = nil
+          @compute_accuracy = true
+
+          init_scope do
+            @predictor = predictor
+          end
+        end
+
+        def call(*args)
+          t = args.pop
+          x = args
+          @y = nil
+          @accuracy = nil
+          @y = @predictor.(*x)
+
+          @loss = @lossfun.call(@y, t)
+          Chainer::Reporter.save_report({loss: @loss}, self)
+          if @compute_accuracy
+            @accuracy = @accfun.call(@y, t)
+            Chainer::Reporter.save_report({accuracy: @accuracy}, self)
+          end
+          @loss
+        end
+      end
+    end
+  end
+end

data/lib/chainer/optimizer.rb

@@ -0,0 +1,69 @@
+module Chainer
+  class Optimizer
+    attr_accessor :target
+
+    def setup(link)
+      @target = link
+      @t = 0
+      @epoch = 0
+      
+      @hooks = {}
+    end
+
+    def _call_hook(hook)
+      if hook.methods.include?(:call_for_each_param)
+        @target.params.each do |param|
+          hook.(param.update_rule, param)
+        end
+      else
+        hook(self)
+      end
+    end
+  end
+
+  class UpdateRule
+    attr_reader :state
+
+    def initialize(parent_hyperparam:)
+      @hooks = {}  
+      @state = nil
+      @enabled = true
+      @hyperparam = Chainer::Hyperparameter.new(parent: parent_hyperparam)
+      @t = 0
+    end
+
+    def update(param)
+      return unless @enabled
+
+      @t += 1
+      prepare(param)
+      @hooks.values.each do |hook|
+        hook.call(param)
+      end
+      update_core(param)
+    end
+
+    def update_core(param)
+      # TODO: support GPU
+      update_core_cpu(param)
+    end
+
+    def update_core_cpu
+      raise NotImplementedError
+    end
+
+    def init_state(param)
+      raise NotImplementedError
+    end
+
+    private
+
+    def prepare(param)
+      if @state.nil?
+        @state = {}
+        init_state(param)
+      end
+      @state.select! { |_, v| v.kind_of?(Numo::NArray) }
+    end
+  end
+end

data/lib/chainer/optimizers/adam.rb

@@ -0,0 +1,62 @@
+module Chainer
+  module Optimizers
+    class AdamRule < UpdateRule
+      def initialize(parent_hyperparam: nil, alpha: nil, beta1: nil, beta2: nil, eps: nil)
+        hyperparam = Hyperparameter.new
+        hyperparam.instance_variable_set('@alpha', 0.001)
+        hyperparam.instance_variable_set('@beta1', 0.9)
+        hyperparam.instance_variable_set('@beta2', 0.999)
+        hyperparam.instance_variable_set('@eps', 1e-8)
+
+        super(parent_hyperparam: parent_hyperparam || hyperparam)
+
+        @hyperparam.instance_variable_set('@alpha', alpha) if alpha
+        @hyperparam.instance_variable_set('@beta1', beta1) if beta1
+        @hyperparam.instance_variable_set('@beta2', beta2) if beta2
+        @hyperparam.instance_variable_set('@eps', eps) if eps
+      end
+
+      def init_state(param)
+        @state[:m] = param.data.new_zeros
+        @state[:v] = param.data.new_zeros
+      end
+
+      def update_core_cpu(param)
+        grad = param.grad
+        return if grad.nil?
+
+        hp = @hyperparam
+
+        @state[:m] += (1 - hp.beta1) * (grad - @state[:m])
+        @state[:v] += (1 - hp.beta2) * (grad * grad - @state[:v])
+        param.data -= lr * @state[:m] / (Numo::NMath.sqrt(@state[:v]) + hp.eps)
+      end
+
+      def lr
+        fix1 = 1.0 - @hyperparam.beta1 ** @t
+        fix2 = 1.0 - @hyperparam.beta2 ** @t
+        @hyperparam.alpha * Math.sqrt(fix2) / fix1
+      end
+    end
+
+    class Adam < GradientMethod
+      def initialize(alpha: nil, beta1: nil, beta2: nil, eps: nil)
+        super()
+        @hyperparam.instance_variable_set('@alpha', alpha || 0.001)
+        @hyperparam.instance_variable_set('@beta1', beta1 || 0.9)
+        @hyperparam.instance_variable_set('@beta2', beta2 || 0.999)
+        @hyperparam.instance_variable_set('@eps', eps || 1e-8)
+      end
+
+      def create_update_rule
+        AdamRule.new(parent_hyperparam: @hyperparam)
+      end
+
+      def lr
+        fix1 = 1.0 - (@hyperparam.beta1 ** @t)
+        fix2 = 1.0 - (@hyperparam.beta2 ** @t)
+        @hyperparam.alpha * Math.sqrt(fix2) / fix1
+      end
+    end
+  end
+end

data/lib/chainer/parameter.rb

@@ -0,0 +1,53 @@
+module Chainer
+  class Parameter < Variable
+    attr_accessor :initializer, :grad_initializer, :update_rule
+
+    def initialize(initializer: nil, shape: nil, name: nil)
+      if initializer.nil?
+        initializer = Chainer::Initializers.nan()
+      elsif initializer.kind_of?(Numeric)
+        initializer = Initializers::Constant.new(initializer)
+      end
+
+      if shape.nil?
+        if @initializer.kind_of?(Numo::NArray)
+          super(initializer, name: name)
+        else
+          super(name: name)
+          @initializer = initializer
+          dtype = initializer.respond_to?(:dtype) ? initializer.dtype : 'DFloat'
+          @grad_initializer = Chainer::Initializers.nan()
+        end
+      else
+        if initializer.kind_of?(Numo::NArray)
+          initializer = Initializers::Constant.new(initializer)
+        end
+        data = Chainer::Initializers.generate_array(initializer, shape)
+        grad = Numo::NArray[*[1, 2]].new_fill(-922337203)
+        super(data, name: name, grad: grad)
+      end
+
+      @update_rule = nil
+    end
+
+    def cleargrad
+      super
+      @grad_initializer = nil if self.data.nil?
+    end
+
+    def init(shape)
+      data = Chainer::Initializers.generate_array(@initializer, shape)
+      ginit = @grad_initializer
+      grad = ginit.nil? ? nil : Chainer::Initializers.generate_array(ginit, shape)
+
+      @data[0] = data
+      @node.grad = grad
+    end
+
+    def update
+      if @update_rule
+        @update_rule.update(self)
+      end
+    end
+  end
+end

data/lib/chainer/reporter.rb

@@ -0,0 +1,130 @@
+module Chainer
+  module ReportService
+    @@reporters = []
+  end
+
+  class Reporter
+    include ReportService
+
+    def initialize
+      @observer_names = {}
+      @observation = {}
+    end
+
+    def self.save_report(values, observer=nil)
+      reporter = @@reporters[-1]
+      reporter.report(values, observer)
+    end
+
+    def report(values, observer=nil)
+      # TODO: keep_graph_on_report option
+      if observer
+        observer_id = observer.object_id
+        unless @observer_names.keys.include?(observer_id)
+          raise "Given observer is not registered to the reporter."
+        end
+        observer_name = @observer_names[observer_id]
+        values.each do |key, value|
+          name = "#{observer_name}/#{key}"
+          @observation[name] = value
+        end
+      else
+        @observation.update(values)
+      end
+    end
+
+    def add_observer(name, observer)
+      @observer_names[observer.object_id] = name
+    end
+
+    def scope(observation)
+      @@reporters << self
+      old = @observation
+      @observation = observation
+      yield
+      @observation = old
+      @@reporters.pop
+    end
+  end
+
+  class Summary
+    def initialize
+      @x = 0
+      @x2 = 0
+      @n = 0
+    end
+
+    # Adds a scalar value.
+    # Args:
+    #   value: Scalar value to accumulate.
+    def add(value)
+      @x += value
+      @x2 += value * value
+      @n += 1
+    end
+
+    # Computes the mean.
+    def compute_mean
+      @x.to_f / @n
+    end
+
+    # Computes and returns the mean and standard deviation values.
+    # Returns:
+    #   array: Mean and standard deviation values.    
+    def make_statistics
+      mean = @x / @n
+      var = @x2 / @n - mean * mean
+      std = Math.sqrt(var)
+      [mean, std]
+    end
+  end
+
+  # Online summarization of a sequence of dictionaries.
+  # ``DictSummary`` computes the statistics of a given set of scalars online.
+  # It only computes the statistics for scalar values and variables of scalar values in the dictionaries.
+  class DictSummary
+    def initialize
+      @summaries = Hash.new { |h,k| h[k] = Summary.new }
+    end
+
+    # Adds a dictionary of scalars.
+    # Args:
+    #   d (dict): Dictionary of scalars to accumulate. Only elements of
+    #             scalars, zero-dimensional arrays, and variables of
+    #             zero-dimensional arrays are accumulated.
+    def add(d)
+      d.each do |k, v|
+        v = v.data if v.kind_of?(Chainer::Variable)
+        if v.class.method_defined?(:to_i) || (v.class.method_defined?(:ndim) && v.ndim == 0)
+          @summaries[k].add(v)
+        end 
+      end
+    end
+
+    # Creates a dictionary of mean values.
+    # It returns a single dictionary that holds a mean value for each entry added to the summary.
+    # 
+    # Returns:
+    #   dict: Dictionary of mean values.
+    def compute_mean
+      @summaries.each_with_object({}) { |(name, summary), h| h[name] = summary.compute_mean }
+    end
+
+    # Creates a dictionary of statistics.
+    # It returns a single dictionary that holds mean and standard deviation
+    # values for every entry added to the summary. For an entry of name
+    # ``'key'``, these values are added to the dictionary by names ``'key'`` and ``'key.std'``, respectively.
+    # 
+    # Returns:
+    #   dict: Dictionary of statistics of all entries.
+    def make_statistics
+      stats = {}
+      @summaries.each do |name, summary|
+        mean, std = summary.make_statistics
+        stats[name] = mean
+        stats[name + '.std'] = std
+      end
+      stats
+    end
+  end
+end

data/lib/chainer/training/extension.rb

@@ -0,0 +1,25 @@
+module Chainer
+  module Training
+    class Extension
+      PRIORITY_WRITER = 300
+      PRIORITY_EDITOR = 200
+      PRIORITY_READER = 100
+
+      attr_accessor :name, :priority
+
+      def initialize
+      end
+
+      def call(trainer)
+      end
+
+      def default_name
+        self.class.to_s
+      end
+
+      def priority
+        @priority || PRIORITY_READER
+      end
+    end
+  end
+end

data/lib/chainer/training/extensions/evaluator.rb

@@ -0,0 +1,26 @@
+module Chainer
+  module Training
+    module Extensions
+      class Evaluator < Extension
+        def initialize(iterator, target, converter: nil, device: nil, eval_hook: nil, eval_func: nil)
+          @priority = Extension::PRIORITY_WRITER
+
+          if iterator.kind_of?(Dataset::Iterator)
+            iterator = { main: iterator }
+          end
+          @iterators = iterator
+
+          if target.kind_of?(Link)
+            target = { main: target }
+          end
+          @targets = target
+
+          @converter = converter || Dataset::Convert.method(:concat_examples)
+          @device = device
+          @eval_hook = eval_hook
+          @eval_func = eval_func
+        end
+      end
+    end
+  end
+end