red-chainer 0.1.0

data/lib/chainer/dataset/iterator.rb

@@ -0,0 +1,15 @@
+module Chainer
+  module Dataset
+    class Iterator
+      def next
+        raise NotImplementedError
+      end
+
+      def finalize
+      end
+
+      def serialize(serializer)
+      end
+    end
+  end
+end

data/lib/chainer/datasets/mnist.rb

@@ -0,0 +1,89 @@
+require 'zlib'
+
+module Chainer
+  module Datasets
+    module Mnist
+      def self.get_mnist(withlabel: true, ndim: 1, scale: 1.0, dtype: Numo::DFloat, label_dtype: Numo::Int32)
+        train_raw = retrieve_mnist_training
+        train = preprocess_mnist(train_raw, withlabel, ndim, scale, dtype, label_dtype)
+
+        test_raw = retrieve_mnist_test
+        test = preprocess_mnist(test_raw, withlabel, ndim, scale, dtype, label_dtype)
+        [train, test]
+      end
+
+      def self.preprocess_mnist(raw, withlabel, ndim, scale, image_dtype, label_dtype)
+        images = raw[:x]
+        if ndim == 2
+          images = images.reshape(-1, 28, 28)
+        elsif ndim == 3
+          images = images.reshape(-1, 1, 28, 28)
+        elsif ndim != 1
+          raise "invalid ndim for MNIST dataset"
+        end
+
+        images = images.cast_to(image_dtype)
+        images *= scale / 255.0
+        
+        if withlabel
+          labels = raw[:y].cast_to(label_dtype)
+          TupleDataset.new(images, labels)
+        else
+          images
+        end
+      end
+
+      def self.retrieve_mnist_training
+        urls = ['http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz',
+                'http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz']
+        retrieve_mnist('train.npz', urls)
+      end
+
+      def self.retrieve_mnist_test
+        urls = ['http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz',
+                'http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz']
+        retrieve_mnist('test.npz', urls)
+      end
+
+      def self.retrieve_mnist(name, urls)
+        root = Chainer::Dataset::Download.get_dataset_directory('pfnet/chainer/mnist')
+        path = File.expand_path(name, root)
+        Chainer::Dataset::Download.cache_or_load_file(path) do
+          make_npz(path, urls)
+        end
+      end
+
+      def self.make_npz(path, urls)
+        x_url, y_url = urls
+        x_path = Chainer::Dataset::Download.cached_download(x_url)
+        y_path = Chainer::Dataset::Download.cached_download(y_url)
+
+        x = nil
+        y = nil
+
+        Zlib::GzipReader.open(x_path) do |fx|
+          Zlib::GzipReader.open(y_path) do |fy|
+            fx.read(4)
+            fy.read(4)
+
+            n = fx.read(4).unpack('i>')[0]
+            fy.read(4)
+            fx.read(8)
+
+            x = Numo::UInt8.new(n, 784).rand(n)
+            y = Numo::UInt8.new(n).rand(n)
+
+            n.times do |i|
+              y[i] = fy.read(1).ord
+              784.times do |j|
+                x[i, j] = fx.read(1).ord
+              end
+            end
+          end
+        end
+
+        { x: x, y: y}
+      end
+    end
+  end
+end

data/lib/chainer/datasets/tuple_dataset.rb

@@ -0,0 +1,33 @@
+module Chainer
+  module Datasets
+    class TupleDataset
+      def initialize(*datasets)
+        if datasets.empty?
+          raise "no datasets are given"
+        end
+        length = datasets[0].shape[0]
+
+        datasets.each_with_index do |dataset, idx|
+          raise "dataset of the index #{idx} has a wrong length" unless dataset.shape[0] == length
+        end
+
+        @datasets = datasets
+        @length = length
+      end
+
+      def [](index)
+        batches = @datasets.map { |dataset| dataset.ndim > 1 ? dataset[index, 0...dataset.shape[1]] : dataset[index] }
+        if index.kind_of?(Enumerable)
+          length = batches[0].shape[0]
+          length.times.map {|i| batches.map { |m| m[i] } }
+        else
+          batches
+        end
+      end
+
+      def size
+        @length
+      end
+    end
+  end
+end

data/lib/chainer/function.rb

@@ -0,0 +1,80 @@
+module Chainer
+  class Function
+
+    attr_reader :rank, :inputs, :outputs, :retain_after_backward
+    attr_accessor :output_data
+
+    def initialize
+      @rank = 0
+    end
+
+    def call(*inputs)
+      inputs = inputs.map do |x|
+        if x.kind_of?(Chainer::Variable)
+          x
+        else
+          Variable.new(x, requires_grad: false)
+        end
+      end
+
+      in_data = inputs.map(&:data)
+      requires_grad = inputs.any?(&:requires_grad)
+
+      @input_indexes_to_retain = nil
+      @output_indexes_to_retain = nil
+      outputs = forward(in_data)
+
+      ret = outputs.map do |y|
+        Variable.new(y, requires_grad: requires_grad)
+      end
+
+      if Chainer.configuration.enable_backprop
+        @rank = inputs.map(&:rank).max || 0
+
+        ret.each { |y| y.creator = self }
+
+        @inputs = inputs.map(&:node)
+        @outputs = ret.map { |y| WeakRef.new(y.node) }
+
+        @input_indexes_to_retain = 0...inputs.size if @input_indexes_to_retain.nil?
+        @input_indexes_to_retain.each do |index|
+          inputs[index].retain_data()
+        end
+        remove_instance_variable(:@input_indexes_to_retain)
+
+        unless @output_indexes_to_retain.nil?
+          @output_indexes_to_retain.each do |index|
+            ret[index].retain_data()
+          end
+          remove_instance_variable(:@output_indexes_to_retain)
+        end
+      end
+
+      ret.size == 1 ? ret[0] : ret
+    end
+
+    def forward(inputs)
+      # TODO: GPU branch processing
+      forward_cpu(inputs)
+    end
+
+    def forward_cpu(inputs)
+      raise NotImplementedError
+    end
+
+    def backward(inputs, grad_outputs)
+      backward_cpu(inputs, grad_outputs)
+    end
+
+    def retain_inputs(indexes)
+      @input_indexes_to_retain = indexes
+    end
+
+    def retain_outputs(indexes, retain_after_backward: false)
+      @output_indexes_to_retain = indexes
+      if retain_after_backward
+        @retain_after_backward = retain_after_backward
+      end
+    end
+  end
+end

data/lib/chainer/functions/activation/log_softmax.rb

@@ -0,0 +1,37 @@
+module Chainer
+  module Functions
+    module Activation
+      def self.logsumexp(x)
+        m = x.max(axis: 1, keepdims: true)
+        y = x - m
+        y = Numo::NMath.exp(y)
+        s = y.sum(axis: 1, keepdims: true)
+        s = Numo::NMath.log(s)
+        m + s
+      end
+
+      def self.log_softmax(x)
+        log_z = logsumexp(x)
+        x - log_z
+      end
+
+      class LogSoftmax < Function
+        def self.relu(x)
+          self.new.(x)
+        end
+
+        def forward_cpu(x)
+          retain_inputs([])
+          retain_outputs([0])
+          x[0][x[0]<=0] = 0
+          [Utils::Array.force_array(x[0])] 
+        end
+
+        def backward_cpu(x, gy)
+          y = output_data[0]
+          [Utils::Array.force_array(gy[0] * (y > 0))]
+        end
+      end
+    end
+  end
+end

data/lib/chainer/functions/activation/relu.rb

@@ -0,0 +1,23 @@
+module Chainer
+  module Functions
+    module Activation
+      class Relu < Function
+        def self.relu(x)
+          self.new.(x)
+        end
+
+        def forward_cpu(x)
+          retain_inputs([])
+          retain_outputs([0])
+          x[0][x[0]<=0] = 0
+          [Utils::Array.force_array(x[0])] 
+        end
+
+        def backward_cpu(x, gy)
+          y = output_data[0]
+          [Utils::Array.force_array(gy[0] * (y > 0))]
+        end
+      end
+    end
+  end
+end

data/lib/chainer/functions/connection/linear.rb

@@ -0,0 +1,48 @@
+module Chainer
+  module Functions
+    module Connection
+      class LinearFunction < Chainer::Function
+        def self.linear(x, w, b=nil)
+          if b.nil?
+            self.new.(x, w)
+          else
+            self.new.(x, w, b)
+          end
+        end
+
+        def forward(inputs)
+          x = as_mat(inputs[0])
+          w = inputs[1]
+
+          y = x.dot(w.transpose).cast_to(x.class)
+          if inputs.size == 3
+            b = inputs[2]
+            y += b
+          end
+          return [y]
+        end
+
+        def backward(inputs, grad_outputs)
+          x = as_mat(inputs[0])
+          w = inputs[1]
+          gy = grad_outputs[0]
+          gx = gy.dot(w).cast_to(x.class).reshape(*inputs[0].shape)
+          gw = gy.transpose.dot(x).cast_to(w.class)
+          if inputs.size == 3
+            gb = gy.sum(0)
+            [gx, gw, gb]
+          else
+            [gx, gw]
+          end
+        end
+
+        private
+
+        def as_mat(x)
+          return x if x.ndim == 2
+          x.reshape(x.size, -1)
+        end
+      end
+    end
+  end
+end

data/lib/chainer/functions/evaluation/accuracy.rb

@@ -0,0 +1,42 @@
+module Chainer
+  module Functions
+    module Evaluation
+      class Accuracy < Function
+        def self.accuracy(y, t, ignore_label: nil)
+          self.new(ignore_label: ignore_label).(y, t)
+        end
+
+        def initialize(ignore_label: nil)
+          @ignore_label = ignore_label
+        end
+
+        def forward(inputs)
+          y, t = inputs
+          if @ignore_label
+            mask = t.eq(@ignore_label)
+            ignore_cnt = mask.count
+
+            # this work
+            pred = y.max_index(axis: 1).to_a.map.with_index { |val, idx| val - y.shape[1] * idx}
+            pred = y.class[*pred].reshape(*t.shape)
+            pred[mask] = @ignore_label
+            count = pred.eq(t).count - ignore_cnt
+
+            total = t.size - ignore_cnt
+
+            if total == 0
+              [y.class.cast(0.0)]
+            else
+              [y.class.cast(count.to_f / total)]
+            end
+          else
+            pred = y.max_index(axis: 1).to_a.map.with_index { |val, idx| val - y.shape[1] * idx}
+            pred = y.class[*pred].reshape(*t.shape)
+
+            [y.class.cast(y.class[pred.eq(t)].mean)]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chainer/functions/loss/softmax_cross_entropy.rb

@@ -0,0 +1,134 @@
+module Chainer
+  module Functions
+    module Loss
+      class SoftmaxCrossEntropy < Function
+        def self.softmax_cross_entropy(x, t, normalize: true, cache_score: true, class_weight: nil, ignore_label: -1, reduce: 'mean')
+          self.new(normalize: normalize, cache_score: cache_score, class_weight: class_weight, ignore_label: ignore_label, reduce: reduce).(x, t)
+        end
+
+        def initialize(normalize: true, cache_score: true, class_weight: nil, ignore_label: -1, reduce: 'mean')
+          @normalize = normalize
+          @cache_score = cache_score
+          @class_weight = class_weight
+
+          unless class_weight.nil?
+            if @class_weight.ndim != 1
+              raise ArgumentError 'class_weight.ndim should be 1'
+            elsif @class_weight.dtype != Numo::DFloat
+              raise ArgumentError 'The dtype of class_weight should be \'Numo::DFloat\''
+            elsif @class_weight.kind_of?(Chainer::Variable)
+              raise ArgumentError 'class_weight should be a Numo::NArray, not a chainer.Variable'
+            end
+          end
+
+          @ignore_label = ignore_label
+          unless ['mean', 'no'].include?(reduce)
+            raise ArgumentError "only 'mean' and 'no' are valid for 'reduce', but #{reduce} is given"
+          end
+
+          @reduce = reduce
+        end
+
+        def forward_cpu(inputs)
+          x, t = inputs
+          log_y = Activation.log_softmax(x)
+
+          if @cache_score
+            @y = Numo::NMath.exp(log_y)
+          end
+          if @class_weight
+            shape = x.ndim.times.map { |e| e == 1 ? -1 : 1 } 
+            log_y += broadcast_to(@class_weight.reshape(*shape), x.shape)
+          end
+          log_yd = rollaxis(log_y, 1)
+          begin
+            log_yd = log_yd.reshape(log_yd.size, -1)
+          rescue ArgumentError
+          end
+
+          ravel_arr = t.dup.flatten.dup
+          ravel_arr[ravel_arr<0] = 0
+          arange_arr = t.class.new(t.size).seq
+
+          # https://github.com/chainer/chainer/blob/v2.0.2/chainer/functions/loss/softmax_cross_entropy.py#L79
+          log_p = []
+          arange_arr.each do |col_idx|
+            log_p << log_yd[ravel_arr, col_idx][col_idx]
+          end
+          log_p = Numo::NArray.[](*log_p)
+
+          log_p[log_p.eq(@ignore_label)] = 0
+
+          if @reduce == 'mean'
+            if @normalize
+              count = t.ne(@ignore_label).count
+            else
+              count = x.size
+            end
+            @coeff = 1.0 / [count, 1].max
+
+            y = log_p.sum(keepdims: true) * (-@coeff)
+            [y.reshape(())]
+          else
+            [-log_p.reshape(t.shape)]
+          end
+        end
+
+        def backward_cpu(inputs, grad_outputs)
+          x, t = inputs
+          gloss = grad_outputs[0]
+
+          if self.instance_variable_defined?(:'@y')
+            y = @y.dup
+          else
+            y = Activation.log_softmax(x)
+            y = Numo::NMath.exp(y)
+          end
+
+          if y.ndim == 2
+            gx = y
+            t[t<0] = 0
+            t.each_with_index do |v, idx|
+              gx[(idx * 10)...(idx * 10 + 10)][v] -= 1
+            end
+
+            if @class_weight
+              shape = x.ndim.times.map { |d| d == 1 ? -1 : 1 }
+              c = broadcast_to(@class_weight.reshape(shape), x.shape)
+              c = c[Numo::DFloat.new(t.size).seq, t]
+              gx *= broadcast_to(t.expand_dims(1), gx.shape)
+            end
+
+            bit = t.flatten.dup
+            bit[t.ne(@ignore_label)] = 1
+            bit[bit.ne(1)] = 0
+            gx *= bit.reshape(t.size, 1)
+          else
+            raise 'TODO: ndim > 2 backward'
+          end
+
+          if @reduce == 'mean'
+            gx *= gloss * @coeff
+          else
+            raise 'TODO: reduce'
+          end
+          return [gx, nil]
+        end
+
+
+        private
+
+        def broadcast_to(array, shape)
+          array.class.tile(array, shape[0]).reshape(*shape)
+        end
+
+        def rollaxis(y, axis, start: 0)
+          axes = (0...y.ndim).to_a
+          axes.delete_at(axis)
+          axes.insert(start, axis)
+          y.transpose(*axes)
+        end
+      end
+    end
+  end
+end