nn 1.8 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac474651871e2134d4e2372cf8254d21f3e2e6e53173d9b0a2897e72a5c20979
-  data.tar.gz: fe0813922ccb9f7a1351d9bc4a7377ed99c7ff4e4140b537074a7540afe8ed69
+  metadata.gz: 8f77c817ea492d035851bf8552ad2a97928f6762acb455ae23de0e3ee8f40871
+  data.tar.gz: 1f162719087671733c8afd5279bca59859474dd366677acbcf79032a9fff5eba
 SHA512:
-  metadata.gz: 9b84f7be4c0aa1b0bf00d24b3e5e7df1f47cc4c73174a4b54c9d8b1e691db414c95ed174213b3aa7275d26752684cb9cd927d6b2e879951e1f278deb15727010
-  data.tar.gz: 7b50ce21afdb88cd306d1e38fa1c7718740cdc313e2ed669558f1d251c439cc939d996fe85cf76b4b16d9b902ebacecf343b81065fc4b0ccc84578d64400d948
+  metadata.gz: 492e639590f4b81083a669f51ee192cb9a758ee0bbe950539c74367322ab78a9ad77e6075f90e56f28eecbc57ba42df91455c88e109fe2ec5565ceb77730bafc
+  data.tar.gz: d8745f38ed5ca0d75da462c6a8cf1233ea8e9c69e10918b87f37c23c8344690075bba1f6c13cf16267f51a92993e1fe58c0297251c309adcbbd3c4c85f339221

data/README.md

@@ -14,4 +14,4 @@ MNISTで98%以上の精度を出せるぐらいの性能はあります。
 
 ## ライセンス
 
-この宝石は、[MITライセンス](https://opensource.org/licenses/MIT)の条件でオープンソースとして入手できます。
+このgemは、[MITライセンス](https://opensource.org/licenses/MIT)の条件でオープンソースとして入手できます。

data/document.txt

@@ -13,6 +13,11 @@ class NN
 
 <クラスメソッド>
 load(file_name) : NN
+Marshal形式で保存された学習結果を読み込みます。
+  String file_name  読み込むMarshalファイル名
+  戻り値  NNのインスタンス
+
+load_json(file_name) : NN
 JSON形式で保存された学習結果を読み込みます。
   String file_name  読み込むJSONファイル名
   戻り値  NNのインスタンス
@@ -52,14 +57,7 @@ initialize(num_nodes,
   Float dropout_ratio  ドロップアウトさせるノードの比率
   bool use_batch_norm  バッチノーマライゼーションを使用するか否か
 
-train(x_train, y_train, x_test, y_test, epochs,
-      learning_rate_decay: 0,
-      save_dir: nil,
-      save_interval: 1,
-      test: nil,
-      border: nil,
-      tolerance: 0.5,
-      &block) : void
+train(x_train, y_train, x_test, y_test, epochs, func = nil, &block) : void
 学習を行います。
   Array<Array<Numeric>> | SFloat x_train  トレーニング用入力データ。
   Array<Array<Numeric>> | SFloat y_train　トレーニング用正解データ。
@@ -71,8 +69,9 @@ train(x_train, y_train, x_test, y_test, epochs,
                                               nilを指定すると、エポックごとにテストを行いません。
   Float border  学習の早期終了判定に使用するテストデータの正答率。
                 nilの場合、学習の早期終了を行いません。
-  Proc &block(SFloat x, SFloat y) : Array<SFloat>  入力層のミニバッチを取得します。ブロックの戻り値は、ミニバッチを[x, y]の
-                                                   形で指定してください。入力層をミニバッチ単位で正規化したい場合に使用します。
+  Proc func(SFloat x, SFloat y) : Array<SFloat>  入力層のミニバッチを取得します。ブロックの戻り値は、ミニバッチを[x, y]の
+                                                 形で指定してください。入力層をミニバッチ単位で正規化したい場合に使用します。
+  Proc block(Integer epoch) : void  1エポックの学習が終わった後で行いたい処理を、ブロックで渡します。
 
 test(x_test, y_test, tolerance = 0.5, &block) : Float
 テストデータを用いて、テストを行います。
@@ -95,7 +94,7 @@ accurate(x_test, y_test, tolera)
   戻り値  テストデータの正答率。
 
 learn(x_train, y_train, &block) : Float
-入力データを元に、1回だけ学習を行います。途中で学習を切り上げるなど、柔軟な学習を行いたい場合に使用します。
+入力データを元に、1回だけ学習を行います。柔軟な学習を行いたい場合に使用します。
   Array<Array<Numeric>> | SFloat x_train  入力データ
   Array<Array<Numeric>> | SFloat y_train  正解データ
   Proc &block(SFloat x, SFloat y) : Array<SFloat>  入力層のミニバッチを取得します。ブロックの戻り値は、ミニバッチを[x, y]の
@@ -106,42 +105,23 @@ learn(x_train, y_train, &block) : Float
 
 run(x) : Array<Array<Numeric>>
 入力データから出力値を二次元配列で得ます。
-  Array<Array<Float>> | SFloat x  入力データ
+  Array<Array<Float>> x  入力データ
+  戻り値  出力ノードの値
+
+run(x) : SFloat
+入力データから出力値をSFloat形式で得ます。
+  SFloat x  入力データ
   戻り値  出力ノードの値
 
 save(file_name) : void
+学習結果をMarshal形式で保存します。
+  String file_name  書き込むMarshalファイル名
+
+save_json(file_name) : void
 学習結果をJSON形式で保存します。
   String file_name  書き込むJSONファイル名
 
 
-[サンプル1 XOR]
-
-#ライブラリの読み込み
-require "nn"
-
-x = [
-  [0, 0],
-  [1, 0],
-  [0, 1],
-  [1, 1],
-]
-
-y = [[0], [1], [1], [0]]
-
-#ニューラルネットワークの初期化
-nn = NN.new([2, 4, 1], #ノード数
-  learning_rate: 0.1, #学習率
-  batch_size: 4, #ミニバッチの数
-  activation: [:sigmoid, :identity] #活性化関数
-)
-
-#学習を行う
-nn.train(x, y, 20000)
-
-#学習結果の確認
-p nn.run(x)
-
-
 [MNISTデータを読み込む]
 MNISTをRubyでも簡単に試せるよう、MNISTを扱うためのモジュールを用意しました。
 次のリンク(http://yann.lecun.com/exdb/mnist/)から、
@@ -156,54 +136,10 @@ MNIST.load_trainで学習用データを読み込み、MNIST.load_testでテス
 (RubyでのMNISTの読み込みは、以下のリンクを参考にさせていただきました。)
 http://d.hatena.ne.jp/n_shuyo/20090913/mnist
 
-
-[サンプル2 MNIST]
-
-#ライブラリの読み込み
-require "nn"
-require "nn/mnist"
-
-#MNISTのトレーニング用データを読み込む
-x_train, y_train = MNIST.load_train
-
-#y_trainを10クラスに配列でカテゴライズする
-y_train = MNIST.categorical(y_train)
-
-#MNISTのテスト用データを読み込む
-x_test, y_test = MNIST.load_test
-
-#y_testを10クラスにカテゴライズする
-y_test = MNIST.categorical(y_test)
-
-puts "load mnist"
-
-#ニューラルネットワークの初期化
-nn = NN.new([784, 100, 100, 10], #ノード数
-  learning_rate: 0.1, #学習率
-  batch_size: 100, #ミニバッチの数
-  activation: [:relu, :softmax], #活性化関数
-  momentum: 0.9, #モーメンタム係数
-  use_batch_norm: true, #バッチノーマライゼーションを使用する
-)
-
-#学習を行う
-nn.train(x_train, y_train, 10, test: [x_test, y_test]) do |x_batch, y_batch|
-  x_batch /= 255 #ミニバッチを0~1の範囲で正規化
-  [x_batch, y_batch]
-end
-
-#学習結果のテストを行う
-nn.test(x_test, y_test) do |x_batch, y_batch|
-  x_batch /= 255 #ミニバッチを0~1の範囲で正規化
-  [x_batch, y_batch]
-end
-
-
 [お断り]
 作者は、ニューラルネットワークを勉強し始めたばかりの初心者です。
 そのため、バグや実装のミスもあるかと思いますが、温かい目で見守っていただけると、幸いでございます。
 
-
 [更新履歴]
 2018/3/8  バージョン1.0公開
 2018/3/11 バージョン1.1公開
@@ -213,3 +149,6 @@ end
 2018/3/22 バージョン1.5公開
 2018/4/15 バージョン1.6公開
 2018/5/4  バージョン1.8公開
+2018/5/16 バージョン2.0公開
+2018/6/10 バージョン2.0.1公開
+2018/6/10 バージョン2.1.0公開

data/lib/nn.rb

@@ -2,7 +2,7 @@ require "numo/narray"
 require "json"
 
 class NN
-  VERSION = "1.8"
+  VERSION = "2.0"
 
   include Numo
 
@@ -64,37 +64,19 @@ class NN
     nn
   end
 
-  def train(x_train, y_train, epochs,
-            learning_rate_decay: 0,
-            save_dir: nil,
-            save_interval: 1,
-            test: nil,
-            border: nil,
-            tolerance: 0.5,
-            &block)
+  def train(x_train, y_train, epochs, func = nil, &block)
     num_train_data = x_train.is_a?(SFloat) ? x_train.shape[0] : x_train.length
     (1..epochs).each do |epoch|
       loss = nil
       (num_train_data.to_f / @batch_size).ceil.times do
-        loss = learn(x_train, y_train, &block)
+        loss = learn(x_train, y_train, &func)
         if loss.nan?
           puts "loss is nan"
           return
         end
       end
-      if save_dir && epoch % save_interval == 0
-        save("#{save_dir}/epoch#{epoch}.json")
-      end
-      msg = "epoch #{epoch}/#{epochs} loss: #{loss}"
-      if test
-        acc = accurate(*test, tolerance, &block)
-        puts "#{msg} accurate: #{acc}"
-        break if border && acc >= border
-      else
-        puts msg
-      end
-      @learning_rate -= learning_rate_decay
-      @learning_rate = 1e-7 if @learning_rate < 1e-7
+      puts "epoch #{epoch}/#{epochs} loss: #{loss}"
+      block.call(epoch) if block
     end
   end
 
@@ -160,9 +142,11 @@ class NN
   end
 
   def run(x)
-    x = SFloat.cast(x) if x.is_a?(Array)
-    out = forward(x, false)
-    out.to_a
+    if x.is_a?(Array)
+      forward(SFloat.cast(x), false).to_a
+    else
+      forward(x, false)
+    end
   end
 
   def save(file_name)
@@ -257,22 +241,30 @@ class NN
   def update_weight_and_bias
     @layers.select{|layer| layer.is_a?(Affine)}.each.with_index do |layer, i|
       weight_amount = layer.d_weight.mean(0) * @learning_rate
-      @weight_amounts[i] = weight_amount + @momentum * @weight_amounts[i]
-      @weights[i] -= @weight_amounts[i]
       bias_amount = layer.d_bias.mean * @learning_rate
-      @bias_amounts[i] = bias_amount + @momentum * @bias_amounts[i]
-      @biases[i] -= @bias_amounts[i]
+      if @momentum > 0
+        weight_amount += @momentum * @weight_amounts[i]
+        @weight_amounts[i] = weight_amount
+        bias_amount += @momentum * @bias_amounts[i]
+        @bias_amounts[i] = bias_amount
+      end
+      @weights[i] -= weight_amount
+      @biases[i] -= bias_amount
     end
   end
 
   def update_gamma_and_beta
     @layers.select{|layer| layer.is_a?(BatchNorm)}.each.with_index do |layer, i|
       gamma_amount = layer.d_gamma.mean * @learning_rate
-      @gamma_amounts[i] = gamma_amount + @momentum * @gamma_amounts[i]
-      @gammas[i] -= @gamma_amounts[i]
       beta_amount = layer.d_beta.mean * @learning_rate
-      @beta_amounts[i] = beta_amount + @momentum * @beta_amounts[i]
-      @betas[i] -= @beta_amounts[i]
+      if @momentum > 0
+        gamma_amount += @momentum * @gamma_amounts[i]
+        @gamma_amounts[i] = gamma_amount
+        beta_amount += @momentum * @beta_amounts[i]
+        @beta_amounts[i] = beta_amount
+      end
+      @gammas[i] -= gamma_amount
+      @betas[i] -= gamma_amount
     end
   end
 end
@@ -298,8 +290,11 @@ class NN::Affine
 
   def backward(dout)
     x = @x.reshape(*@x.shape, 1)
-    d_ridge = @nn.weight_decay * @nn.weights[@index]
-    @d_weight = x.dot(dout.reshape(dout.shape[0], 1, dout.shape[1])) + d_ridge
+    @d_weight = x.dot(dout.reshape(dout.shape[0], 1, dout.shape[1]))
+    if @nn.weight_decay > 0
+      dridge = @nn.weight_decay * @nn.weights[@index]
+      @d_weight += dridge
+    end
     @d_bias = dout
     dout.dot(@nn.weights[@index].transpose)
   end

data/nn.gemspec

@@ -5,7 +5,7 @@ require "nn"
 
 Gem::Specification.new do |spec|
   spec.name          = "nn"
-  spec.version       = NN::VERSION
+  spec.version       = NN::VERSION + ".0"
   spec.authors       = ["unagiootoro"]
   spec.email         = ["ootoro838861@outlook.jp"]
 

data/nn.rb

@@ -0,0 +1,441 @@
+require "numo/narray"
+require "json"
+
+class NN
+  VERSION = "2.1"
+
+  include Numo
+
+  attr_accessor :weights
+  attr_accessor :biases
+  attr_accessor :gammas
+  attr_accessor :betas
+  attr_accessor :learning_rate
+  attr_accessor :batch_size
+  attr_accessor :activation
+  attr_accessor :momentum
+  attr_accessor :weight_decay
+  attr_accessor :dropout_ratio
+  attr_reader :training
+
+  def initialize(num_nodes,
+                 learning_rate: 0.01,
+                 batch_size: 1,
+                 activation: %i(relu identity),
+                 momentum: 0,
+                 weight_decay: 0,
+                 use_dropout: false,
+                 dropout_ratio: 0.5,
+                 use_batch_norm: false)
+    SFloat.srand(rand(2 ** 64))
+    @num_nodes = num_nodes
+    @learning_rate = learning_rate
+    @batch_size = batch_size
+    @activation = activation
+    @momentum = momentum
+    @weight_decay = weight_decay
+    @use_dropout = use_dropout
+    @dropout_ratio = dropout_ratio
+    @use_batch_norm = use_batch_norm
+    init_weight_and_bias
+    init_gamma_and_beta if @use_batch_norm
+    @training = true
+    init_layers
+  end
+
+  def self.load(file_name)
+    Marshal.load(File.binread(file_name))
+  end
+
+  def self.load_json(file_name)
+    json = JSON.parse(File.read(file_name))
+    nn = self.new(json["num_nodes"],
+      learning_rate: json["learning_rate"],
+      batch_size: json["batch_size"],
+      activation: json["activation"].map(&:to_sym),
+      momentum: json["momentum"],
+      weight_decay: json["weight_decay"],
+      use_dropout: json["use_dropout"],
+      dropout_ratio: json["dropout_ratio"],
+      use_batch_norm: json["use_batch_norm"],
+    )
+    nn.weights = json["weights"].map{|weight| SFloat.cast(weight)}
+    nn.biases = json["biases"].map{|bias| SFloat.cast(bias)}
+    if json["use_batch_norm"]
+      nn.gammas = json["gammas"].map{|gamma| SFloat.cast(gamma)}
+      nn.betas = json["betas"].map{|beta| SFloat.cast(beta)}
+    end
+    nn
+  end
+
+  def train(x_train, y_train, epochs, func = nil, &block)
+    num_train_data = x_train.is_a?(SFloat) ? x_train.shape[0] : x_train.length
+    (1..epochs).each do |epoch|
+      loss = nil
+      (num_train_data.to_f / @batch_size).ceil.times do
+        loss = learn(x_train, y_train, &func)
+        if loss.nan?
+          puts "loss is nan"
+          return
+        end
+      end
+      puts "epoch #{epoch}/#{epochs} loss: #{loss}"
+      block.call(epoch) if block
+    end
+  end
+
+  def test(x_test, y_test, tolerance = 0.5, &block)
+    acc = accurate(x_test, y_test, tolerance, &block)
+    puts "accurate: #{acc}"
+    acc
+  end
+
+  def accurate(x_test, y_test, tolerance = 0.5, &block)
+    correct = 0
+    num_test_data = x_test.is_a?(SFloat) ? x_test.shape[0] : x_test.length
+    (num_test_data.to_f / @batch_size).ceil.times do |i|
+      x = SFloat.zeros(@batch_size, @num_nodes.first)
+      y = SFloat.zeros(@batch_size, @num_nodes.last)
+      @batch_size.times do |j|
+        k = i * @batch_size + j
+        break if k >= num_test_data
+        if x_test.is_a?(SFloat)
+          x[j, true] = x_test[k, true]
+          y[j, true] = y_test[k, true]
+        else
+          x[j, true] = SFloat.cast(x_test[k])
+          y[j, true] = SFloat.cast(y_test[k])
+        end
+      end
+      x, y = block.call(x, y) if block
+      out = forward(x, false)
+      @batch_size.times do |j|
+        vout = out[j, true]
+        vy = y[j, true]
+        case @activation[1]
+        when :identity
+          correct += 1 unless (NMath.sqrt((vout - vy) ** 2) < tolerance).to_a.include?(0)
+        when :softmax
+          correct += 1 if vout.max_index == vy.max_index
+        end
+      end
+    end
+    correct.to_f / num_test_data
+  end
+
+  def learn(x_train, y_train, &block)
+    x = SFloat.zeros(@batch_size, @num_nodes.first)
+    y = SFloat.zeros(@batch_size, @num_nodes.last)
+    @batch_size.times do |i|
+      if x_train.is_a?(SFloat)
+        r = rand(x_train.shape[0])
+        x[i, true] = x_train[r, true]
+        y[i, true] = y_train[r, true]
+      else
+        r = rand(x_train.length)
+        x[i, true] = SFloat.cast(x_train[r])
+        y[i, true] = SFloat.cast(y_train[r])
+      end
+    end
+    x, y = block.call(x, y) if block
+    forward(x)
+    backward(y)
+    update_weight_and_bias
+    update_gamma_and_beta if @use_batch_norm
+    @layers[-1].loss(y)
+  end
+
+  def run(x)
+    if x.is_a?(Array)
+      forward(SFloat.cast(x), false).to_a
+    else
+      forward(x, false)
+    end
+  end
+
+  def save(file_name)
+    File.binwrite(file_name, Marshal.dump(self))
+  end
+
+  def save_json(file_name)
+    json = {
+      "version" => VERSION,
+      "num_nodes" => @num_nodes,
+      "learning_rate" => @learning_rate,
+      "batch_size" => @batch_size,
+      "activation" => @activation,
+      "momentum" => @momentum,
+      "weight_decay" => @weight_decay,
+      "use_dropout" => @use_dropout,
+      "dropout_ratio" => @dropout_ratio,
+      "use_batch_norm" => @use_batch_norm,
+      "weights" => @weights.map(&:to_a),
+      "biases" => @biases.map(&:to_a),
+    }
+    if @use_batch_norm
+      json_batch_norm = {
+        "gammas" => @gammas,
+        "betas" => @betas
+      }
+      json.merge!(json_batch_norm)
+    end
+    File.write(file_name, JSON.dump(json))
+  end
+
+  private
+
+  def init_weight_and_bias
+    @weights = Array.new(@num_nodes.length - 1)
+    @biases = Array.new(@num_nodes.length - 1)
+    @weight_amounts = Array.new(@num_nodes.length - 1, 0)
+    @bias_amounts = Array.new(@num_nodes.length - 1, 0)
+    @num_nodes[0...-1].each_index do |i|
+      weight = SFloat.new(@num_nodes[i], @num_nodes[i + 1]).rand_norm
+      bias = SFloat.new(@num_nodes[i + 1]).rand_norm
+      if @activation[0] == :relu
+        @weights[i] = weight / Math.sqrt(@num_nodes[i]) * Math.sqrt(2)
+        @biases[i] = bias / Math.sqrt(@num_nodes[i]) * Math.sqrt(2)
+      else
+        @weights[i] = weight / Math.sqrt(@num_nodes[i])
+        @biases[i] = bias / Math.sqrt(@num_nodes[i])
+      end
+    end
+  end
+
+  def init_gamma_and_beta
+    @gammas = Array.new(@num_nodes.length - 2, 1)
+    @betas = Array.new(@num_nodes.length - 2, 0)
+    @gamma_amounts = Array.new(@num_nodes.length - 2, 0)
+    @beta_amounts = Array.new(@num_nodes.length - 2, 0)
+  end
+
+  def init_layers
+    @layers = []
+    @num_nodes[0...-2].each_index do |i|
+      @layers << Affine.new(self, i)
+      @layers << BatchNorm.new(self, i) if @use_batch_norm
+      @layers << case @activation[0]
+      when :sigmoid
+        Sigmoid.new
+      when :relu
+        ReLU.new
+      end
+      @layers << Dropout.new(self) if @use_dropout
+    end
+    @layers << Affine.new(self, -1)
+    @layers << case @activation[1]
+    when :identity
+      Identity.new(self)
+    when :softmax
+      Softmax.new(self)
+    end
+  end
+
+  def forward(x, training = true)
+    @training = training
+    @layers.each do |layer|
+      x = layer.forward(x)
+    end
+    x
+  end
+
+  def backward(y)
+    dout = @layers[-1].backward(y)
+    @layers[0...-1].reverse.each do |layer|
+      dout = layer.backward(dout)
+    end
+  end
+
+  def update_weight_and_bias
+    @layers.select{|layer| layer.is_a?(Affine)}.each.with_index do |layer, i|
+      weight_amount = layer.d_weight * @learning_rate
+      bias_amount = layer.d_bias * @learning_rate
+      if @momentum > 0
+        weight_amount += @momentum * @weight_amounts[i]
+        @weight_amounts[i] = weight_amount
+        bias_amount += @momentum * @bias_amounts[i]
+        @bias_amounts[i] = bias_amount
+      end
+      @weights[i] -= weight_amount
+      @biases[i] -= bias_amount
+    end
+  end
+
+  def update_gamma_and_beta
+    @layers.select{|layer| layer.is_a?(BatchNorm)}.each.with_index do |layer, i|
+      gamma_amount = layer.d_gamma * @learning_rate
+      beta_amount = layer.d_beta * @learning_rate
+      if @momentum > 0
+        gamma_amount += @momentum * @gamma_amounts[i]
+        @gamma_amounts[i] = gamma_amount
+        beta_amount += @momentum * @beta_amounts[i]
+        @beta_amounts[i] = beta_amount
+      end
+      @gammas[i] -= gamma_amount
+      @betas[i] -= gamma_amount
+    end
+  end
+end
+
+
+class NN::Affine
+  include Numo
+
+  attr_reader :d_weight
+  attr_reader :d_bias
+
+  def initialize(nn, index)
+    @nn = nn
+    @index = index
+    @d_weight = nil
+    @d_bias = nil
+  end
+
+  def forward(x)
+    @x = x
+    @x.dot(@nn.weights[@index]) + @nn.biases[@index]
+  end
+
+  def backward(dout)
+    x = @x.reshape(*@x.shape, 1)
+    @d_weight = x.dot(dout.reshape(dout.shape[0], 1, dout.shape[1])).mean(0)
+    if @nn.weight_decay > 0
+      dridge = @nn.weight_decay * @nn.weights[@index]
+      @d_weight += dridge
+    end
+    @d_bias = dout.mean
+    dout.dot(@nn.weights[@index].transpose)
+  end
+end
+
+
+class NN::Sigmoid
+  include Numo
+
+  def forward(x)
+    @out = 1.0 / (1 + NMath.exp(-x))
+  end
+
+  def backward(dout)
+    dout * (1.0 - @out) * @out
+  end
+end
+
+
+class NN::ReLU
+  def forward(x)
+    @x = x.clone
+    x[x < 0] = 0
+    x
+  end
+
+  def backward(dout)
+    @x[@x > 0] = 1.0
+    @x[@x <= 0] = 0.0
+    dout * @x
+  end
+end
+
+
+class NN::Identity
+  def initialize(nn)
+    @nn = nn
+  end
+
+  def forward(x)
+    @out = x
+  end
+
+  def backward(y)
+    @out - y
+  end
+
+  def loss(y)
+    ridge = 0.5 * @nn.weight_decay * @nn.weights.reduce(0){|sum, weight| sum + (weight ** 2).sum}
+    0.5 * ((@out - y) ** 2).sum / @nn.batch_size + ridge
+  end
+end
+
+
+class NN::Softmax
+  include Numo
+
+  def initialize(nn)
+    @nn = nn
+  end
+
+  def forward(x)
+    @out = NMath.exp(x) / NMath.exp(x).sum(1).reshape(x.shape[0], 1)
+  end
+
+  def backward(y)
+    @out - y
+  end
+
+  def loss(y)
+    ridge = 0.5 * @nn.weight_decay * @nn.weights.reduce(0){|sum, weight| sum + (weight ** 2).sum}
+    -(y * NMath.log(@out + 1e-7)).sum / @nn.batch_size + ridge
+  end
+end
+
+
+class NN::Dropout
+  include Numo
+
+  def initialize(nn)
+    @nn = nn
+    @mask = nil
+  end
+
+  def forward(x)
+    if @nn.training
+      @mask = SFloat.ones(*x.shape).rand < @nn.dropout_ratio
+      x[@mask] = 0
+    else
+      x *= (1 - @nn.dropout_ratio)
+    end
+    x
+  end
+
+  def backward(dout)
+    dout[@mask] = 0 if @nn.training
+    dout
+  end
+end
+
+
+class NN::BatchNorm
+  include Numo
+
+  attr_reader :d_gamma
+  attr_reader :d_beta
+
+  def initialize(nn, index)
+    @nn = nn
+    @index = index
+  end
+
+  def forward(x)
+    @x = x
+    @mean = x.mean(0)
+    @xc = x - @mean
+    @var = (@xc ** 2).mean(0)
+    @std = NMath.sqrt(@var + 1e-7)
+    @xn = @xc / @std
+    out = @nn.gammas[@index] * @xn + @nn.betas[@index]
+    out.reshape(*@x.shape)
+  end
+
+  def backward(dout)
+    @d_beta = dout.sum(0).mean
+    @d_gamma = (@xn * dout).sum(0).mean
+    dxn = @nn.gammas[@index] * dout
+    dxc = dxn / @std
+    dstd = -((dxn * @xc) / (@std ** 2)).sum(0)
+    dvar = 0.5 * dstd / @std
+    dxc += (2.0 / @nn.batch_size) * @xc * dvar
+    dmean = dxc.sum(0)
+    dx = dxc - dmean / @nn.batch_size
+    dx.reshape(*@x.shape)
+  end
+end

data/sample/cifar10_program.rb

@@ -0,0 +1,38 @@
+require "nn"
+require "nn/cifar10"
+
+x_train = []
+y_train = []
+
+(1..5).each do |i|
+  x_train2, y_train2 = CIFAR10.load_train(i)
+  x_train.concat(x_train2)
+  y_train.concat(CIFAR10.categorical(y_train2))
+end
+GC.start
+
+x_test, y_test = CIFAR10.load_test
+y_test = CIFAR10.categorical(y_test)
+GC.start
+
+puts "load cifar10"
+
+nn = NN.new([3072, 100, 100, 10],
+  learning_rate: 0.1,
+  batch_size: 32,
+  activation: [:relu, :softmax],
+  momentum: 0.9,
+  use_dropout: true,
+  dropout_ratio: 0.2,
+  use_batch_norm: true,
+)
+
+func = -> x, y do
+  x /= 255
+  [x, y]
+end
+
+nn.train(x_train, y_train, 20, func) do |epoch|
+  nn.test(x_test, y_test, &func)
+  nn.learning_rate *= 0.99
+end

data/sample/mnist_program.rb

@@ -0,0 +1,38 @@
+#ライブラリの読み込み
+require "nn"
+require "nn/mnist"
+
+#MNISTのトレーニング用データを読み込む
+x_train, y_train = MNIST.load_train
+
+#y_trainを10クラスに配列でカテゴライズする
+y_train = MNIST.categorical(y_train)
+
+#MNISTのテスト用データを読み込む
+x_test, y_test = MNIST.load_test
+
+#y_testを10クラスにカテゴライズする
+y_test = MNIST.categorical(y_test)
+
+puts "load mnist"
+
+#ニューラルネットワークの初期化
+nn = NN.new([784, 100, 100, 10], #ノード数
+  learning_rate: 0.1, #学習率
+  batch_size: 100, #ミニバッチの数
+  activation: [:relu, :softmax], #活性化関数
+  momentum: 0.9, #モーメンタム係数
+  use_batch_norm: true, #バッチノーマライゼーションを使用する
+)
+
+#ミニバッチを0~1の範囲で正規化
+func = -> x_batch, y_batch do
+  x_batch /= 255
+  [x_batch, y_batch]
+end
+
+#学習を行う
+nn.train(x_train, y_train, 10, func) do
+  #学習結果のテストを行う
+  nn.test(x_test, y_test, &func)
+end

data/sample/xor.rb

@@ -0,0 +1,24 @@
+#ライブラリの読み込み
+require "nn"
+
+x = [
+  [0, 0],
+  [1, 0],
+  [0, 1],
+  [1, 1],
+]
+
+y = [[0], [1], [1], [0]]
+
+#ニューラルネットワークの初期化
+nn = NN.new([2, 4, 1], #ノード数
+  learning_rate: 0.1, #学習率
+  batch_size: 4, #ミニバッチの数
+  activation: [:sigmoid, :identity] #活性化関数
+)
+
+#学習を行う
+nn.train(x, y, 20000)
+
+#学習結果の確認
+p nn.run(x)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nn
 version: !ruby/object:Gem::Version
-  version: '1.8'
+  version: 2.0.0
 platform: ruby
 authors:
 - unagiootoro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-05-03 00:00:00.000000000 Z
+date: 2018-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -70,6 +70,10 @@ files:
 - lib/nn/cifar10.rb
 - lib/nn/mnist.rb
 - nn.gemspec
+- nn.rb
+- sample/cifar10_program.rb
+- sample/mnist_program.rb
+- sample/xor.rb
 homepage: https://github.com/unagiootoro/nn.git
 licenses:
 - MIT