ruby-dnn 0.5.8 → 0.5.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '040681c989e47e7c183f46ff921285db8c5fd541112f1922b07262c27383d96a'
-  data.tar.gz: 84a1130bed58297aac0414c3eb5842aec3635e4f8aeae826fd1f7d7fbdecdc97
+  metadata.gz: c6e9ccaa1dd5279ca0c9b41ac4ae9573e6ca0ca35c0ce9450bb6e3a9024eae31
+  data.tar.gz: 2c1247ccf2b6a906f39fb5cbbdda9b837e54453c429ee041e8e1736513ff6ed5
 SHA512:
-  metadata.gz: 39c4c5b6f2557ae9bb39d67d248c33488f20aba8ec665221d744b9f1ee1f10f310e88e38689a53202536d58cc19ccf1b5cb2579bd50589888d6fd0318b451628
-  data.tar.gz: 7d2f2311a1123b5bc34fac29b0f7ee73b5638121ef1771a385be9a129a1fea49a3b0385ccb98fa2f7f0985e26cc681b167a20ff31b5926e15c16cc8c640b3ce9
+  metadata.gz: 04d4050badafcd44004c92e34fec1a57c8318da220a8372eccc381bed67741b60ae01a95b57eb56a4c70c0591a10ba8ca643bf86399f471dd606acb25503bd99
+  data.tar.gz: 67528cd4ca207086c6e54d8e5bd232b072638a189e57b2fb45433ab7322ce35424b2d5657bb52f31188d867b9163dd4280f8505dbe61c9e11b26e3b8f0b7ffd0

data/API-Reference.ja.md

@@ -2,7 +2,7 @@
 ruby-dnnのAPIリファレンスです。このリファレンスでは、APIを利用するうえで必要となるクラスとメソッドしか記載していません。
 そのため、プログラムの詳細が必要な場合は、ソースコードを参照してください。
 
-最終更新バージョン:0.5.7
+最終更新バージョン:0.5.9
 
 # module DNN
 ruby-dnnの名前空間をなすモジュールです。
@@ -661,6 +661,20 @@ Float momentum
   モーメンタム係数。
 
 
+# class Nesterov < SGD
+Nesterovによるオプティマイザです。
+
+## 【Instance methods】
+
+## def initialize(learning_rate = 0.01, momentum: 0.9)
+コンストラクタ。
+### arguments
+* Float learning_rate  
+  学習率。
+* Float momentum  
+  モーメンタム係数。
+
+
 # class AdaGrad < Optimizer
 AdaGradによるオプティマイザです。
 

data/lib/dnn/core/model.rb

@@ -102,6 +102,9 @@ module DNN
               verbose: true,
               batch_proc: nil,
               &epoch_proc)
+      unless compiled?
+        raise DNN_Error.new("The model is not compiled.")
+      end
       @batch_size = batch_size
       num_train_data = x.shape[0]
       (1..epochs).each do |epoch|
@@ -168,7 +171,11 @@ module DNN
         x_batch, y_batch = batch_proc.call(x_batch, y_batch) if batch_proc
         out = forward(x_batch, false)
         @batch_size.times do |j|
-         correct += 1 if out[j, true].max_index == y_batch[j, true].max_index
+          if @layers[-1].shape == [1]
+            correct += 1 if out[j, 0].round == y_batch[j, 0].round
+          else
+            correct += 1 if out[j, true].max_index == y_batch[j, true].max_index
+          end
         end
       end
       correct.to_f / x.shape[0]
@@ -183,6 +190,9 @@ module DNN
     end
   
     def forward(x, training)
+      unless compiled?
+        raise DNN_Error.new("The model is not compiled.")
+      end
       @training = training
       @layers.each do |layer|
         x = layer.forward(x)

data/lib/dnn/core/optimizers.rb

@@ -50,6 +50,27 @@ module DNN
         super({momentum: @momentum})
       end
     end
+
+
+    class Nesterov < SGD
+      def self.load_hash(hash)
+        self.new(hash[:learning_rate], momentum: hash[:momentum])
+      end
+
+      def initialize(learning_rate = 0.01, momentum: 0.9)
+        super(learning_rate, momentum: momentum)
+      end
+    
+      def update(layer)
+        @v[layer] ||= {}
+        layer.params.each_key do |key|
+          @v[layer][key] ||= 0
+          amount = layer.grads[key] * @learning_rate
+          @v[layer][key] = @v[layer][key] * @momentum - amount
+          layer.params[key] = (layer.params[key] + @momentum**2 * @v[layer][key]) - (1 + @momentum) * amount
+        end
+      end
+    end
     
     
     class AdaGrad < Optimizer

data/lib/dnn/core/rnn_layers.rb

@@ -126,5 +126,165 @@ module DNN
       end
     end
 
+
+    class LSTM_Dense
+      include Xumo
+
+      def initialize(params, grads)
+        @params = params
+        @grads = grads
+        @tanh = Tanh.new
+        @g_tanh = Tanh.new
+        @forget_sigmoid = Sigmoid.new
+        @in_sigmoid = Sigmoid.new
+        @out_sigmoid = Sigmoid.new
+      end
+
+      def forward(x, h, cell)
+        @x = x
+        @h = h
+        @cell = cell
+        num_nodes = h.shape[1]
+        a = x.dot(@params[:weight]) + h.dot(@params[:weight2]) + @params[:bias]
+
+        @forget = @forget_sigmoid.forward(a[true, 0...num_nodes])
+        @g = @g_tanh.forward(a[(num_nodes * 2)...(num_nodes * 3)])
+        @in = @in_sigmoid.forward(a[true, num_nodes...(num_nodes * 2)])
+        @out = @out_sigmoid.forward(a[true, (num_nodes * 3)..-1])
+
+        @cell2 = @forget * cell + @g * @in
+        @tanh_cell2 = @tanh.forward(@cell2)
+        @h2 = @out * @tanh_cell2
+        [@h2, @cell2]
+      end
+
+      def backward(dh2, dcell2)
+        dh2_tmp = @tanh_cell2 * dh2
+        dcell2_tmp = @tanh.backward(@out * dh2) + dcell2
+
+        dout = @out_sigmoid.backward(dh2_tmp)
+        din = @in_sigmoid.backward(dcell2_tmp * @g)
+        dg = @g_tanh.backward(dcell2_tmp * @in)
+        dforget = @forget_sigmoid.backward(dcell2_tmp * @cell)
+
+        da = SFloat.hstack([dforget, dg, din, dout])
+
+        @grads[:weight] += @x.transpose.dot(da)
+        @grads[:weight2] += @h.transpose.dot(da)
+        @grads[:bias] += da.sum(0)
+        dx = da.dot(@params[:weight].transpose)
+        dh = da.dot(@params[:weight2].transpose)
+        dcell = dcell2_tmp * @forget
+        [dx, dh, dcell]
+      end
+    end
+
+
+    # In development
+    class LSTM < HasParamLayer
+      include Initializers
+      include Activations
+
+      attr_reader :num_nodes
+      attr_reader :stateful
+      attr_reader :weight_decay
+
+      def self.load_hash(hash)
+        self.new(hash[:num_nodes],
+                 stateful: hash[:stateful],
+                 weight_initializer: Util.load_hash(hash[:weight_initializer]),
+                 bias_initializer: Util.load_hash(hash[:bias_initializer]),
+                 weight_decay: hash[:weight_decay])
+      end
+
+      def initialize(num_nodes,
+                     stateful: false,
+                     weight_initializer: nil,
+                     bias_initializer: nil,
+                     weight_decay: 0)
+        super()
+        @num_nodes = num_nodes
+        @stateful = stateful
+        @weight_initializer = (weight_initializer || RandomNormal.new)
+        @bias_initializer = (bias_initializer || Zeros.new)
+        @weight_decay = weight_decay
+        @layers = []
+        @h = nil
+        @cell = nil
+      end
+
+      def forward(xs)
+        @xs_shape = xs.shape
+        hs = SFloat.zeros(xs.shape[0], *shape)
+        h = nil
+        cell = nil
+        if @stateful
+          h = @h if @h
+          cell = @cell if @cell
+        end
+        h ||= SFloat.zeros(xs.shape[0], @num_nodes)
+        cell ||= SFloat.zeros(xs.shape[0], @num_nodes)
+        xs.shape[1].times do |t|
+          x = xs[true, t, false]
+          h, cell = @layers[t].forward(x, h, cell)
+          hs[true, t, false] = h
+        end
+        @h = h
+        @cell = cell
+        hs
+      end
+
+      def backward(dh2s)
+        @grads[:weight] = SFloat.zeros(*@params[:weight].shape)
+        @grads[:weight2] = SFloat.zeros(*@params[:weight2].shape)
+        @grads[:bias] = SFloat.zeros(*@params[:bias].shape)
+        dxs = SFloat.zeros(@xs_shape)
+        dh = 0
+        dcell = 0
+        (0...dh2s.shape[1]).to_a.reverse.each do |t|
+          dh2 = dh2s[true, t, false]
+          dx, dh, dcell = @layers[t].backward(dh2 + dh, dcell)
+          dxs[true, t, false] = dx
+        end
+        dxs
+      end
+
+      def shape
+        [@time_length, @num_nodes]
+      end
+
+      def ridge
+        if @weight_decay > 0
+          0.5 * (@weight_decay * (@params[:weight]**2).sum + @weight_decay * (@params[:weight]**2).sum)
+        else
+          0
+        end
+      end
+
+      def to_hash
+        super({num_nodes: @num_nodes,
+               stateful: @stateful,
+               weight_initializer: @weight_initializer.to_hash,
+               bias_initializer: @bias_initializer.to_hash,
+               weight_decay: @weight_decay})
+      end
+
+      private
+    
+      def init_params
+        @time_length = prev_layer.shape[0] 
+        num_prev_nodes = prev_layer.shape[1]
+        @params[:weight] = SFloat.new(num_prev_nodes, @num_nodes * 4)
+        @params[:weight2] = SFloat.new(@num_nodes, @num_nodes * 4)
+        @params[:bias] = SFloat.new(@num_nodes * 4)
+        @weight_initializer.init_param(self, :weight)
+        @weight_initializer.init_param(self, :weight2)
+        @bias_initializer.init_param(self, :bias)
+        @time_length.times do |t|
+          @layers << LSTM_Dense.new(@params, @grads)
+        end
+      end
+    end
+
   end
 end

data/lib/dnn/version.rb

@@ -1,3 +1,3 @@
 module DNN
-  VERSION = "0.5.8"
+  VERSION = "0.5.9"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby-dnn
 version: !ruby/object:Gem::Version
-  version: 0.5.8
+  version: 0.5.9
 platform: ruby
 authors:
 - unagiootoro