ruby-dnn 0.7.3 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c2ef751fa76ea2918c5f706e1828eba514385b831dc042fa561a08d38da977a
-  data.tar.gz: b906f6a9c63620378b284a2be3ca3bb8e007534b945493b9f8323f77c012cd0f
+  metadata.gz: 308a84624d71cb5b27d4b72f1ea69880dbfebf226ac9a5b44cf5775cc7e22703
+  data.tar.gz: baa0b39dcca002f79eb660129cf0042dfda7d4768e9b2067d7135ce877047770
 SHA512:
-  metadata.gz: 94e7af50f4cdc556b252e3afed0703aa7d9e9d519f2e1e24253a1a27b388fe9d00444edfc271343365dc39e8a02d326df27e923216834509cb639986576664de
-  data.tar.gz: 5fc8acfeed67fff5b41247ab6c3f03b68f96f48bdda877951e23d1fb12f2cc6c113a6102b8b531cd41d1c596d2ab6fa96a243607d38182d804ab1f5396e5439c
+  metadata.gz: a98278bb5cfd211bcf1231a4f5b0f13fa9d7ee4684d0334a6d2052d77b405d1873a65bd11d1f2df2f1f29926de484d4d0a7cbd8ef8d88cb4735923904eeb91fd
+  data.tar.gz: 06d52f9698bf600e0bb63c7cc703697d3bb1ec9827871e6dd595a7fdd17cf3e63980d8abe376c9418db9412309b5aed39187c8b988388fd2bbaf09afc4702aa7

data/API-Reference.ja.md

@@ -2,7 +2,7 @@
 ruby-dnnのAPIリファレンスです。このリファレンスでは、APIを利用するうえで必要となるクラスとメソッドしか記載していません。
 そのため、プログラムの詳細が必要な場合は、ソースコードを参照してください。
 
-最終更新バージョン:0.7.0
+最終更新バージョン:0.8.0
 
 # module DNN
 ruby-dnnの名前空間をなすモジュールです。
@@ -299,7 +299,7 @@ Hash
 ## def initialize(dim_or_shape)
 コンストラクタ
 ### arguments
-* Integer|Array dim_or_shape  
+* Integer | Array dim_or_shape  
 入力層のdimentionまたはshapeを指定します。引数がIntegerだとdimentionとみなし、Arrayだとshapeとみなします。
 
 
@@ -600,15 +600,11 @@ Numo::SFloat y
 
 ## 【Instance methods】
 
-## def initialize(momentum: 0.9, running_mean: nil, running_var: nil)
+## def initialize(momentum: 0.9
 コンストラクタ。
 ### arguments
 * Float momenum: 0.9  
 推論時に使用する平均と分散を求めるための指数平均移動の係数。
-* Float running_mean: nil  
-推論時に使用する平均。
-* Float running_var: nil  
-推論時に使用する分散。
 
 
 # module Activations
@@ -745,11 +741,11 @@ Float max
 一様分布の最大値。
 
 ## 【Instance methods】
-## def initialize(min = -0.25, max = 0.25)
+## def initialize(min = -0.05, max = 0.05)
 ### arguments
-* Float min = -0.25  
+* Float min = -0.05  
 一様分布の最小値。
-* Float max = 0.25  
+* Float max = 0.05  
 一様分布の最大値。
 
 
@@ -782,8 +778,8 @@ Float learning_rate
 * Float learning_rate  
   Optimizerの学習率。
 
-## abstruct def update(layer)
-layerのgradsを元に、layerのparamsを更新します。全てのOptimizerを継承するクラスは、このメソッドを実装する必要があります。
+## abstruct def update(params)
+paramsが持つ全ての学習パラメータにおいて、gradを元に、dataを更新します。全てのOptimizerを継承するクラスは、このメソッドを実装する必要があります。
 ### arguments
 * Layer layer  
   paramsを更新するレイヤー。

data/lib/dnn.rb

@@ -12,6 +12,7 @@ module DNN; end
 require_relative "dnn/version"
 require_relative "dnn/core/error"
 require_relative "dnn/core/model"
+require_relative "dnn/core/learning_param"
 require_relative "dnn/core/initializers"
 require_relative "dnn/core/layers"
 require_relative "dnn/core/activations"

data/lib/dnn/core/cnn_layers.rb

@@ -1,6 +1,6 @@
 module DNN
   module Layers
-    #This module is used for convolution.
+    # This module is used for convolution.
     module Conv2DModule
       private
 
@@ -107,20 +107,20 @@ module DNN
         x = padding(x, @pad) if @padding
         @x_shape = x.shape
         @col = im2col(x, *@out_size, *@filter_size, @strides)
-        out = @col.dot(@params[:weight]) + @params[:bias]
+        out = @col.dot(@weight.data) + @bias.data
         out.reshape(x.shape[0], *@out_size, out.shape[3])
       end
 
       def backward(dout)
         dout = dout.reshape(dout.shape[0..2].reduce(:*), dout.shape[3])
-        @grads[:weight] = @col.transpose.dot(dout)
+        @weight.grad = @col.transpose.dot(dout)
         if @l1_lambda > 0
-          @grads[:weight] += dlasso
+          @weight.grad += dlasso
         elsif @l2_lambda > 0
-          @grads[:weight] += dridge
+          @weight.grad += dridge
         end
-        @grads[:bias] = dout.sum(0)
-        dcol = dout.dot(@params[:weight].transpose)
+        @bias.grad = dout.sum(0)
+        dcol = dout.dot(@weight.data.transpose)
         dx = col2im(dcol, @x_shape, *@out_size, *@filter_size, @strides)
         @padding ? back_padding(dx, @pad) : dx
       end
@@ -140,13 +140,14 @@ module DNN
     
       def init_params
         num_prev_filter = prev_layer.shape[2]
-        @params[:weight] = Xumo::SFloat.new(num_prev_filter * @filter_size.reduce(:*), @num_filters)
-        @params[:bias] = Xumo::SFloat.new(@num_filters)
+        @weight.data = Xumo::SFloat.new(num_prev_filter * @filter_size.reduce(:*), @num_filters)
+        @bias.data = Xumo::SFloat.new(@num_filters)
         super()
       end
     end
 
-    #Super class of all pooling2D class.
+    
+    # Super class of all pooling2D class.
     class Pool2D < Layer
       include Conv2DModule
 
@@ -185,8 +186,7 @@ module DNN
       end
 
       def to_hash
-        super({pool_width: @pool_width,
-               pool_height: @pool_height,
+        super({pool_size: @pool_size,
                strides: @strides,
                padding: @padding})
       end

data/lib/dnn/core/initializers.rb

@@ -2,9 +2,8 @@ module DNN
   module Initializers
 
     class Initializer
-      def init_param(layer, param_key, param)
-        layer.params[param_key] = param
-      end
+      # Classes that inherit from this class must implement this method.
+      # def init_param(param) end
 
       def to_hash(merge_hash = nil)
         hash = {class: self.class.name}
@@ -15,8 +14,8 @@ module DNN
 
 
     class Zeros < Initializer
-      def init_param(layer, param_key)
-        super(layer, param_key, layer.params[param_key].fill(0))
+      def init_param(param)
+        param.data = param.data.fill(0)
       end
     end
     
@@ -34,8 +33,8 @@ module DNN
         @std = std
       end
 
-      def init_param(layer, param_key)
-        super(layer, param_key, layer.params[param_key].rand_norm(@mean, @std))
+      def init_param(param)
+        param.data = param.data.rand_norm(@mean, @std)
       end
 
       def to_hash
@@ -57,8 +56,8 @@ module DNN
         @max = max
       end
 
-      def init_param(layer, param_key)
-        super(layer, param_key, layer.params[param_key].rand(@min, @max))
+      def init_param(param)
+        param.data = param.data.rand(@min, @max)
       end
 
       def to_hash
@@ -68,17 +67,17 @@ module DNN
     
     
     class Xavier < Initializer
-      def init_param(layer, param_key)
-        num_prev_nodes = layer.prev_layer.shape.reduce(:*)
-        super(layer, param_key, layer.params[param_key].rand_norm / Math.sqrt(num_prev_nodes))
+      def init_param(param)
+        num_prev_nodes = param.layer.prev_layer.shape.reduce(:*)
+        param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes)
       end
     end
     
     
     class He < Initializer
-      def init_param(layer, param_key)
-        num_prev_nodes = layer.prev_layer.shape.reduce(:*)
-        super(layer, param_key, layer.params[param_key].rand_norm / Math.sqrt(num_prev_nodes) * Math.sqrt(2))
+      def init_param(param)
+        num_prev_nodes = param.layer.prev_layer.shape.reduce(:*)
+        param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes) * Math.sqrt(2)
       end
     end
 

data/lib/dnn/core/layers.rb

@@ -49,12 +49,10 @@ module DNN
     class HasParamLayer < Layer
       attr_accessor :trainable # Setting false prevents learning of parameters.
       attr_reader :params      # The parameters of the layer.
-      attr_reader :grads       # Differential value of parameter of layer.
     
       def initialize
-        super
+        super()
         @params = {}
-        @grads = {}
         @trainable = true
       end
     
@@ -68,12 +66,13 @@ module DNN
     
       # Update the parameters.
       def update
-        @model.optimizer.update(self) if @trainable
+        @model.optimizer.update(@params) if @trainable
       end
     
       private
       
       # Initialize of the parameters.
+      # Classes that inherit from this class must implement this method.
       def init_params() end
     end
     
@@ -120,11 +119,13 @@ module DNN
         @bias_initializer = (bias_initializer || Zeros.new)
         @l1_lambda = l1_lambda
         @l2_lambda = l2_lambda
+        @params[:weight] = @weight = LearningParam.new(self)
+        @params[:bias] = @bias = LearningParam.new(self)
       end
 
       def lasso
         if @l1_lambda > 0
-          @l1_lambda * @params[:weight].abs.sum
+          @l1_lambda * @weight.data.abs.sum
         else
           0
         end
@@ -132,20 +133,20 @@ module DNN
 
       def ridge
         if @l2_lambda > 0
-          0.5 * @l2_lambda * (@params[:weight]**2).sum
+          0.5 * @l2_lambda * (@weight.data**2).sum
         else
           0
         end
       end
 
       def dlasso
-        dlasso = Xumo::SFloat.ones(*@params[:weight].shape)
-        dlasso[@params[:weight] < 0] = -1
+        dlasso = Xumo::SFloat.ones(*@weight.data.shape)
+        dlasso[@weight.data < 0] = -1
         @l1_lambda * dlasso
       end
 
       def dridge
-        @l2_lambda * @params[:weight]
+        @l2_lambda * @weight.data
       end
 
       def to_hash(merge_hash)
@@ -158,8 +159,8 @@ module DNN
       private
 
       def init_params
-        @weight_initializer.init_param(self, :weight)
-        @bias_initializer.init_param(self, :bias)
+        @weight_initializer.init_param(@weight)
+        @bias_initializer.init_param(@bias)
       end
     end
     
@@ -187,18 +188,18 @@ module DNN
     
       def forward(x)
         @x = x
-        @x.dot(@params[:weight]) + @params[:bias]
+        @x.dot(@weight.data) + @bias.data
       end
     
       def backward(dout)
-        @grads[:weight] = @x.transpose.dot(dout)
+        @weight.grad = @x.transpose.dot(dout)
         if @l1_lambda > 0
-          @grads[:weight] += dlasso
+          @weight.grad += dlasso
         elsif @l2_lambda > 0
-          @grads[:weight] += dridge
+          @weight.grad += dridge
         end
-        @grads[:bias] = dout.sum(0)
-        dout.dot(@params[:weight].transpose)
+        @bias.grad = dout.sum(0)
+        dout.dot(@weight.data.transpose)
       end
     
       def shape
@@ -213,8 +214,8 @@ module DNN
     
       def init_params
         num_prev_nodes = prev_layer.shape[0]
-        @params[:weight] = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
-        @params[:bias] = Xumo::SFloat.new(@num_nodes)
+        @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
+        @bias.data = Xumo::SFloat.new(@num_nodes)
         super()
       end
     end
@@ -317,22 +318,22 @@ module DNN
       attr_reader :momentum
 
       def self.load_hash(hash)
-        running_mean = Xumo::SFloat.cast(hash[:running_mean])
-        running_var = Xumo::SFloat.cast(hash[:running_var])
-        self.new(momentum: hash[:momentum], running_mean: running_mean, running_var: running_var)
+        self.new(momentum: hash[:momentum])
       end
 
-      def initialize(momentum: 0.9, running_mean: nil, running_var: nil)
+      def initialize(momentum: 0.9)
         super()
         @momentum = momentum
-        @running_mean = running_mean
-        @running_var = running_var
+        @params[:gamma] = @gamma = LearningParam.new(self)
+        @params[:beta] = @beta = LearningParam.new(self)
+        @params[:running_mean] = nil
+        @params[:running_var] = nil
       end
 
       def build(model)
         super
-        @running_mean ||= Xumo::SFloat.zeros(*shape)
-        @running_var ||= Xumo::SFloat.zeros(*shape)
+        @params[:running_mean] ||= Xumo::SFloat.zeros(*shape)
+        @params[:running_var] ||= Xumo::SFloat.zeros(*shape)
       end
 
       def forward(x)
@@ -343,20 +344,20 @@ module DNN
           @std = Xumo::NMath.sqrt(var + 1e-7)
           xn = @xc / @std
           @xn = xn
-          @running_mean = @momentum * @running_mean + (1 - @momentum) * mean
-          @running_var = @momentum * @running_var + (1 - @momentum) * var
+          @params[:running_mean] = @momentum * @params[:running_mean] + (1 - @momentum) * mean
+          @params[:running_var] = @momentum * @params[:running_var] + (1 - @momentum) * var
         else
-          xc = x - @running_mean
-          xn = xc / Xumo::NMath.sqrt(@running_var + 1e-7)
+          xc = x - @params[:running_mean]
+          xn = xc / Xumo::NMath.sqrt(@params[:running_var] + 1e-7)
         end
-        @params[:gamma] * xn + @params[:beta]
+        @gamma.data * xn + @beta.data
       end
     
       def backward(dout)
         batch_size = dout.shape[0]
-        @grads[:beta] = dout.sum(0)
-        @grads[:gamma] = (@xn * dout).sum(0)
-        dxn = @params[:gamma] * dout
+        @beta.grad = dout.sum(0)
+        @gamma.grad = (@xn * dout).sum(0)
+        dxn = @gamma.data * dout
         dxc = dxn / @std
         dstd = -((dxn * @xc) / (@std**2)).sum(0)
         dvar = 0.5 * dstd / @std
@@ -366,16 +367,14 @@ module DNN
       end
 
       def to_hash
-        super({momentum: @momentum,
-               running_mean: @running_mean.to_a,
-               running_var: @running_var.to_a})
+        super({momentum: @momentum})
       end
     
       private
     
       def init_params
-        @params[:gamma] = Xumo::SFloat.ones(*shape)
-        @params[:beta] = Xumo::SFloat.zeros(*shape)
+        @gamma.data = Xumo::SFloat.ones(*shape)
+        @beta.data = Xumo::SFloat.zeros(*shape)
       end
     end
   end

data/lib/dnn/core/learning_param.rb

@@ -0,0 +1,9 @@
+class DNN::LearningParam
+  attr_accessor :data
+  attr_accessor :grad
+  attr_reader :layer
+
+  def initialize(layer)
+    @layer = layer
+  end
+end

data/lib/dnn/core/model.rb

@@ -35,7 +35,12 @@ module DNN
         hash_params = has_param_layers_params[has_param_layers_index]
         hash_params.each do |key, (shape, base64_param)|
           bin = Base64.decode64(base64_param)
-          layer.params[key] = Xumo::SFloat.from_binary(bin).reshape(*shape)
+          data = Xumo::SFloat.from_binary(bin).reshape(*shape)
+          if layer.params[key].is_a?(LearningParam)
+            layer.params[key].data = data
+          else
+            layer.params[key] = data
+          end
         end
         has_param_layers_index += 1
       end
@@ -55,13 +60,14 @@ module DNN
     def to_json
       hash_layers = @layers.map { |layer| layer.to_hash }
       hash = {version: VERSION, layers: hash_layers, optimizer: @optimizer.to_hash}
-      JSON.dump(hash)
+      JSON.pretty_generate(hash)
     end
     
     def params_to_json
-      has_param_layers = @layers.select { |layer| layer.is_a?(HasParamLayer) }
+      has_param_layers = @layers.select { |layer| layer.is_a?(Layers::HasParamLayer) }
       has_param_layers_params = has_param_layers.map do |layer|
         layer.params.map { |key, param|
+          param = param.data if param.is_a?(LearningParam)
           base64_param = Base64.encode64(param.to_binary)
           [key, [param.shape, base64_param]]
         }.to_h
@@ -160,7 +166,7 @@ module DNN
       loss_value
     end
   
-    def accurate(x, y, batch_size = 1, &batch_proc)
+    def accurate(x, y, batch_size = 100, &batch_proc)
       input_data_shape_check(x, y)
       batch_size = batch_size >= x.shape[0] ? x.shape[0] : batch_size
       correct = 0

data/lib/dnn/core/optimizers.rb

@@ -9,8 +9,9 @@ module DNN
         @learning_rate = learning_rate
       end
 
-      # Update layer has params.
-      def update(layer) end
+      # Update params.
+      # Classes that inherit from this class must implement this method.
+      # def update(params) end
 
       def to_hash(merge_hash = nil)
         hash = {class: self.class.name, learning_rate: @learning_rate}
@@ -33,16 +34,15 @@ module DNN
         @v = {}
       end
     
-      def update(layer)
-        @v[layer] ||= {}
-        layer.params.each_key do |key|
-          amount = layer.grads[key] * @learning_rate
+      def update(params)
+        params.select { |key, param| param.is_a?(LearningParam) }.each_value do |param|
+          amount = param.grad * @learning_rate
           if @momentum > 0
-            @v[layer][key] ||= 0
-            amount += @momentum * @v[layer][key]
-            @v[layer][key] = amount
+            @v[param] ||= 0
+            amount += @momentum * @v[param]
+            @v[param] = amount
           end
-          layer.params[key] -= amount
+          param.data -= amount
         end
       end
 
@@ -61,13 +61,12 @@ module DNN
         super(learning_rate, momentum: momentum)
       end
     
-      def update(layer)
-        @v[layer] ||= {}
-        layer.params.each_key do |key|
-          @v[layer][key] ||= 0
-          amount = layer.grads[key] * @learning_rate
-          @v[layer][key] = @v[layer][key] * @momentum - amount
-          layer.params[key] = (layer.params[key] + @momentum**2 * @v[layer][key]) - (1 + @momentum) * amount
+      def update(params)
+        params.select { |key, param| param.is_a?(LearningParam) }.each_value do |param|
+          @v[param] ||= 0
+          amount = param.grad * @learning_rate
+          @v[param] = @v[param] * @momentum - amount
+          param.data = (param.data + @momentum**2 * @v[param]) - (1 + @momentum) * amount
         end
       end
     end
@@ -83,12 +82,11 @@ module DNN
         self.new(hash[:learning_rate])
       end
     
-      def update(layer)
-        @g[layer] ||= {}
-        layer.params.each_key do |key|
-          @g[layer][key] ||= 0
-          @g[layer][key] += layer.grads[key]**2
-          layer.params[key] -= (@learning_rate / Xumo::NMath.sqrt(@g[layer][key] + 1e-7)) * layer.grads[key]
+      def update(params)
+        params.select { |key, param| param.is_a?(LearningParam) }.each_value do |param|
+          @g[param] ||= 0
+          @g[param] += param.grad**2
+          param.data -= (@learning_rate / Xumo::NMath.sqrt(@g[param] + 1e-7)) * param.grad
         end
       end
     end
@@ -107,12 +105,11 @@ module DNN
         @g = {}
       end
     
-      def update(layer)
-        @g[layer] ||= {}
-        layer.params.each_key do |key|
-          @g[layer][key] ||= 0
-          @g[layer][key] = @alpha * @g[layer][key] + (1 - @alpha) * layer.grads[key]**2
-          layer.params[key] -= (@learning_rate / Xumo::NMath.sqrt(@g[layer][key] + 1e-7)) * layer.grads[key]
+      def update(params)
+        params.select { |key, param| param.is_a?(LearningParam) }.each_value do |param|
+          @g[param] ||= 0
+          @g[param] = @alpha * @g[param] + (1 - @alpha) * param.grad**2
+          param.data -= (@learning_rate / Xumo::NMath.sqrt(@g[param] + 1e-7)) * param.grad
         end
       end
 
@@ -136,16 +133,14 @@ module DNN
         @s = {}
       end
 
-      def update(layer)
-        @h[layer] ||= {}
-        @s[layer] ||= {}
-        layer.params.each_key do |key|
-          @h[layer][key] ||= Xumo::SFloat.zeros(*layer.params[key].shape)
-          @s[layer][key] ||= Xumo::SFloat.zeros(*layer.params[key].shape)
-          @h[layer][key] = @rho * @h[layer][key] + (1 - @rho) * layer.grads[key]**2
-          v = (Xumo::NMath.sqrt(@s[layer][key] + 1e-6) / Xumo::NMath.sqrt(@h[layer][key] + 1e-6)) * layer.grads[key]
-          @s[layer][key] = @rho * @s[layer][key] + (1 - @rho) * v**2
-          layer.params[key] -= v
+      def update(params)
+        params.select { |key, param| param.is_a?(LearningParam) }.each_value do |param|
+          @h[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @s[param] ||= Xumo::SFloat.zeros(*param.data.shape)
+          @h[param] = @rho * @h[param] + (1 - @rho) * param.grad**2
+          v = (Xumo::NMath.sqrt(@s[param] + 1e-6) / Xumo::NMath.sqrt(@h[param] + 1e-6)) * param.grad
+          @s[param] = @rho * @s[param] + (1 - @rho) * v**2
+          param.data -= v
         end
       end
 
@@ -172,17 +167,15 @@ module DNN
         @v = {}
       end
 
-      def update(layer)
+      def update(params)
         @iter += 1
-        @m[layer] ||= {}
-        @v[layer] ||= {}
         lr = @learning_rate * Math.sqrt(1 - @beta2**@iter) / (1 - @beta1**@iter) 
-        layer.params.each_key do |key|
-          @m[layer][key] ||= 0
-          @v[layer][key] ||= 0
-          @m[layer][key] += (1 - @beta1) * (layer.grads[key] - @m[layer][key])
-          @v[layer][key] += (1 - @beta2) * (layer.grads[key]**2 - @v[layer][key])
-          layer.params[key] -= lr * @m[layer][key] / Xumo::NMath.sqrt(@v[layer][key] + 1e-7)
+        params.select { |key, param| param.is_a?(LearningParam) }.each_value do |param|
+          @m[param] ||= 0
+          @v[param] ||= 0
+          @m[param] += (1 - @beta1) * (param.grad - @m[param])
+          @v[param] += (1 - @beta2) * (param.grad**2 - @v[param])
+          param.data -= lr * @m[param] / Xumo::NMath.sqrt(@v[param] + 1e-7)
         end
       end
 

data/lib/dnn/core/rnn_layers.rb

@@ -5,9 +5,11 @@ module DNN
     class RNN < Connection
       include Activations
 
-      attr_accessor :h
       attr_reader :num_nodes
       attr_reader :stateful
+      attr_reader :weight
+      attr_reader :weight2
+      attr_reader :bias
 
       def initialize(num_nodes,
                      stateful: false,
@@ -22,26 +24,27 @@ module DNN
         @stateful = stateful
         @return_sequences = return_sequences
         @layers = []
-        @h = nil
+        @params[:h] = nil
+        @params[:weight2] = @weight2 = LearningParam.new(self)
       end
 
       def forward(xs)
         @xs_shape = xs.shape
         hs = Xumo::SFloat.zeros(xs.shape[0], @time_length, @num_nodes)
-        h = (@stateful && @h) ? @h : Xumo::SFloat.zeros(xs.shape[0], @num_nodes)
+        h = (@stateful && @params[:h]) ? @params[:h] : Xumo::SFloat.zeros(xs.shape[0], @num_nodes)
         xs.shape[1].times do |t|
           x = xs[true, t, false]
           h = @layers[t].forward(x, h)
           hs[true, t, false] = h
         end
-        @h = h
+        @params[:h] = h
         @return_sequences ? hs : h
       end
 
       def backward(dh2s)
-        @grads[:weight] = Xumo::SFloat.zeros(*@params[:weight].shape)
-        @grads[:weight2] = Xumo::SFloat.zeros(*@params[:weight2].shape)
-        @grads[:bias] = Xumo::SFloat.zeros(*@params[:bias].shape)
+        @weight.grad = Xumo::SFloat.zeros(*@weight.data.shape)
+        @weight2.grad = Xumo::SFloat.zeros(*@weight2.data.shape)
+        @bias.grad = Xumo::SFloat.zeros(*@bias.data.shape)
         unless @return_sequences
           dh = dh2s
           dh2s = Xumo::SFloat.zeros(dh.shape[0], @time_length, dh.shape[1])
@@ -61,8 +64,7 @@ module DNN
         hash = {
           num_nodes: @num_nodes,
           stateful: @stateful,
-          return_sequences: @return_sequences,
-          h: @h.to_a
+          return_sequences: @return_sequences
         }
         hash.merge!(merge_hash) if merge_hash
         super(hash)
@@ -73,12 +75,12 @@ module DNN
       end
 
       def reset_state
-        @h = @h.fill(0) if @h
+        @params[:h] = @params[:h].fill(0) if @params[:h]
       end
 
       def lasso
         if @l1_lambda > 0
-          @l1_lambda * (@params[:weight].abs.sum + @params[:weight2].abs.sum)
+          @l1_lambda * (@weight.data.abs.sum + @weight2.data.abs.sum)
         else
           0
         end
@@ -86,30 +88,30 @@ module DNN
 
       def ridge
         if @l2_lambda > 0
-          0.5 * (@l2_lambda * ((@params[:weight]**2).sum + (@params[:weight2]**2).sum))
+          0.5 * (@l2_lambda * ((@weight.data**2).sum + (@weight2.data**2).sum))
         else
           0
         end
       end
 
       def dlasso
-        dlasso = Xumo::SFloat.ones(*@params[:weight].shape)
-        dlasso[@params[:weight] < 0] = -1
+        dlasso = Xumo::SFloat.ones(*@weight.data.shape)
+        dlasso[@weight.data < 0] = -1
         @l1_lambda * dlasso
       end
 
       def dridge
-        @l2_lambda * @params[:weight]
+        @l2_lambda * @weight.data
       end
 
       def dlasso2
-        dlasso = Xumo::SFloat.ones(*@params[:weight2].shape)
-        dlasso[@params[:weight2] < 0] = -1
+        dlasso = Xumo::SFloat.ones(*@weight2.data.shape)
+        dlasso[@weight2.data < 0] = -1
         @l1_lambda * dlasso
       end
 
       def dridge2
-        @l2_lambda * @params[:weight2]
+        @l2_lambda * @weight2.data
       end
 
       private
@@ -129,30 +131,32 @@ module DNN
       def forward(x, h)
         @x = x
         @h = h
-        h2 = x.dot(@rnn.params[:weight]) + h.dot(@rnn.params[:weight2]) + @rnn.params[:bias]
+        h2 = x.dot(@rnn.weight.data) + h.dot(@rnn.weight2.data) + @rnn.bias.data
         @activation.forward(h2)
       end
 
       def backward(dh2)
         dh2 = @activation.backward(dh2)
-        @rnn.grads[:weight] += @x.transpose.dot(dh2)
-        @rnn.grads[:weight2] += @h.transpose.dot(dh2)
+        @rnn.weight.grad += @x.transpose.dot(dh2)
+        @rnn.weight2.grad += @h.transpose.dot(dh2)
         if @rnn.l1_lambda > 0
-          @rnn.grads[:weight] += dlasso
-          @rnn.grads[:weight2] += dlasso2
+          @rnn.weight.grad += dlasso
+          @rnn.weight2.grad += dlasso2
         elsif @rnn.l2_lambda > 0
-          @rnn.grads[:weight] += dridge
-          @grads[:weight2] += dridge2
+          @rnn.weight.grad += dridge
+          @rnn.weight2.grad += dridge2
         end
-        @rnn.grads[:bias] += dh2.sum(0)
-        dx = dh2.dot(@rnn.params[:weight].transpose)
-        dh = dh2.dot(@rnn.params[:weight2].transpose)
+        @rnn.bias.grad += dh2.sum(0)
+        dx = dh2.dot(@rnn.weight.data.transpose)
+        dh = dh2.dot(@rnn.weight2.data.transpose)
         [dx, dh]
       end
     end
 
 
     class SimpleRNN < RNN
+      attr_reader :activation
+      
       def self.load_hash(hash)
         simple_rnn = self.new(hash[:num_nodes],
                               stateful: hash[:stateful],
@@ -162,7 +166,6 @@ module DNN
                               bias_initializer: Util.load_hash(hash[:bias_initializer]),
                               l1_lambda: hash[:l1_lambda],
                               l2_lambda: hash[:l2_lambda])
-        simple_rnn.h = Xumo::SFloat.cast(hash[:h])
         simple_rnn
       end
 
@@ -193,12 +196,12 @@ module DNN
       def init_params
         super()
         num_prev_nodes = prev_layer.shape[1]
-        @params[:weight] = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
-        @params[:weight2] = Xumo::SFloat.new(@num_nodes, @num_nodes)
-        @params[:bias] = Xumo::SFloat.new(@num_nodes)
-        @weight_initializer.init_param(self, :weight)
-        @weight_initializer.init_param(self, :weight2)
-        @bias_initializer.init_param(self, :bias)
+        @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
+        @weight2.data = Xumo::SFloat.new(@num_nodes, @num_nodes)
+        @bias.data = Xumo::SFloat.new(@num_nodes)
+        @weight_initializer.init_param(@weight)
+        @weight_initializer.init_param(@weight2)
+        @bias_initializer.init_param(@bias)
         @time_length.times do |t|
           @layers << SimpleRNN_Dense.new(self)
         end
@@ -221,7 +224,7 @@ module DNN
         @h = h
         @c = c
         num_nodes = h.shape[1]
-        a = x.dot(@rnn.params[:weight]) + h.dot(@rnn.params[:weight2]) + @rnn.params[:bias]
+        a = x.dot(@rnn.weight.data) + h.dot(@rnn.weight2.data) + @rnn.bias.data
 
         @forget = @forget_sigmoid.forward(a[true, 0...num_nodes])
         @g = @g_tanh.forward(a[true, num_nodes...(num_nodes * 2)])
@@ -245,18 +248,18 @@ module DNN
 
         da = Xumo::SFloat.hstack([dforget, dg, din, dout])
 
-        @rnn.grads[:weight] += @x.transpose.dot(da)
-        @rnn.grads[:weight2] += @h.transpose.dot(da)
+        @rnn.weight.grad += @x.transpose.dot(da)
+        @rnn.weight2.grad += @h.transpose.dot(da)
         if @rnn.l1_lambda > 0
-          @rnn.grads[:weight] += dlasso
-          @rnn.grads[:weight2] += dlasso2
+          @rnn.weight.grad += dlasso
+          @rnn.weight2.grad += dlasso2
         elsif @rnn.l2_lambda > 0
-          @rnn.grads[:weight] += dridge
-          @rnn.grads[:weight2] += dridge2
+          @rnn.weight.grad += dridge
+          @rnn.weight2.grad += dridge2
         end
-        @rnn.grads[:bias] += da.sum(0)
-        dx = da.dot(@rnn.params[:weight].transpose)
-        dh = da.dot(@rnn.params[:weight2].transpose)
+        @rnn.bias.grad += da.sum(0)
+        dx = da.dot(@rnn.weight.data.transpose)
+        dh = da.dot(@rnn.weight2.data.transpose)
         dc = dc2_tmp * @forget
         [dx, dh, dc]
       end
@@ -264,8 +267,6 @@ module DNN
 
 
     class LSTM < RNN
-      attr_accessor :c
-
       def self.load_hash(hash)
         lstm = self.new(hash[:num_nodes],
                         stateful: hash[:stateful],
@@ -274,8 +275,6 @@ module DNN
                         bias_initializer: Util.load_hash(hash[:bias_initializer]),
                         l1_lambda: hash[:l1_lambda],
                         l2_lambda: hash[:l2_lambda])
-        lstm.h = Xumo::SFloat.cast(hash[:h])
-        lstm.c = Xumo::SFloat.cast(hash[:c])
         lstm
       end
 
@@ -287,7 +286,7 @@ module DNN
                      l1_lambda: 0,
                      l2_lambda: 0)
         super
-        @c = nil
+        @params[:c] = nil
       end
 
       def forward(xs)
@@ -296,8 +295,8 @@ module DNN
         h = nil
         c = nil
         if @stateful
-          h = @h if @h
-          c = @c if @c
+          h = @params[:h] if @params[:h]
+          c = @params[:c] if @params[:c]
         end
         h ||= Xumo::SFloat.zeros(xs.shape[0], @num_nodes)
         c ||= Xumo::SFloat.zeros(xs.shape[0], @num_nodes)
@@ -306,15 +305,15 @@ module DNN
           h, c = @layers[t].forward(x, h, c)
           hs[true, t, false] = h
         end
-        @h = h
-        @c = c
+        @params[:h] = h
+        @params[:c] = c
         @return_sequences ? hs : h
       end
 
       def backward(dh2s)
-        @grads[:weight] = Xumo::SFloat.zeros(*@params[:weight].shape)
-        @grads[:weight2] = Xumo::SFloat.zeros(*@params[:weight2].shape)
-        @grads[:bias] = Xumo::SFloat.zeros(*@params[:bias].shape)
+        @weight.grad = Xumo::SFloat.zeros(*@weight.data.shape)
+        @weight2.grad = Xumo::SFloat.zeros(*@weight2.data.shape)
+        @bias.grad = Xumo::SFloat.zeros(*@bias.data.shape)
         unless @return_sequences
           dh = dh2s
           dh2s = Xumo::SFloat.zeros(dh.shape[0], @time_length, dh.shape[1])
@@ -333,11 +332,7 @@ module DNN
 
       def reset_state
         super()
-        @c = @c.fill(0) if @c
-      end
-
-      def to_hash
-        super({c: @c.to_a})
+        @params[:c] = @params[:c].fill(0) if @params[:c]
       end
 
       private
@@ -345,12 +340,12 @@ module DNN
       def init_params
         super()
         num_prev_nodes = prev_layer.shape[1]
-        @params[:weight] = Xumo::SFloat.new(num_prev_nodes, @num_nodes * 4)
-        @params[:weight2] = Xumo::SFloat.new(@num_nodes, @num_nodes * 4)
-        @params[:bias] = Xumo::SFloat.new(@num_nodes * 4)
-        @weight_initializer.init_param(self, :weight)
-        @weight_initializer.init_param(self, :weight2)
-        @bias_initializer.init_param(self, :bias)
+        @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes * 4)
+        @weight2.data = Xumo::SFloat.new(@num_nodes, @num_nodes * 4)
+        @bias.data = Xumo::SFloat.new(@num_nodes * 4)
+        @weight_initializer.init_param(@weight)
+        @weight_initializer.init_param(@weight2)
+        @bias_initializer.init_param(@bias)
         @time_length.times do |t|
           @layers << LSTM_Dense.new(self)
         end
@@ -370,16 +365,16 @@ module DNN
         @x = x
         @h = h
         num_nodes = h.shape[1]
-        @weight_a = @rnn.params[:weight][true, 0...(num_nodes * 2)]
-        @weight2_a = @rnn.params[:weight2][true, 0...(num_nodes * 2)]
-        bias_a = @rnn.params[:bias][0...(num_nodes * 2)]
+        @weight_a = @rnn.weight.data[true, 0...(num_nodes * 2)]
+        @weight2_a = @rnn.weight2.data[true, 0...(num_nodes * 2)]
+        bias_a = @rnn.bias.data[0...(num_nodes * 2)]
         a = x.dot(@weight_a) + h.dot(@weight2_a) + bias_a
         @update = @update_sigmoid.forward(a[true, 0...num_nodes])
         @reset = @reset_sigmoid.forward(a[true, num_nodes..-1])
 
-        @weight_h = @rnn.params[:weight][true, (num_nodes * 2)..-1]
-        @weight2_h = @rnn.params[:weight2][true, (num_nodes * 2)..-1]
-        bias_h = @rnn.params[:bias][(num_nodes * 2)..-1]
+        @weight_h = @rnn.weight.data[true, (num_nodes * 2)..-1]
+        @weight2_h = @rnn.weight2.data[true, (num_nodes * 2)..-1]
+        bias_h = @rnn.bias.data[(num_nodes * 2)..-1]
         @tanh_h = @tanh.forward(x.dot(@weight_h) + (h * @reset).dot(@weight2_h) + bias_h)
         h2 = (1 - @update) * h + @update * @tanh_h
         h2
@@ -404,16 +399,16 @@ module DNN
         dh += da.dot(@weight2_a.transpose)
         dbias_a = da.sum(0)
 
-        @rnn.grads[:weight] += Xumo::SFloat.hstack([dweight_a, dweight_h])
-        @rnn.grads[:weight2] += Xumo::SFloat.hstack([dweight2_a, dweight2_h])
+        @rnn.weight.grad += Xumo::SFloat.hstack([dweight_a, dweight_h])
+        @rnn.weight2.grad += Xumo::SFloat.hstack([dweight2_a, dweight2_h])
         if @rnn.l1_lambda > 0
-          @rnn.grads[:weight] += dlasso
-          @rnn.grads[:weight2] += dlasso2
+          @rnn.weight.grad += dlasso
+          @rnn.weight2.grad += dlasso2
         elsif @rnn.l2_lambda > 0
-          @rnn.grads[:weight] += dridge
-          @rnn.grads[:weight2] += dridge2
+          @rnn.weight.grad += dridge
+          @rnn.weight2.grad += dridge2
         end
-        @rnn.grads[:bias] += Xumo::SFloat.hstack([dbias_a, dbias_h])
+        @rnn.bias.grad += Xumo::SFloat.hstack([dbias_a, dbias_h])
         [dx, dh]
       end
     end
@@ -428,7 +423,6 @@ module DNN
                        bias_initializer: Util.load_hash(hash[:bias_initializer]),
                        l1_lambda: hash[:l1_lambda],
                        l2_lambda: hash[:l2_lambda])
-        gru.h = Xumo::SFloat.cast(hash[:h])
         gru
       end
 
@@ -447,12 +441,12 @@ module DNN
       def init_params
         super()
         num_prev_nodes = prev_layer.shape[1]
-        @params[:weight] = Xumo::SFloat.new(num_prev_nodes, @num_nodes * 3)
-        @params[:weight2] = Xumo::SFloat.new(@num_nodes, @num_nodes * 3)
-        @params[:bias] = Xumo::SFloat.new(@num_nodes * 3)
-        @weight_initializer.init_param(self, :weight)
-        @weight_initializer.init_param(self, :weight2)
-        @bias_initializer.init_param(self, :bias)
+        @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes * 3)
+        @weight2.data = Xumo::SFloat.new(@num_nodes, @num_nodes * 3)
+        @bias.data = Xumo::SFloat.new(@num_nodes * 3)
+        @weight_initializer.init_param(@weight)
+        @weight_initializer.init_param(@weight2)
+        @bias_initializer.init_param(@bias)
         @time_length.times do |t|
           @layers << GRU_Dense.new(self)
         end

data/lib/dnn/version.rb

@@ -1,3 +1,3 @@
 module DNN
-  VERSION = "0.7.3"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-dnn
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.8.0
 platform: ruby
 authors:
 - unagiootoro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-12-17 00:00:00.000000000 Z
+date: 2019-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -111,6 +111,7 @@ files:
 - lib/dnn/core/error.rb
 - lib/dnn/core/initializers.rb
 - lib/dnn/core/layers.rb
+- lib/dnn/core/learning_param.rb
 - lib/dnn/core/model.rb
 - lib/dnn/core/optimizers.rb
 - lib/dnn/core/rnn_layers.rb
@@ -145,8 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: ruby deep learning library.