ruby-dnn 0.9.2 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc2897efaefa857bc21c4a4237c4c8d6fac3ec508708ba0935874050f8dab7f9
-  data.tar.gz: 2eaea58d620043e47c197f1ace3ec94f8108d84dbe3883a0455b64f39c8d6f82
+  metadata.gz: 2926b7a9ca6f3766ee46bba676837bd4cb98d55220ba37e4ef12109ef244a444
+  data.tar.gz: 57db99b2200d3d0946c6bb886f97308f6b09e550f5ab66802f61a7d13bd9b8fe
 SHA512:
-  metadata.gz: e74b37d9d31af87cc833b4237b2f90051199a2020d645b733e5af2eb6f07b26b87debaac651818a95fca27ffeedb3c08a62c55191e3213b6b7a8c85762ba91fd
-  data.tar.gz: b4b25e79cd9ac57e464d70248178048473e4e6a26bcae34da9b5b63f109c336be5f3ccec700a23dc9dce427b7014be81b41ba547fa59ad3f5d9ebb5fa5d881c6
+  metadata.gz: 19c77e661bb6b208c279ce80d85a115572e124bad50dd97f3c8ec1097aa1bf4f245cef11bee58c3ffbc5c8c3a47f4aa6e9e3bb89db0ba8c1c7ff6ef2066764b3
+  data.tar.gz: 9a8d414fcb3401b883bde2cd54f6b320817ee66389e43971f62a4d9f85e1888db36842d704c3c2392f27f68266f6dcd8faf207723dfca25195651966e3713652

data/lib/dnn.rb

@@ -18,6 +18,7 @@ require_relative "dnn/core/initializers"
 require_relative "dnn/core/layers"
 require_relative "dnn/core/activations"
 require_relative "dnn/core/losses"
+require_relative "dnn/core/regularizers"
 require_relative "dnn/core/cnn_layers"
 require_relative "dnn/core/rnn_layers"
 require_relative "dnn/core/optimizers"

data/lib/dnn/core/layers.rb

@@ -135,34 +135,11 @@ module DNN
         @params[:bias] = @bias = Param.new
       end
 
-      def lasso
-        if @l1_lambda > 0
-          @l1_lambda * @weight.data.abs.sum
-        else
-          0
-        end
-      end
-
-      def ridge
-        if @l2_lambda > 0
-          0.5 * @l2_lambda * (@weight.data**2).sum
-        else
-          0
-        end
-      end
-
-      def d_lasso
-        if @l1_lambda > 0
-          dlasso = Xumo::SFloat.ones(*@weight.data.shape)
-          dlasso[@weight.data < 0] = -1
-          @weight.grad += @l1_lambda * dlasso
-        end
-      end
-
-      def d_ridge
-        if @l2_lambda > 0
-          @weight.grad += @l2_lambda * @weight.data
-        end
+      def regularizers
+        regularizers = []
+        regularizers << Lasso.new(@l1_lambda, @weight) if @l1_lambda > 0
+        regularizers << Ridge.new(@l2_lambda, @weight) if @l2_lambda > 0
+        regularizers
       end
 
       def to_hash(merge_hash)

data/lib/dnn/core/losses.rb

@@ -3,19 +3,25 @@ module DNN
 
     class Loss
       def forward(out, y, layers)
-        regularize = layers.select { |layer| layer.is_a?(Connection) }
-                           .reduce(0) { |sum, layer| sum + layer.lasso + layer.ridge }
-        loss(out, y) + regularize
+        loss_value = loss(out, y)
+        regularizers = layers.select { |layer| layer.is_a?(Connection) }
+                             .map { |layer| layer.regularizers }.flatten
+        
+        regularizers.each do |regularizer|
+          loss_value = regularizer.forward(loss_value)
+        end
+        loss_value
       end
 
       def backward(y)
         raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'backward'")
       end
 
-      def d_regularize(layers)
+      def regularizes_backward(layers)
         layers.select { |layer| layer.is_a?(Connection) }.each do |layer|
-          layer.d_lasso
-          layer.d_ridge
+          layer.regularizers.each do |regularizer|
+            regularizer.backward
+          end
         end
       end
 

data/lib/dnn/core/model.rb

@@ -256,7 +256,7 @@ module DNN
       loss_value = @loss.forward(out, y, get_all_layers)
       dout = @loss.backward(y)
       backward(dout)
-      @loss.d_regularize(get_all_layers)
+      @loss.regularizes_backward(get_all_layers)
       update
       loss_value
     end

data/lib/dnn/core/regularizers.rb

@@ -0,0 +1,36 @@
+module DNN
+
+  class Lasso
+    def initialize(l1_lambda, param)
+      @l1_lambda = l1_lambda
+      @param = param
+    end
+
+    def forward(x)
+      x + @l1_lambda * @param.data.abs.sum
+    end
+
+    def backward
+      dlasso = Xumo::SFloat.ones(*@param.data.shape)
+      dlasso[@param.data < 0] = -1
+      @param.grad += @l1_lambda * dlasso
+    end
+  end
+
+
+  class Ridge
+    def initialize(l2_lambda, param)
+      @l2_lambda = l2_lambda
+      @param = param
+    end
+
+    def forward(x)
+      x + 0.5 * @l2_lambda * (@param.data**2).sum
+    end
+
+    def backward
+      @param.grad += @l2_lambda * @param.data
+    end
+  end
+
+end

data/lib/dnn/core/rnn_layers.rb

@@ -9,7 +9,7 @@ module DNN
       attr_reader :num_nodes
       # @return [Bool] Maintain state between batches.
       attr_reader :stateful
-      # @return [Bool] Only the last of each cell of RNN is left.
+      # @return [Bool] Set the false, only the last of each cell of RNN is left.
       attr_reader :return_sequences
 
       def initialize(num_nodes,
@@ -82,38 +82,17 @@ module DNN
         @hidden.data = @hidden.data.fill(0) if @hidden.data
       end
 
-      def lasso
+      def regularizers
+        regularizers = []
         if @l1_lambda > 0
-          @l1_lambda * (@weight.data.abs.sum + @weight2.data.abs.sum)
-        else
-          0
+          regularizers << Lasso.new(@l1_lambda, @weight)
+          regularizers << Lasso.new(@l1_lambda, @weight2)
         end
-      end
-
-      def ridge
-        if @l2_lambda > 0
-          0.5 * (@l2_lambda * ((@weight.data**2).sum + (@weight2.data**2).sum))
-        else
-          0
-        end
-      end
-
-      def d_lasso
-        if @l1_lambda > 0
-          dlasso = Xumo::SFloat.ones(*@weight.data.shape)
-          dlasso[@weight.data < 0] = -1
-          @weight.grad += @l1_lambda * dlasso
-          dlasso2 = Xumo::SFloat.ones(*@weight2.data.shape)
-          dlasso2[@weight2.data < 0] = -1
-          @weight2.grad += @l1_lambda * dlasso2
-        end
-      end
-
-      def d_ridge
         if @l2_lambda > 0
-          @weight.grad += l2_lambda * @weight.data
-          @weight2.grad += l2_lambda * @weight2.data
+          regularizers << Ridge.new(@l2_lambda, @weight)
+          regularizers << Ridge.new(@l2_lambda, @weight2)
         end
+        regularizers
       end
 
       private
@@ -181,8 +160,8 @@ module DNN
               return_sequences: return_sequences,
               weight_initializer: weight_initializer,
               bias_initializer: bias_initializer,
-              l1_lambda: 0,
-              l2_lambda: 0)
+              l1_lambda: l1_lambda,
+              l2_lambda: l2_lambda)
         @activation = activation
       end
 
@@ -202,7 +181,7 @@ module DNN
         @weight_initializer.init_param(self, @weight2)
         @bias_initializer.init_param(self, @bias)
         @time_length.times do |t|
-          @layers << SimpleRNN_Dense.new(@weight, @weight2, @bias)
+          @layers << SimpleRNN_Dense.new(@weight, @weight2, @bias, @activation)
         end
       end
     end

data/lib/dnn/version.rb

@@ -1,3 +1,3 @@
 module DNN
-  VERSION = "0.9.2"
+  VERSION = "0.9.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-dnn
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.3
 platform: ruby
 authors:
 - unagiootoro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-06 00:00:00.000000000 Z
+date: 2019-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -120,6 +120,7 @@ files:
 - lib/dnn/core/model.rb
 - lib/dnn/core/optimizers.rb
 - lib/dnn/core/param.rb
+- lib/dnn/core/regularizers.rb
 - lib/dnn/core/rnn_layers.rb
 - lib/dnn/core/utils.rb
 - lib/dnn/lib/cifar10.rb