ruby-dnn 0.8.8 → 0.9.0

data/lib/dnn/core/cnn_layers.rb

@@ -84,17 +84,17 @@ module DNN
 
       def self.load_hash(hash)
         Conv2D.new(hash[:num_filters], hash[:filter_size],
-                   weight_initializer: Util.load_hash(hash[:weight_initializer]),
-                   bias_initializer: Util.load_hash(hash[:bias_initializer]),
+                   weight_initializer: Utils.load_hash(hash[:weight_initializer]),
+                   bias_initializer: Utils.load_hash(hash[:bias_initializer]),
                    strides: hash[:strides],
                    padding: hash[:padding],
                    l1_lambda: hash[:l1_lambda],
                    l2_lambda: hash[:l2_lambda])
       end
 
-      def build(model)
+      def build(input_shape)
         super
-        prev_h, prev_w = prev_layer.shape[0..1]
+        prev_h, prev_w = input_shape[0..1]
         @out_size = out_size(prev_h, prev_w, *@filter_size, @strides)
         out_w, out_h = @out_size
         if @padding
@@ -120,7 +120,7 @@ module DNN
         @padding ? back_padding(dx, @pad) : dx
       end
 
-      def shape
+      def output_shape
         [*@out_size, @num_filters]
       end
 
@@ -134,7 +134,7 @@ module DNN
       private
     
       def init_params
-        num_prev_filter = prev_layer.shape[2]
+        num_prev_filter = @input_shape[2]
         @weight.data = Xumo::SFloat.new(num_prev_filter * @filter_size.reduce(:*), @num_filters)
         @bias.data = Xumo::SFloat.new(@num_filters)
         super()
@@ -164,10 +164,10 @@ module DNN
         @padding = padding
       end
 
-      def build(model)
+      def build(input_shape)
         super
-        prev_w, prev_h = prev_layer.shape[0..1]
-        @num_channel = prev_layer.shape[2]
+        prev_h, prev_w = input_shape[0..1]
+        @num_channel = input_shape[2]
         @out_size = out_size(prev_h, prev_w, *@pool_size, @strides)
         out_w, out_h = @out_size
         if @padding
@@ -176,7 +176,7 @@ module DNN
         end
       end
 
-      def shape
+      def output_shape
         [*@out_size, @num_channel]
       end
 
@@ -251,14 +251,14 @@ module DNN
         UnPool2D.new(hash[:unpool_size])
       end
 
-      def build(model)
+      def build(input_shape)
         super
-        prev_h, prev_w = prev_layer.shape[0..1]
+        prev_h, prev_w = input_shape[0..1]
         unpool_h, unpool_w = @unpool_size
         out_h = prev_h * unpool_h
         out_w = prev_w * unpool_w
         @out_size = [out_h, out_w]
-        @num_channel = prev_layer.shape[2]
+        @num_channel = input_shape[2]
       end
 
       def forward(x)
@@ -275,7 +275,7 @@ module DNN
         dout[true, true, 0, true, 0, true].clone
       end
 
-      def shape
+      def output_shape
         [*@out_size, @num_channel]
       end
 

data/lib/dnn/core/dataset.rb

@@ -0,0 +1,18 @@
+class DNN::Dataset
+  def initialize(x_datas, y_datas)
+    @x_datas = x_datas
+    @y_datas = y_datas
+    @num_datas = x_datas.shape[0]
+    @indexes = @num_datas.times.to_a.shuffle
+  end
+
+  def get_batch(batch_size)
+    if @indexes.length < batch_size
+      @indexes = @num_datas.times.to_a.shuffle
+    end
+    batch_indexes = @indexes.shift(batch_size)
+    x_batch = @x_datas[batch_indexes, false]
+    y_batch = @y_datas[batch_indexes, false]
+    [x_batch, y_batch]
+  end
+end

data/lib/dnn/core/initializers.rb

@@ -2,13 +2,16 @@ module DNN
   module Initializers
 
     class Initializer
-      # Classes that inherit from this class must implement this method.
+      def initialize(seed = false)
+        @seed = seed == true ? rand(1 << 31) : seed
+      end
+
       def init_param(layer, param)
         raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'init_params'")
       end
 
       def to_hash(merge_hash = nil)
-        hash = {class: self.class.name}
+        hash = {class: self.class.name, seed: @seed}
         hash.merge!(merge_hash) if merge_hash
         hash
       end
@@ -23,11 +26,14 @@ module DNN
 
 
     class Const < Initializer
+      attr_reader :const
+
       def self.load_hash(hash)
         self.new(hash[:const])
       end
 
       def initialize(const)
+        super()
         @const = const
       end
 
@@ -46,15 +52,17 @@ module DNN
       attr_reader :std
       
       def self.load_hash(hash)
-        self.new(hash[:mean], hash[:std])
+        self.new(hash[:mean], hash[:std], hash[:seed])
       end
 
-      def initialize(mean = 0, std = 0.05)
+      def initialize(mean = 0, std = 0.05, seed = true)
+        super(seed)
         @mean = mean
         @std = std
       end
 
       def init_param(layer, param)
+        Xumo::SFloat.srand(@seed)
         param.data = param.data.rand_norm(@mean, @std)
       end
 
@@ -69,15 +77,17 @@ module DNN
       attr_reader :max
 
       def self.load_hash(hash)
-        self.new(hash[:min], hash[:max])
+        self.new(hash[:min], hash[:max], hash[:seed])
       end
 
-      def initialize(min = -0.05, max = 0.05)
+      def initialize(min = -0.05, max = 0.05, seed = true)
+        super(seed)
         @min = min
         @max = max
       end
 
       def init_param(layer, param)
+        Xumo::SFloat.srand(@seed)
         param.data = param.data.rand(@min, @max)
       end
 
@@ -88,16 +98,26 @@ module DNN
     
     
     class Xavier < Initializer
+      def initialize(seed = true)
+        super
+      end
+
       def init_param(layer, param)
-        num_prev_nodes = layer.prev_layer.shape.reduce(:*)
+        Xumo::SFloat.srand(@seed)
+        num_prev_nodes = layer.input_shape.reduce(:*)
         param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes)
       end
     end
     
     
     class He < Initializer
+      def initialize(seed = true)
+        super
+      end
+
       def init_param(layer, param)
-        num_prev_nodes = layer.prev_layer.shape.reduce(:*)
+        Xumo::SFloat.srand(@seed)
+        num_prev_nodes = layer.input_shape.reduce(:*)
         param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes) * Math.sqrt(2)
       end
     end

data/lib/dnn/core/layers.rb

@@ -3,13 +3,15 @@ module DNN
 
     # Super class of all optimizer classes.
     class Layer
+      attr_reader :input_shape
+
       def initialize
         @built = false
       end
 
       # Build the layer.
-      def build(model)
-        @model = model
+      def build(input_shape)
+        @input_shape = input_shape
         @built = true
       end
       
@@ -19,20 +21,17 @@ module DNN
       end
 
       # Forward propagation.
-      # Classes that inherit from this class must implement this method.
       def forward(x)
         raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
       end
 
       # Backward propagation.
-      # Classes that inherit from this class must implement this method.
       def backward(dout)
         raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'update'")
       end
-    
-      # Get the shape of the layer.
-      def shape
-        prev_layer.shape
+
+      def output_shape
+        @input_shape
       end
 
       # Layer to a hash.
@@ -41,11 +40,6 @@ module DNN
         hash.merge!(merge_hash) if merge_hash
         hash
       end
-    
-      # Get the previous layer.
-      def prev_layer
-        @model.get_prev_layer(self)
-      end
     end
     
     
@@ -60,8 +54,8 @@ module DNN
         @trainable = true
       end
     
-      def build(model)
-        @model = model
+      def build(input_shape)
+        @input_shape = input_shape
         unless @built
           @built = true
           init_params
@@ -69,14 +63,13 @@ module DNN
       end
     
       # Update the parameters.
-      def update
-        @model.optimizer.update(@params) if @trainable
+      def update(optimizer)
+        optimizer.update(@params) if @trainable
       end
     
       private
       
       # Initialize of the parameters.
-      # Classes that inherit from this class must implement this method.
       def init_params
         raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'init_params'")
       end
@@ -84,15 +77,18 @@ module DNN
     
     
     class InputLayer < Layer
-      attr_reader :shape
-
       def self.load_hash(hash)
-        self.new(hash[:shape])
+        self.new(hash[:input_shape])
       end
 
-      def initialize(dim_or_shape)
+      def initialize(input_dim_or_shape)
         super()
-        @shape = dim_or_shape.is_a?(Array) ? dim_or_shape : [dim_or_shape]
+        @input_shape = input_dim_or_shape.is_a?(Array) ? input_dim_or_shape : [input_dim_or_shape]
+      end
+
+      def build
+        @built = true
+        @input_shape
       end
 
       def forward(x)
@@ -104,7 +100,7 @@ module DNN
       end
 
       def to_hash
-        super({shape: @shape})
+        super({input_shape: @input_shape})
       end
     end
 
@@ -113,6 +109,8 @@ module DNN
     class Connection < HasParamLayer
       attr_reader :l1_lambda # L1 regularization
       attr_reader :l2_lambda # L2 regularization
+      attr_reader :weight_initializer
+      attr_reader :bias_initializer
 
       def initialize(weight_initializer: Initializers::RandomNormal.new,
                      bias_initializer: Initializers::Zeros.new,
@@ -143,7 +141,7 @@ module DNN
         end
       end
 
-      def dlasso
+      def d_lasso
         if @l1_lambda > 0
           dlasso = Xumo::SFloat.ones(*@weight.data.shape)
           dlasso[@weight.data < 0] = -1
@@ -151,7 +149,7 @@ module DNN
         end
       end
 
-      def dridge
+      def d_ridge
         if @l2_lambda > 0
           @weight.grad += @l2_lambda * @weight.data
         end
@@ -178,8 +176,8 @@ module DNN
 
       def self.load_hash(hash)
         self.new(hash[:num_nodes],
-                 weight_initializer: Util.load_hash(hash[:weight_initializer]),
-                 bias_initializer: Util.load_hash(hash[:bias_initializer]),
+                 weight_initializer: Utils.load_hash(hash[:weight_initializer]),
+                 bias_initializer: Utils.load_hash(hash[:bias_initializer]),
                  l1_lambda: hash[:l1_lambda],
                  l2_lambda: hash[:l2_lambda])
       end
@@ -205,7 +203,7 @@ module DNN
         dout.dot(@weight.data.transpose)
       end
     
-      def shape
+      def output_shape
         [@num_nodes]
       end
 
@@ -216,7 +214,7 @@ module DNN
       private
     
       def init_params
-        num_prev_nodes = prev_layer.shape[0]
+        num_prev_nodes = @input_shape[0]
         @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
         @bias.data = Xumo::SFloat.new(@num_nodes)
         super()
@@ -226,90 +224,64 @@ module DNN
 
     class Flatten < Layer
       def forward(x)
-        @shape = x.shape
-        x.reshape(x.shape[0], x.shape[1..-1].reduce(:*))
+        x.reshape(x.shape[0], *output_shape)
       end
     
       def backward(dout)
-        dout.reshape(*@shape)
+        dout.reshape(dout.shape[0], *@input_shape)
       end
-    
-      def shape
-        [prev_layer.shape.reduce(:*)]
+
+      def output_shape
+        [@input_shape.reduce(:*)]
       end
     end
 
 
     class Reshape < Layer
-      attr_reader :shape
-      
-      def initialize(shape)
-        super()
-        @shape = shape
-        @x_shape = nil
+      def self.load_hash(hash)
+        self.new(hash[:output_shape])
       end
 
-      def self.load_hash(hash)
-        self.new(hash[:shape])
+      def initialize(output_shape)
+        super()
+        @output_shape = output_shape
       end
 
       def forward(x)
-        @x_shape = x.shape
-        x.reshape(x.shape[0], *@shape)
+        x.reshape(x.shape[0], *@output_shape)
       end
 
       def backward(dout)
-        dout.reshape(*@x_shape)
-      end
-
-      def to_hash
-        super({shape: @shape})
-      end
-    end
-
-
-    class OutputLayer < Layer
-      # Classes that inherit from this class must implement this method.
-      def loss(x)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
+        dout.reshape(dout.shape[0], *@input_shape)
       end
 
-      def dloss
-        @model.get_all_layers.select { |layer| layer.is_a?(Connection) }.each do |layer|
-          layer.dlasso
-          layer.dridge
-        end
+      def output_shape
+        @output_shape
       end
-      
-      private
 
-      def lasso
-        @model.get_all_layers.select { |layer| layer.is_a?(Connection) }
-                             .reduce(0) { |sum, layer| sum + layer.lasso }
-      end
-    
-      def ridge
-        @model.get_all_layers.select { |layer| layer.is_a?(Connection) }
-                             .reduce(0) { |sum, layer| sum + layer.ridge }
+      def to_hash
+        super({output_shape: @output_shape})
       end
     end
-    
+
     
     class Dropout < Layer
       attr_reader :dropout_ratio
 
       def self.load_hash(hash)
-        self.new(hash[:dropout_ratio])
+        self.new(hash[:dropout_ratio], hash[:seed])
       end
 
-      def initialize(dropout_ratio = 0.5)
+      def initialize(dropout_ratio = 0.5, seed = rand(1 << 31))
         super()
         @dropout_ratio = dropout_ratio
+        @seed = seed
         @mask = nil
       end
-    
-      def forward(x)
-        if @model.training?
+
+      def forward(x, learning_phase)
+        if learning_phase
+          Xumo::SFloat.srand(@seed)
           @mask = Xumo::SFloat.ones(*x.shape).rand < @dropout_ratio
           x[@mask] = 0
         else
@@ -318,13 +290,13 @@ module DNN
         x
       end
     
-      def backward(dout)
-        dout[@mask] = 0 if @model.training?
+      def backward(dout, learning_phase)
+        dout[@mask] = 0 if learning_phase
         dout
       end
 
       def to_hash
-        super({dropout_ratio: @dropout_ratio})
+        super({dropout_ratio: @dropout_ratio, seed: @seed})
       end
     end
     
@@ -341,8 +313,8 @@ module DNN
         @momentum = momentum
       end
 
-      def forward(x)
-        if @model.training?
+      def forward(x, learning_phase)
+        if learning_phase
           mean = x.mean(0)
           @xc = x - mean
           var = (@xc**2).mean(0)
@@ -358,7 +330,7 @@ module DNN
         @gamma.data * xn + @beta.data
       end
     
-      def backward(dout)
+      def backward(dout, learning_phase)
         batch_size = dout.shape[0]
         @beta.grad = dout.sum(0)
         @gamma.grad = (@xn * dout).sum(0)
@@ -378,10 +350,10 @@ module DNN
       private
     
       def init_params
-        @params[:gamma] = @gamma = Param.new(Xumo::SFloat.ones(*shape))
-        @params[:beta] = @beta = Param.new(Xumo::SFloat.zeros(*shape))
-        @params[:running_mean] = @running_mean = Param.new(Xumo::SFloat.zeros(*shape))
-        @params[:running_var] = @running_var = Param.new(Xumo::SFloat.zeros(*shape))
+        @params[:gamma] = @gamma = Param.new(Xumo::SFloat.ones(*output_shape))
+        @params[:beta] = @beta = Param.new(Xumo::SFloat.zeros(*output_shape))
+        @params[:running_mean] = @running_mean = Param.new(Xumo::SFloat.zeros(*output_shape))
+        @params[:running_var] = @running_var = Param.new(Xumo::SFloat.zeros(*output_shape))
       end
     end
   end