ruby-dnn 0.10.4 → 0.12.4

data/lib/dnn/core/embedding.rb

@@ -2,30 +2,42 @@ module DNN
   module Layers
 
     class Embedding < HasParamLayer
-      # @return [Integer] Return the input length.
       attr_reader :input_length
-      # @return [Initializers::Initializer] Return the weight initializer.
+      attr_reader :weight
       attr_reader :weight_initializer
+      attr_reader :weight_regularizer
 
       def self.from_hash(hash)
         self.new(hash[:input_shape], hash[:input_length],
-                 weight_initializer: DNN::Utils.from_hash(hash[:weight_initializer]))
+                 weight_initializer: DNN::Utils.hash_to_obj(hash[:weight_initializer]),
+                 weight_regularizer: DNN::Utils.hash_to_obj(hash[:weight_regularizer]))
       end
-      
+
       # @param [Integer | Array] input_dim_or_shape Set input data dimension or shape.
-      # @param [Integer] input_length input Set the time series length of input data.
-      def initialize(input_dim_or_shape, input_length, weight_initializer: Initializers::RandomUniform.new)
+      # @param [Integer] input_length Set the time series length of input data.
+      # @param [DNN::Initializers::Initializer] weight_initializer Weight initializer.
+      # @param [DNN::Regularizers::Regularizer | NilClass] weight_regularizer Weight regularizer.
+      def initialize(input_dim_or_shape, input_length,
+                     weight_initializer: Initializers::RandomUniform.new,
+                     weight_regularizer: nil)
         super()
         @input_shape = input_dim_or_shape.is_a?(Array) ? input_dim_or_shape : [input_dim_or_shape]
         @input_length = input_length
         @weight_initializer = weight_initializer
+        @weight_regularizer = weight_regularizer
+      end
+
+      def call(input)
+        x, *, learning_phase = *input
+        build unless built?
+        [forward(x), Link.new(nil, self), learning_phase]
       end
 
       def build
         @built = true
-        @params[:weight] = @weight = Param.new(Xumo::SFloat.new(@input_length), 0)
+        @weight = Param.new(Xumo::SFloat.new(@input_length), 0)
         @weight_initializer.init_param(self, @weight)
-        @input_shape
+        @weight_regularizer.param = @weight if @weight_regularizer
       end
 
       def forward(x)
@@ -47,8 +59,17 @@ module DNN
         nil
       end
 
+      def regularizers
+        @weight_regularizer ? [@weight_regularizer] : []
+      end
+
       def to_hash
-        super(input_shape: @input_shape, input_length: @input_length, weight_initializer: @weight_initializer.to_hash)
+        super(input_shape: @input_shape, input_length: @input_length,
+              weight_initializer: @weight_initializer.to_hash, weight_regularizer: @weight_regularizer&.to_hash)
+      end
+
+      def get_params
+        { weight: @weight }
       end
     end
 

data/lib/dnn/core/initializers.rb

@@ -2,16 +2,21 @@ module DNN
   module Initializers
 
     class Initializer
-      def initialize(seed = false)
+      # @param [Boolean | Integer] seed Seed of random number used for masking.
+      #                                 Set true to determine seed as random.
+      def initialize(seed: false)
         @seed = seed == true ? rand(1 << 31) : seed
       end
 
+      # Initialization of learning parameters.
+      # @param [DNN::Layers::Layer] layer Layer that owns learning parameters.
+      # @param [DNN::Param] param Learning parameter to be initialized.
       def init_param(layer, param)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'init_params'")
+        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'init_param'")
       end
 
       def to_hash(merge_hash = nil)
-        hash = {class: self.class.name, seed: @seed}
+        hash = { class: self.class.name, seed: @seed }
         hash.merge!(merge_hash) if merge_hash
         hash
       end
@@ -32,6 +37,7 @@ module DNN
         self.new(hash[:const])
       end
 
+      # @param [Float] const Constant value of initialization.
       def initialize(const)
         super()
         @const = const
@@ -42,21 +48,23 @@ module DNN
       end
 
       def to_hash
-        super({const: @const})
+        super(const: @const)
       end
     end
-    
-    
+
+
     class RandomNormal < Initializer
       attr_reader :mean
       attr_reader :std
-      
+
       def self.from_hash(hash)
-        self.new(hash[:mean], hash[:std], hash[:seed])
+        self.new(hash[:mean], hash[:std], seed: hash[:seed])
       end
 
-      def initialize(mean = 0, std = 0.05, seed = true)
-        super(seed)
+      # @param [Float] mean Average of initialization value.
+      # @param [Float] std Variance of initialization value.
+      def initialize(mean = 0, std = 0.05, seed: true)
+        super(seed: seed)
         @mean = mean
         @std = std
       end
@@ -67,7 +75,7 @@ module DNN
       end
 
       def to_hash
-        super({mean: @mean, std: @std})
+        super(mean: @mean, std: @std)
       end
     end
 
@@ -77,11 +85,13 @@ module DNN
       attr_reader :max
 
       def self.from_hash(hash)
-        self.new(hash[:min], hash[:max], hash[:seed])
+        self.new(hash[:min], hash[:max], seed: hash[:seed])
       end
 
-      def initialize(min = -0.05, max = 0.05, seed = true)
-        super(seed)
+      # @param [Float] min Min of initialization value.
+      # @param [Float] max Max of initialization value.
+      def initialize(min = -0.05, max = 0.05, seed: true)
+        super(seed: seed)
         @min = min
         @max = max
       end
@@ -92,13 +102,13 @@ module DNN
       end
 
       def to_hash
-        super({min: @min, max: @max})
+        super(min: @min, max: @max)
       end
     end
-    
-    
+
+
     class Xavier < Initializer
-      def initialize(seed = true)
+      def initialize(seed: true)
         super
       end
 
@@ -108,10 +118,10 @@ module DNN
         param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes)
       end
     end
-    
-    
+
+
     class He < Initializer
-      def initialize(seed = true)
+      def initialize(seed: true)
         super
       end
 

data/lib/dnn/core/iterator.rb

@@ -0,0 +1,52 @@
+module DNN
+  # This class manages input datas and output datas together.
+  class Iterator
+    # @param [Numo::SFloat] x_datas input datas.
+    # @param [Numo::SFloat] y_datas output datas.
+    # @param [Boolean] random Set true to return batches randomly. Setting false returns batches in order of index.
+    def initialize(x_datas, y_datas, random: true)
+      @x_datas = x_datas
+      @y_datas = y_datas
+      @random = random
+      @num_datas = x_datas.shape[0]
+      reset
+    end
+
+    # Return the next batch.
+    # @param [Integer] batch_size Required batch size.
+    def next_batch(batch_size)
+      raise DNN_Error.new("This iterator has not next batch. Please call reset.") unless has_next?
+      if @indexes.length <= batch_size
+        batch_indexes = @indexes
+        @has_next = false
+      else
+        batch_indexes = @indexes.shift(batch_size)
+      end
+      x_batch = @x_datas[batch_indexes, false]
+      y_batch = @y_datas[batch_indexes, false]
+      [x_batch, y_batch]
+    end
+
+    # Reset input datas and output datas.
+    def reset
+      @has_next = true
+      @indexes = @num_datas.times.to_a
+      @indexes.shuffle! if @random
+    end
+
+    # Return the true if has next batch.
+    def has_next?
+      @has_next
+    end
+
+    def foreach(batch_size, &block)
+      step = 0
+      while has_next?
+        x_batch, y_batch = next_batch(batch_size)
+        block.call(x_batch, y_batch, step)
+        step += 1
+      end
+      reset
+    end
+  end
+end

data/lib/dnn/core/layers.rb

@@ -1,37 +1,48 @@
 module DNN
   module Layers
 
-    # Super class of all optimizer classes.
+    # Super class of all layer classes.
     class Layer
-      # @return [Bool] learning_phase Return the true if learning.
-      attr_accessor :learning_phase
-      # @return [Array] Return the shape of the input data.
       attr_reader :input_shape
 
+      def self.call(x, *args)
+        self.new(*args).(x)
+      end
+
       def initialize
         @built = false
       end
 
+      # Forward propagation and create a link.
+      # @param [Array] input Array of the form [x_input_data, prev_link, learning_phase].
+      def call(input)
+        x, prev_link, learning_phase = *input
+        build(x.shape[1..-1]) unless built?
+        y = forward(x)
+        link = Link.new(prev_link, self)
+        [y, link, learning_phase]
+      end
+
       # Build the layer.
       # @param [Array] input_shape Setting the shape of the input data.
       def build(input_shape)
         @input_shape = input_shape
-        @learning_phase = true
         @built = true
       end
-      
-      # Does the layer have already been built?
-      # @return [Bool] If layer have already been built then return true.
+
+      # @return [Boolean] If layer have already been built then return true.
       def built?
         @built
       end
 
       # Forward propagation.
+      # @param [Numo::SFloat] x Input data.
       def forward(x)
         raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
       end
 
       # Backward propagation.
+      # @param [Numo::SFloat] dy Differential value of output data.
       def backward(dy)
         raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'backward'")
       end
@@ -45,73 +56,88 @@ module DNN
 
       # Layer to a hash.
       def to_hash(merge_hash = nil)
-        hash = {class: self.class.name}
+        hash = { class: self.class.name }
         hash.merge!(merge_hash) if merge_hash
         hash
       end
     end
-    
-    
+
+
     # This class is a superclass of all classes with learning parameters.
     class HasParamLayer < Layer
-      # @return [Bool] trainable Setting false prevents learning of parameters.
+      # @return [Boolean] Setting false prevents learning of parameters.
       attr_accessor :trainable
-      # @return [Array] The parameters of the layer.
-      attr_reader :params
-    
+
       def initialize
         super()
-        @params = {}
         @trainable = true
       end
+
+      # @return [Array] The parameters of the layer.
+      def get_params
+        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'get_params'")
+      end
     end
-    
-    
+
+
     class InputLayer < Layer
+      def self.call(input)
+        shape = input.is_a?(Array) ? input[0].shape : input.shape
+        self.new(shape[1..-1]).(input)
+      end
+
       def self.from_hash(hash)
         self.new(hash[:input_shape])
       end
 
+      # @param [Array] input_dim_or_shape Setting the shape or dimension of the input data.
       def initialize(input_dim_or_shape)
         super()
         @input_shape = input_dim_or_shape.is_a?(Array) ? input_dim_or_shape : [input_dim_or_shape]
       end
 
+      def call(input)
+        build unless built?
+        x, prev_link, learning_phase = *input
+        link = prev_link ? Link.new(prev_link, self) : Link.new(nil, self)
+        [forward(x), link, learning_phase]
+      end
+
       def build
         @built = true
-        @input_shape
       end
 
       def forward(x)
+        unless x.shape[1..-1] == @input_shape
+          raise DNN_ShapeError.new("The shape of x does not match the input shape. input shape is #{@input_shape}, but x shape is #{x.shape[1..-1]}.")
+        end
         x
       end
-    
+
       def backward(dy)
         dy
       end
 
       def to_hash
-        super({input_shape: @input_shape})
+        super(input_shape: @input_shape)
       end
     end
 
 
     # It is a superclass of all connection layers.
     class Connection < HasParamLayer
-      # @return [DNN::Initializers::Initializer] Weight initializer.
+      attr_reader :weight
+      attr_reader :bias
       attr_reader :weight_initializer
-      # @return [DNN::Initializers::Initializer] Bias initializer.
       attr_reader :bias_initializer
-      # @return [DNN::Regularizers::Regularizer] Weight regularization.
       attr_reader :weight_regularizer
-      # @return [DNN::Regularizers::Regularizer] Bias regularization.
       attr_reader :bias_regularizer
 
       # @param [DNN::Initializers::Initializer] weight_initializer Weight initializer.
       # @param [DNN::Initializers::Initializer] bias_initializer Bias initializer.
-      # @param [DNN::Regularizers::Regularizer] weight_regularizer Weight regularization.
-      # @param [DNN::Regularizers::Regularizer] bias_regularizer Bias regularization.
-      # @param [Bool] use_bias whether to use bias.
+      # @param [DNN::Regularizers::Regularizer | NilClass] weight_regularizer Weight regularizer.
+      # @param [DNN::Regularizers::Regularizer | NilClass] bias_regularizer Bias regularizer.
+      # @param [Boolean] use_bias Whether to use bias.
       def initialize(weight_initializer: Initializers::RandomNormal.new,
                      bias_initializer: Initializers::Zeros.new,
                      weight_regularizer: nil,
@@ -122,12 +148,8 @@ module DNN
         @bias_initializer = bias_initializer
         @weight_regularizer = weight_regularizer
         @bias_regularizer = bias_regularizer
-        @params[:weight] = @weight = Param.new(nil, 0)
-        if use_bias
-          @params[:bias] = @bias = Param.new(nil, 0)
-        else
-          @bias = nil
-        end
+        @weight = Param.new(nil, 0)
+        @bias = use_bias ? Param.new(nil, 0) : nil
       end
 
       def regularizers
@@ -137,17 +159,21 @@ module DNN
         regularizers
       end
 
-      # @return [Bool] Return whether to use bias.
+      # @return [Boolean] Return whether to use bias.
       def use_bias
         @bias ? true : false
       end
 
       def to_hash(merge_hash)
-        super({weight_initializer: @weight_initializer.to_hash,
-               bias_initializer: @bias_initializer.to_hash,
-               weight_regularizer: @weight_regularizer&.to_hash,
-               bias_regularizer: @bias_regularizer&.to_hash,
-               use_bias: use_bias}.merge(merge_hash))
+        super({ weight_initializer: @weight_initializer.to_hash,
+                bias_initializer: @bias_initializer.to_hash,
+                weight_regularizer: @weight_regularizer&.to_hash,
+                bias_regularizer: @bias_regularizer&.to_hash,
+                use_bias: use_bias }.merge(merge_hash))
+      end
+
+      def get_params
+        { weight: @weight, bias: @bias }
       end
 
       private def init_weight_and_bias
@@ -159,23 +185,21 @@ module DNN
         end
       end
     end
-    
-    
-    # Full connnection layer.
+
+
     class Dense < Connection
-      # @return [Integer] number of nodes.
       attr_reader :num_nodes
 
       def self.from_hash(hash)
         self.new(hash[:num_nodes],
-                 weight_initializer: Utils.from_hash(hash[:weight_initializer]),
-                 bias_initializer: Utils.from_hash(hash[:bias_initializer]),
-                 weight_regularizer: Utils.from_hash(hash[:weight_regularizer]),
-                 bias_regularizer: Utils.from_hash(hash[:bias_regularizer]),
+                 weight_initializer: Utils.hash_to_obj(hash[:weight_initializer]),
+                 bias_initializer: Utils.hash_to_obj(hash[:bias_initializer]),
+                 weight_regularizer: Utils.hash_to_obj(hash[:weight_regularizer]),
+                 bias_regularizer: Utils.hash_to_obj(hash[:bias_regularizer]),
                  use_bias: hash[:use_bias])
       end
 
-      # @param [Integer] num_nodes number of nodes.
+      # @param [Integer] num_nodes Number of nodes.
       def initialize(num_nodes,
                      weight_initializer: Initializers::RandomNormal.new,
                      bias_initializer: Initializers::Zeros.new,
@@ -204,7 +228,7 @@ module DNN
         y += @bias.data if @bias
         y
       end
-    
+
       def backward(dy)
         if @trainable
           @weight.grad += @x.transpose.dot(dy)
@@ -212,22 +236,22 @@ module DNN
         end
         dy.dot(@weight.data.transpose)
       end
-    
+
       def output_shape
         [@num_nodes]
       end
 
       def to_hash
-        super({num_nodes: @num_nodes})
+        super(num_nodes: @num_nodes)
       end
     end
-    
+
 
     class Flatten < Layer
       def forward(x)
         x.reshape(x.shape[0], *output_shape)
       end
-    
+
       def backward(dy)
         dy.reshape(dy.shape[0], *@input_shape)
       end
@@ -261,50 +285,59 @@ module DNN
       end
 
       def to_hash
-        super({output_shape: @output_shape})
+        super(output_shape: @output_shape)
       end
     end
 
-    
+
     class Dropout < Layer
-      # @return [Float] dropout ratio.
       attr_accessor :dropout_ratio
-      # @return [Float] Use 'weight scaling inference rule'.
       attr_reader :use_scale
 
       def self.from_hash(hash)
         self.new(hash[:dropout_ratio], seed: hash[:seed], use_scale: hash[:use_scale])
       end
 
+      # @param [Float] dropout_ratio Nodes dropout ratio.
+      # @param [Integer] seed Seed of random number used for masking.
+      # @param [Boolean] use_scale Set to true to scale the output according to the dropout ratio.
       def initialize(dropout_ratio = 0.5, seed: rand(1 << 31), use_scale: true)
         super()
         @dropout_ratio = dropout_ratio
         @seed = seed
         @use_scale = use_scale
         @mask = nil
+        @rnd = Random.new(@seed)
       end
 
-      def forward(x)
+      def call(input)
+        x, prev_link, learning_phase = *input
+        build(x.shape[1..-1]) unless built?
+        y = forward(x, learning_phase)
+        link = Link.new(prev_link, self)
+        [y, link, learning_phase]
+      end
+
+      def forward(x, learning_phase)
         if learning_phase
-          Xumo::SFloat.srand(@seed)
+          Xumo::SFloat.srand(@rnd.rand(1 << 31))
           @mask = Xumo::SFloat.ones(*x.shape).rand < @dropout_ratio
           x[@mask] = 0
-        else
-          x *= (1 - @dropout_ratio) if @use_scale
+        elsif @use_scale
+          x *= (1 - @dropout_ratio)
         end
         x
       end
-    
+
       def backward(dy)
         dy[@mask] = 0
         dy
       end
 
       def to_hash
-        super({dropout_ratio: @dropout_ratio, seed: @seed, use_scale: @use_scale})
+        super(dropout_ratio: @dropout_ratio, seed: @seed, use_scale: @use_scale)
       end
     end
-    
+
   end
-  
 end