ruby-dnn 0.10.1 → 0.10.2

data/lib/dnn/core/dataset.rb

@@ -1,34 +1,34 @@
-# This class manages input datas and output datas together.
-class DNN::Dataset
-  # @param [Numo::SFloat] x_datas input datas.
-  # @param [Numo::SFloat] y_datas output datas.
-  # @param [Bool] random Set true to return batches randomly. Setting false returns batches in order of index.
-  def initialize(x_datas, y_datas, random = true)
-    @x_datas = x_datas
-    @y_datas = y_datas
-    @random = random
-    @num_datas = x_datas.shape[0]
-    reset_indexs
-  end
-
-  # Return the next batch.
-  # If the number of remaining data < batch size, if random = true, shuffle the data again and return a batch.
-  # If random = false, all remaining data will be returned regardless of the batch size.
-  def next_batch(batch_size)
-    if @indexes.length < batch_size
-      batch_indexes = @indexes unless @random
-      reset_indexs
-      batch_indexes = @indexes.shift(batch_size) if @random
-    else
-      batch_indexes = @indexes.shift(batch_size)
-    end
-    x_batch = @x_datas[batch_indexes, false]
-    y_batch = @y_datas[batch_indexes, false]
-    [x_batch, y_batch]
-  end
-
-  private def reset_indexs
-    @indexes = @num_datas.times.to_a
-    @indexes.shuffle! if @random
-  end
-end
+# This class manages input datas and output datas together.
+class DNN::Dataset
+  # @param [Numo::SFloat] x_datas input datas.
+  # @param [Numo::SFloat] y_datas output datas.
+  # @param [Bool] random Set true to return batches randomly. Setting false returns batches in order of index.
+  def initialize(x_datas, y_datas, random = true)
+    @x_datas = x_datas
+    @y_datas = y_datas
+    @random = random
+    @num_datas = x_datas.shape[0]
+    reset_indexs
+  end
+
+  # Return the next batch.
+  # If the number of remaining data < batch size, if random = true, shuffle the data again and return a batch.
+  # If random = false, all remaining data will be returned regardless of the batch size.
+  def next_batch(batch_size)
+    if @indexes.length < batch_size
+      batch_indexes = @indexes unless @random
+      reset_indexs
+      batch_indexes = @indexes.shift(batch_size) if @random
+    else
+      batch_indexes = @indexes.shift(batch_size)
+    end
+    x_batch = @x_datas[batch_indexes, false]
+    y_batch = @y_datas[batch_indexes, false]
+    [x_batch, y_batch]
+  end
+
+  private def reset_indexs
+    @indexes = @num_datas.times.to_a
+    @indexes.shuffle! if @random
+  end
+end

data/lib/dnn/core/embedding.rb

@@ -0,0 +1,56 @@
+module DNN
+  module Layers
+
+    class Embedding < HasParamLayer
+      # @return [Integer] Return the input length.
+      attr_reader :input_length
+      # @return [Initializers::Initializer] Return the weight initializer.
+      attr_reader :weight_initializer
+
+      def self.from_hash(hash)
+        self.new(hash[:input_shape], hash[:input_length],
+                 weight_initializer: DNN::Utils.from_hash(hash[:weight_initializer]))
+      end
+      
+      # @param [Integer | Array] input_dim_or_shape Set input data dimension or shape.
+      # @param [Integer] input_length input Set the time series length of input data.
+      def initialize(input_dim_or_shape, input_length, weight_initializer: Initializers::RandomUniform.new)
+        super()
+        @input_shape = input_dim_or_shape.is_a?(Array) ? input_dim_or_shape : [input_dim_or_shape]
+        @input_length = input_length
+        @weight_initializer = weight_initializer
+      end
+
+      def build
+        @built = true
+        @params[:weight] = @weight = Param.new(Xumo::SFloat.new(@input_length), 0)
+        @weight_initializer.init_param(self, @weight)
+        @input_shape
+      end
+
+      def forward(x)
+        @x = x
+        y = Xumo::SFloat.zeros(*x.shape)
+        x.shape[0].times do |i|
+          y[i, false] = @weight.data[x[i, false]]
+        end
+        y
+      end
+
+      def backward(dy)
+        @weight.grad += Xumo::SFloat.zeros(*@weight.data.shape)
+        @x.shape[0].times do |i|
+          @x.shape[1].times do |j|
+            @weight.grad[@x[i, j]] += dy[i, j]
+          end
+        end
+        nil
+      end
+
+      def to_hash
+        super(input_shape: @input_shape, input_length: @input_length, weight_initializer: @weight_initializer.to_hash)
+      end
+    end
+
+  end
+end

data/lib/dnn/core/error.rb

@@ -1,5 +1,5 @@
-module DNN
-  class DNN_Error < StandardError; end
-
-  class DNN_ShapeError < DNN_Error; end
-end
+module DNN
+  class DNN_Error < StandardError; end
+
+  class DNN_ShapeError < DNN_Error; end
+end

data/lib/dnn/core/initializers.rb

@@ -1,126 +1,126 @@
-module DNN
-  module Initializers
-
-    class Initializer
-      def initialize(seed = false)
-        @seed = seed == true ? rand(1 << 31) : seed
-      end
-
-      def init_param(layer, param)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'init_params'")
-      end
-
-      def to_hash(merge_hash = nil)
-        hash = {class: self.class.name, seed: @seed}
-        hash.merge!(merge_hash) if merge_hash
-        hash
-      end
-    end
-
-
-    class Zeros < Initializer
-      def init_param(layer, param)
-        param.data = param.data.fill(0)
-      end
-    end
-
-
-    class Const < Initializer
-      attr_reader :const
-
-      def self.from_hash(hash)
-        self.new(hash[:const])
-      end
-
-      def initialize(const)
-        super()
-        @const = const
-      end
-
-      def init_param(layer, param)
-        param.data = param.data.fill(@const)
-      end
-
-      def to_hash
-        super({const: @const})
-      end
-    end
-    
-    
-    class RandomNormal < Initializer
-      attr_reader :mean
-      attr_reader :std
-      
-      def self.from_hash(hash)
-        self.new(hash[:mean], hash[:std], hash[:seed])
-      end
-
-      def initialize(mean = 0, std = 0.05, seed = true)
-        super(seed)
-        @mean = mean
-        @std = std
-      end
-
-      def init_param(layer, param)
-        Xumo::SFloat.srand(@seed)
-        param.data = param.data.rand_norm(@mean, @std)
-      end
-
-      def to_hash
-        super({mean: @mean, std: @std})
-      end
-    end
-
-
-    class RandomUniform < Initializer
-      attr_reader :min
-      attr_reader :max
-
-      def self.from_hash(hash)
-        self.new(hash[:min], hash[:max], hash[:seed])
-      end
-
-      def initialize(min = -0.05, max = 0.05, seed = true)
-        super(seed)
-        @min = min
-        @max = max
-      end
-
-      def init_param(layer, param)
-        Xumo::SFloat.srand(@seed)
-        param.data = param.data.rand(@min, @max)
-      end
-
-      def to_hash
-        super({min: @min, max: @max})
-      end
-    end
-    
-    
-    class Xavier < Initializer
-      def initialize(seed = true)
-        super
-      end
-
-      def init_param(layer, param)
-        Xumo::SFloat.srand(@seed)
-        num_prev_nodes = layer.input_shape.reduce(:*)
-        param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes)
-      end
-    end
-    
-    
-    class He < Initializer
-      def initialize(seed = true)
-        super
-      end
-
-      def init_param(layer, param)
-        Xumo::SFloat.srand(@seed)
-        num_prev_nodes = layer.input_shape.reduce(:*)
-        param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes) * Math.sqrt(2)
-      end
-    end
-
-  end
-end
+module DNN
+  module Initializers
+
+    class Initializer
+      def initialize(seed = false)
+        @seed = seed == true ? rand(1 << 31) : seed
+      end
+
+      def init_param(layer, param)
+        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'init_params'")
+      end
+
+      def to_hash(merge_hash = nil)
+        hash = {class: self.class.name, seed: @seed}
+        hash.merge!(merge_hash) if merge_hash
+        hash
+      end
+    end
+
+
+    class Zeros < Initializer
+      def init_param(layer, param)
+        param.data = param.data.fill(0)
+      end
+    end
+
+
+    class Const < Initializer
+      attr_reader :const
+
+      def self.from_hash(hash)
+        self.new(hash[:const])
+      end
+
+      def initialize(const)
+        super()
+        @const = const
+      end
+
+      def init_param(layer, param)
+        param.data = param.data.fill(@const)
+      end
+
+      def to_hash
+        super({const: @const})
+      end
+    end
+    
+    
+    class RandomNormal < Initializer
+      attr_reader :mean
+      attr_reader :std
+      
+      def self.from_hash(hash)
+        self.new(hash[:mean], hash[:std], hash[:seed])
+      end
+
+      def initialize(mean = 0, std = 0.05, seed = true)
+        super(seed)
+        @mean = mean
+        @std = std
+      end
+
+      def init_param(layer, param)
+        Xumo::SFloat.srand(@seed)
+        param.data = param.data.rand_norm(@mean, @std)
+      end
+
+      def to_hash
+        super({mean: @mean, std: @std})
+      end
+    end
+
+
+    class RandomUniform < Initializer
+      attr_reader :min
+      attr_reader :max
+
+      def self.from_hash(hash)
+        self.new(hash[:min], hash[:max], hash[:seed])
+      end
+
+      def initialize(min = -0.05, max = 0.05, seed = true)
+        super(seed)
+        @min = min
+        @max = max
+      end
+
+      def init_param(layer, param)
+        Xumo::SFloat.srand(@seed)
+        param.data = param.data.rand(@min, @max)
+      end
+
+      def to_hash
+        super({min: @min, max: @max})
+      end
+    end
+    
+    
+    class Xavier < Initializer
+      def initialize(seed = true)
+        super
+      end
+
+      def init_param(layer, param)
+        Xumo::SFloat.srand(@seed)
+        num_prev_nodes = layer.input_shape.reduce(:*)
+        param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes)
+      end
+    end
+    
+    
+    class He < Initializer
+      def initialize(seed = true)
+        super
+      end
+
+      def init_param(layer, param)
+        Xumo::SFloat.srand(@seed)
+        num_prev_nodes = layer.input_shape.reduce(:*)
+        param.data = param.data.rand_norm / Math.sqrt(num_prev_nodes) * Math.sqrt(2)
+      end
+    end
+
+  end
+end

data/lib/dnn/core/layers.rb

@@ -1,307 +1,307 @@
-module DNN
-  module Layers
-
-    # Super class of all optimizer classes.
-    class Layer
-      # @return [Bool] learning_phase Return the true if learning.
-      attr_accessor :learning_phase
-      # @return [Array] Return the shape of the input data.
-      attr_reader :input_shape
-
-      def initialize
-        @built = false
-      end
-
-      # Build the layer.
-      # @param [Array] input_shape Setting the shape of the input data.
-      def build(input_shape)
-        @input_shape = input_shape
-        @learning_phase = true
-        @built = true
-      end
-      
-      # Does the layer have already been built?
-      # @return [Bool] If layer have already been built then return true.
-      def built?
-        @built
-      end
-
-      # Forward propagation.
-      def forward(x)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
-      end
-
-      # Backward propagation.
-      def backward(dy)
-        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'backward'")
-      end
-
-      # Please reimplement this method as needed.
-      # The default implementation return input_shape.
-      # @return [Array] Return the shape of the output data.
-      def output_shape
-        @input_shape
-      end
-
-      # Layer to a hash.
-      def to_hash(merge_hash = nil)
-        hash = {class: self.class.name}
-        hash.merge!(merge_hash) if merge_hash
-        hash
-      end
-    end
-    
-    
-    # This class is a superclass of all classes with learning parameters.
-    class HasParamLayer < Layer
-      # @return [Bool] trainable Setting false prevents learning of parameters.
-      attr_accessor :trainable
-      # @return [Array] The parameters of the layer.
-      attr_reader :params
-    
-      def initialize
-        super()
-        @params = {}
-        @trainable = true
-      end
-    end
-    
-    
-    class InputLayer < Layer
-      def self.from_hash(hash)
-        self.new(hash[:input_shape])
-      end
-
-      def initialize(input_dim_or_shape)
-        super()
-        @input_shape = input_dim_or_shape.is_a?(Array) ? input_dim_or_shape : [input_dim_or_shape]
-      end
-
-      def build
-        @built = true
-        @input_shape
-      end
-
-      def forward(x)
-        x
-      end
-    
-      def backward(dy)
-        dy
-      end
-
-      def to_hash
-        super({input_shape: @input_shape})
-      end
-    end
-
-
-    # It is a superclass of all connection layers.
-    class Connection < HasParamLayer
-      # @return [DNN::Initializers::Initializer] Weight initializer.
-      attr_reader :weight_initializer
-      # @return [DNN::Initializers::Initializer] Bias initializer.
-      attr_reader :bias_initializer
-      # @return [DNN::Regularizers::Regularizer] Weight regularization.
-      attr_reader :weight_regularizer
-      # @return [DNN::Regularizers::Regularizer] Bias regularization.
-      attr_reader :bias_regularizer
-
-      # @param [DNN::Initializers::Initializer] weight_initializer Weight initializer.
-      # @param [DNN::Initializers::Initializer] bias_initializer Bias initializer.
-      # @param [DNN::Regularizers::Regularizer] weight_regularizer Weight regularization.
-      # @param [DNN::Regularizers::Regularizer] bias_regularizer Bias regularization.
-      # @param [Bool] use_bias whether to use bias.
-      def initialize(weight_initializer: Initializers::RandomNormal.new,
-                     bias_initializer: Initializers::Zeros.new,
-                     weight_regularizer: nil,
-                     bias_regularizer: nil,
-                     use_bias: true)
-        super()
-        @weight_initializer = weight_initializer
-        @bias_initializer = bias_initializer
-        @weight_regularizer = weight_regularizer
-        @bias_regularizer = bias_regularizer
-        @params[:weight] = @weight = Param.new(nil, 0)
-        if use_bias
-          @params[:bias] = @bias = Param.new(nil, 0)
-        else
-          @bias = nil
-        end
-      end
-
-      def regularizers
-        regularizers = []
-        regularizers << @weight_regularizer if @weight_regularizer
-        regularizers << @bias_regularizer if @bias_regularizer
-        regularizers
-      end
-
-      # @return [Bool] Return whether to use bias.
-      def use_bias
-        @bias ? true : false
-      end
-
-      def to_hash(merge_hash)
-        super({weight_initializer: @weight_initializer.to_hash,
-               bias_initializer: @bias_initializer.to_hash,
-               weight_regularizer: @weight_regularizer&.to_hash,
-               bias_regularizer: @bias_regularizer&.to_hash,
-               use_bias: use_bias}.merge(merge_hash))
-      end
-
-      private def init_weight_and_bias
-        @weight_initializer.init_param(self, @weight)
-        @weight_regularizer.param = @weight if @weight_regularizer
-        if @bias
-          @bias_initializer.init_param(self, @bias)
-          @bias_regularizer.param = @bias if @bias_regularizer
-        end
-      end
-    end
-    
-    
-    # Full connnection layer.
-    class Dense < Connection
-      # @return [Integer] number of nodes.
-      attr_reader :num_nodes
-
-      def self.from_hash(hash)
-        self.new(hash[:num_nodes],
-                 weight_initializer: Utils.from_hash(hash[:weight_initializer]),
-                 bias_initializer: Utils.from_hash(hash[:bias_initializer]),
-                 weight_regularizer: Utils.from_hash(hash[:weight_regularizer]),
-                 bias_regularizer: Utils.from_hash(hash[:bias_regularizer]),
-                 use_bias: hash[:use_bias])
-      end
-
-      # @param [Integer] num_nodes number of nodes.
-      def initialize(num_nodes,
-                     weight_initializer: Initializers::RandomNormal.new,
-                     bias_initializer: Initializers::Zeros.new,
-                     weight_regularizer: nil,
-                     bias_regularizer: nil,
-                     use_bias: true)
-        super(weight_initializer: weight_initializer, bias_initializer: bias_initializer,
-              weight_regularizer: weight_regularizer, bias_regularizer: bias_regularizer, use_bias: use_bias)
-        @num_nodes = num_nodes
-      end
-
-      def build(input_shape)
-        super
-        num_prev_nodes = input_shape[0]
-        @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
-        @bias.data = Xumo::SFloat.new(@num_nodes) if @bias
-        init_weight_and_bias
-      end
-
-      def forward(x)
-        @x = x
-        y = x.dot(@weight.data)
-        y += @bias.data if @bias
-        y
-      end
-    
-      def backward(dy)
-        if @trainable
-          @weight.grad += @x.transpose.dot(dy)
-          @bias.grad += dy.sum(0) if @bias
-        end
-        dy.dot(@weight.data.transpose)
-      end
-    
-      def output_shape
-        [@num_nodes]
-      end
-
-      def to_hash
-        super({num_nodes: @num_nodes})
-      end
-    end
-    
-
-    class Flatten < Layer
-      def forward(x)
-        x.reshape(x.shape[0], *output_shape)
-      end
-    
-      def backward(dy)
-        dy.reshape(dy.shape[0], *@input_shape)
-      end
-
-      def output_shape
-        [@input_shape.reduce(:*)]
-      end
-    end
-
-
-    class Reshape < Layer
-      def self.from_hash(hash)
-        self.new(hash[:output_shape])
-      end
-
-      def initialize(output_shape)
-        super()
-        @output_shape = output_shape
-      end
-
-      def forward(x)
-        x.reshape(x.shape[0], *@output_shape)
-      end
-
-      def backward(dy)
-        dy.reshape(dy.shape[0], *@input_shape)
-      end
-
-      def output_shape
-        @output_shape
-      end
-
-      def to_hash
-        super({output_shape: @output_shape})
-      end
-    end
-
-    
-    class Dropout < Layer
-      # @return [Float] dropout ratio.
-      attr_accessor :dropout_ratio
-      # @return [Float] Use 'weight scaling inference rule'.
-      attr_reader :use_scale
-
-      def self.from_hash(hash)
-        self.new(hash[:dropout_ratio], seed: hash[:seed], use_scale: hash[:use_scale])
-      end
-
-      def initialize(dropout_ratio = 0.5, seed: rand(1 << 31), use_scale: true)
-        super()
-        @dropout_ratio = dropout_ratio
-        @seed = seed
-        @use_scale = use_scale
-        @mask = nil
-      end
-
-      def forward(x)
-        if learning_phase
-          Xumo::SFloat.srand(@seed)
-          @mask = Xumo::SFloat.ones(*x.shape).rand < @dropout_ratio
-          x[@mask] = 0
-        else
-          x *= (1 - @dropout_ratio) if @use_scale
-        end
-        x
-      end
-    
-      def backward(dy)
-        dy[@mask] = 0
-        dy
-      end
-
-      def to_hash
-        super({dropout_ratio: @dropout_ratio, seed: @seed, use_scale: @use_scale})
-      end
-    end
-    
-  end
-  
-end
+module DNN
+  module Layers
+
+    # Super class of all optimizer classes.
+    class Layer
+      # @return [Bool] learning_phase Return the true if learning.
+      attr_accessor :learning_phase
+      # @return [Array] Return the shape of the input data.
+      attr_reader :input_shape
+
+      def initialize
+        @built = false
+      end
+
+      # Build the layer.
+      # @param [Array] input_shape Setting the shape of the input data.
+      def build(input_shape)
+        @input_shape = input_shape
+        @learning_phase = true
+        @built = true
+      end
+      
+      # Does the layer have already been built?
+      # @return [Bool] If layer have already been built then return true.
+      def built?
+        @built
+      end
+
+      # Forward propagation.
+      def forward(x)
+        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'forward'")
+      end
+
+      # Backward propagation.
+      def backward(dy)
+        raise NotImplementedError.new("Class '#{self.class.name}' has implement method 'backward'")
+      end
+
+      # Please reimplement this method as needed.
+      # The default implementation return input_shape.
+      # @return [Array] Return the shape of the output data.
+      def output_shape
+        @input_shape
+      end
+
+      # Layer to a hash.
+      def to_hash(merge_hash = nil)
+        hash = {class: self.class.name}
+        hash.merge!(merge_hash) if merge_hash
+        hash
+      end
+    end
+    
+    
+    # This class is a superclass of all classes with learning parameters.
+    class HasParamLayer < Layer
+      # @return [Bool] trainable Setting false prevents learning of parameters.
+      attr_accessor :trainable
+      # @return [Array] The parameters of the layer.
+      attr_reader :params
+    
+      def initialize
+        super()
+        @params = {}
+        @trainable = true
+      end
+    end
+    
+    
+    class InputLayer < Layer
+      def self.from_hash(hash)
+        self.new(hash[:input_shape])
+      end
+
+      def initialize(input_dim_or_shape)
+        super()
+        @input_shape = input_dim_or_shape.is_a?(Array) ? input_dim_or_shape : [input_dim_or_shape]
+      end
+
+      def build
+        @built = true
+        @input_shape
+      end
+
+      def forward(x)
+        x
+      end
+    
+      def backward(dy)
+        dy
+      end
+
+      def to_hash
+        super({input_shape: @input_shape})
+      end
+    end
+
+
+    # It is a superclass of all connection layers.
+    class Connection < HasParamLayer
+      # @return [DNN::Initializers::Initializer] Weight initializer.
+      attr_reader :weight_initializer
+      # @return [DNN::Initializers::Initializer] Bias initializer.
+      attr_reader :bias_initializer
+      # @return [DNN::Regularizers::Regularizer] Weight regularization.
+      attr_reader :weight_regularizer
+      # @return [DNN::Regularizers::Regularizer] Bias regularization.
+      attr_reader :bias_regularizer
+
+      # @param [DNN::Initializers::Initializer] weight_initializer Weight initializer.
+      # @param [DNN::Initializers::Initializer] bias_initializer Bias initializer.
+      # @param [DNN::Regularizers::Regularizer] weight_regularizer Weight regularization.
+      # @param [DNN::Regularizers::Regularizer] bias_regularizer Bias regularization.
+      # @param [Bool] use_bias whether to use bias.
+      def initialize(weight_initializer: Initializers::RandomNormal.new,
+                     bias_initializer: Initializers::Zeros.new,
+                     weight_regularizer: nil,
+                     bias_regularizer: nil,
+                     use_bias: true)
+        super()
+        @weight_initializer = weight_initializer
+        @bias_initializer = bias_initializer
+        @weight_regularizer = weight_regularizer
+        @bias_regularizer = bias_regularizer
+        @params[:weight] = @weight = Param.new(nil, 0)
+        if use_bias
+          @params[:bias] = @bias = Param.new(nil, 0)
+        else
+          @bias = nil
+        end
+      end
+
+      def regularizers
+        regularizers = []
+        regularizers << @weight_regularizer if @weight_regularizer
+        regularizers << @bias_regularizer if @bias_regularizer
+        regularizers
+      end
+
+      # @return [Bool] Return whether to use bias.
+      def use_bias
+        @bias ? true : false
+      end
+
+      def to_hash(merge_hash)
+        super({weight_initializer: @weight_initializer.to_hash,
+               bias_initializer: @bias_initializer.to_hash,
+               weight_regularizer: @weight_regularizer&.to_hash,
+               bias_regularizer: @bias_regularizer&.to_hash,
+               use_bias: use_bias}.merge(merge_hash))
+      end
+
+      private def init_weight_and_bias
+        @weight_initializer.init_param(self, @weight)
+        @weight_regularizer.param = @weight if @weight_regularizer
+        if @bias
+          @bias_initializer.init_param(self, @bias)
+          @bias_regularizer.param = @bias if @bias_regularizer
+        end
+      end
+    end
+    
+    
+    # Full connnection layer.
+    class Dense < Connection
+      # @return [Integer] number of nodes.
+      attr_reader :num_nodes
+
+      def self.from_hash(hash)
+        self.new(hash[:num_nodes],
+                 weight_initializer: Utils.from_hash(hash[:weight_initializer]),
+                 bias_initializer: Utils.from_hash(hash[:bias_initializer]),
+                 weight_regularizer: Utils.from_hash(hash[:weight_regularizer]),
+                 bias_regularizer: Utils.from_hash(hash[:bias_regularizer]),
+                 use_bias: hash[:use_bias])
+      end
+
+      # @param [Integer] num_nodes number of nodes.
+      def initialize(num_nodes,
+                     weight_initializer: Initializers::RandomNormal.new,
+                     bias_initializer: Initializers::Zeros.new,
+                     weight_regularizer: nil,
+                     bias_regularizer: nil,
+                     use_bias: true)
+        super(weight_initializer: weight_initializer, bias_initializer: bias_initializer,
+              weight_regularizer: weight_regularizer, bias_regularizer: bias_regularizer, use_bias: use_bias)
+        @num_nodes = num_nodes
+      end
+
+      def build(input_shape)
+        super
+        num_prev_nodes = input_shape[0]
+        @weight.data = Xumo::SFloat.new(num_prev_nodes, @num_nodes)
+        @bias.data = Xumo::SFloat.new(@num_nodes) if @bias
+        init_weight_and_bias
+      end
+
+      def forward(x)
+        @x = x
+        y = x.dot(@weight.data)
+        y += @bias.data if @bias
+        y
+      end
+    
+      def backward(dy)
+        if @trainable
+          @weight.grad += @x.transpose.dot(dy)
+          @bias.grad += dy.sum(0) if @bias
+        end
+        dy.dot(@weight.data.transpose)
+      end
+    
+      def output_shape
+        [@num_nodes]
+      end
+
+      def to_hash
+        super({num_nodes: @num_nodes})
+      end
+    end
+    
+
+    class Flatten < Layer
+      def forward(x)
+        x.reshape(x.shape[0], *output_shape)
+      end
+    
+      def backward(dy)
+        dy.reshape(dy.shape[0], *@input_shape)
+      end
+
+      def output_shape
+        [@input_shape.reduce(:*)]
+      end
+    end
+
+
+    class Reshape < Layer
+      def self.from_hash(hash)
+        self.new(hash[:output_shape])
+      end
+
+      def initialize(output_shape)
+        super()
+        @output_shape = output_shape
+      end
+
+      def forward(x)
+        x.reshape(x.shape[0], *@output_shape)
+      end
+
+      def backward(dy)
+        dy.reshape(dy.shape[0], *@input_shape)
+      end
+
+      def output_shape
+        @output_shape
+      end
+
+      def to_hash
+        super({output_shape: @output_shape})
+      end
+    end
+
+    
+    class Dropout < Layer
+      # @return [Float] dropout ratio.
+      attr_accessor :dropout_ratio
+      # @return [Float] Use 'weight scaling inference rule'.
+      attr_reader :use_scale
+
+      def self.from_hash(hash)
+        self.new(hash[:dropout_ratio], seed: hash[:seed], use_scale: hash[:use_scale])
+      end
+
+      def initialize(dropout_ratio = 0.5, seed: rand(1 << 31), use_scale: true)
+        super()
+        @dropout_ratio = dropout_ratio
+        @seed = seed
+        @use_scale = use_scale
+        @mask = nil
+      end
+
+      def forward(x)
+        if learning_phase
+          Xumo::SFloat.srand(@seed)
+          @mask = Xumo::SFloat.ones(*x.shape).rand < @dropout_ratio
+          x[@mask] = 0
+        else
+          x *= (1 - @dropout_ratio) if @use_scale
+        end
+        x
+      end
+    
+      def backward(dy)
+        dy[@mask] = 0
+        dy
+      end
+
+      def to_hash
+        super({dropout_ratio: @dropout_ratio, seed: @seed, use_scale: @use_scale})
+      end
+    end
+    
+  end
+  
+end