ruby-dnn 0.10.4 → 0.12.4

data/lib/dnn/core/param.rb

@@ -1,9 +1,11 @@
-class DNN::Param
-  attr_accessor :data
-  attr_accessor :grad
+module DNN
+  class Param
+    attr_accessor :data
+    attr_accessor :grad
 
-  def initialize(data = nil, grad = nil)
-    @data = data
-    @grad = grad
+    def initialize(data = nil, grad = nil)
+      @data = data
+      @grad = grad
+    end
   end
 end

data/lib/dnn/core/regularizers.rb

@@ -13,7 +13,7 @@ module DNN
       end
 
       def to_hash(merge_hash)
-        hash = {class: self.class.name}
+        hash = { class: self.class.name }
         hash.merge!(merge_hash)
         hash
       end
@@ -23,9 +23,10 @@ module DNN
       attr_accessor :l1_lambda
 
       def self.from_hash(hash)
-        L1.new(hash[:l1_lambda])
+        self.new(hash[:l1_lambda])
       end
 
+      # @param [Float] l1_lambda L1 regularizer coefficient.
       def initialize(l1_lambda = 0.01)
         @l1_lambda = l1_lambda
       end
@@ -50,15 +51,16 @@ module DNN
       attr_accessor :l2_lambda
 
       def self.from_hash(hash)
-        L2.new(hash[:l2_lambda])
+        self.new(hash[:l2_lambda])
       end
 
+      # @param [Float] l2_lambda L2 regularizer coefficient.
       def initialize(l2_lambda = 0.01)
         @l2_lambda = l2_lambda
       end
 
       def forward(x)
-        x + 0.5 * @l2_lambda * (@param.data**2).sum
+        x + 0.5 * @l2_lambda * (@param.data ** 2).sum
       end
 
       def backward
@@ -75,9 +77,11 @@ module DNN
       attr_accessor :l2_lambda
 
       def self.from_hash(hash)
-        L1L2.new(hash[:l1_lambda], hash[:l2_lambda])
+        self.new(hash[:l1_lambda], hash[:l2_lambda])
       end
 
+      # @param [Float] l1_lambda L1 regularizer coefficient.
+      # @param [Float] l2_lambda L2 regularizer coefficient.
       def initialize(l1_lambda = 0.01, l2_lambda = 0.01)
         @l1_lambda = l1_lambda
         @l2_lambda = l2_lambda
@@ -85,7 +89,7 @@ module DNN
 
       def forward(x)
         l1 = @l1_lambda * @param.data.abs.sum
-        l2 = 0.5 * @l2_lambda * (@param.data**2).sum
+        l2 = 0.5 * @l2_lambda * (@param.data ** 2).sum
         x + l1 + l2
       end
 
@@ -99,7 +103,6 @@ module DNN
       def to_hash
         super(l1_lambda: l1_lambda, l2_lambda: l2_lambda)
       end
-
     end
 
   end

data/lib/dnn/core/rnn_layers.rb

@@ -3,25 +3,38 @@ module DNN
 
     # Super class of all RNN classes.
     class RNN < Connection
-      include Initializers
-
-      # @return [Integer] number of nodes.
       attr_reader :num_nodes
-      # @return [Bool] Maintain state between batches.
+      attr_reader :recurrent_weight
+      attr_reader :hidden
       attr_reader :stateful
-      # @return [Bool] Set the false, only the last of each cell of RNN is left.
       attr_reader :return_sequences
-      # @return [DNN::Initializers::Initializer] Recurrent weight initializer.
       attr_reader :recurrent_weight_initializer
-      # @return [DNN::Regularizers::Regularizer] Recurrent weight regularization.
       attr_reader :recurrent_weight_regularizer
 
+      def self.from_hash(hash)
+        self.new(hash[:num_nodes],
+                 stateful: hash[:stateful],
+                 return_sequences: hash[:return_sequences],
+                 weight_initializer: Utils.hash_to_obj(hash[:weight_initializer]),
+                 recurrent_weight_initializer: Utils.hash_to_obj(hash[:recurrent_weight_initializer]),
+                 bias_initializer: Utils.hash_to_obj(hash[:bias_initializer]),
+                 weight_regularizer: Utils.hash_to_obj(hash[:weight_regularizer]),
+                 recurrent_weight_regularizer: Utils.hash_to_obj(hash[:recurrent_weight_regularizer]),
+                 bias_regularizer: Utils.hash_to_obj(hash[:bias_regularizer]),
+                 use_bias: hash[:use_bias])
+      end
+
+      # @param [Integer] num_nodes Number of nodes.
+      # @param [Boolean] stateful Maintain state between batches.
+      # @param [Boolean] return_sequences Set the false, only the last of each cell of RNN is left.
+      # @param [DNN::Initializers::Initializer] recurrent_weight_initializer Recurrent weight initializer.
+      # @param [DNN::Regularizers::Regularizer | NilClass] recurrent_weight_regularizer Recurrent weight regularizer.
       def initialize(num_nodes,
                      stateful: false,
                      return_sequences: true,
-                     weight_initializer: RandomNormal.new,
-                     recurrent_weight_initializer: RandomNormal.new,
-                     bias_initializer: Zeros.new,
+                     weight_initializer: Initializers::RandomNormal.new,
+                     recurrent_weight_initializer: Initializers::RandomNormal.new,
+                     bias_initializer: Initializers::Zeros.new,
                      weight_regularizer: nil,
                      recurrent_weight_regularizer: nil,
                      bias_regularizer: nil,
@@ -32,8 +45,8 @@ module DNN
         @stateful = stateful
         @return_sequences = return_sequences
         @layers = []
-        @hidden = @params[:hidden] = Param.new
-        @params[:recurrent_weight] = @recurrent_weight = Param.new(nil, 0)
+        @hidden = Param.new
+        @recurrent_weight = Param.new(nil, 0)
         @recurrent_weight_initializer = recurrent_weight_initializer
         @recurrent_weight_regularizer = recurrent_weight_regularizer
       end
@@ -49,7 +62,7 @@ module DNN
       def forward(xs)
         @xs_shape = xs.shape
         hs = Xumo::SFloat.zeros(xs.shape[0], @time_length, @num_nodes)
-        h = (@stateful && @hidden.data) ? @hidden.data : Xumo::SFloat.zeros(xs.shape[0], @num_nodes)
+        h = @stateful && @hidden.data ? @hidden.data : Xumo::SFloat.zeros(xs.shape[0], @num_nodes)
         xs.shape[1].times do |t|
           x = xs[true, t, false]
           @layers[t].trainable = @trainable
@@ -92,6 +105,10 @@ module DNN
         super(hash)
       end
 
+      def get_params
+        { weight: @weight, recurrent_weight: @recurrent_weight, bias: @bias, hidden: @hidden }
+      end
+
       # Reset the state of RNN.
       def reset_state
         @hidden.data = @hidden.data.fill(0) if @hidden.data
@@ -113,7 +130,7 @@ module DNN
     end
 
 
-    class SimpleRNN_Dense
+    class SimpleRNNDense
       attr_accessor :trainable
 
       def initialize(weight, recurrent_weight, bias, activation)
@@ -147,32 +164,30 @@ module DNN
 
 
     class SimpleRNN < RNN
-      include Activations
-
       attr_reader :activation
-      
+
       def self.from_hash(hash)
-        simple_rnn = self.new(hash[:num_nodes],
-                              stateful: hash[:stateful],
-                              return_sequences: hash[:return_sequences],
-                              activation: Utils.from_hash(hash[:activation]),
-                              weight_initializer: Utils.from_hash(hash[:weight_initializer]),
-                              recurrent_weight_initializer: Utils.from_hash(hash[:recurrent_weight_initializer]),
-                              bias_initializer: Utils.from_hash(hash[:bias_initializer]),
-                              weight_regularizer: Utils.from_hash(hash[:weight_regularizer]),
-                              recurrent_weight_regularizer: Utils.from_hash(hash[:recurrent_weight_regularizer]),
-                              bias_regularizer: Utils.from_hash(hash[:bias_regularizer]),
-                              use_bias: hash[:use_bias])
-        simple_rnn
+        self.new(hash[:num_nodes],
+                 stateful: hash[:stateful],
+                 return_sequences: hash[:return_sequences],
+                 activation: Utils.hash_to_obj(hash[:activation]),
+                 weight_initializer: Utils.hash_to_obj(hash[:weight_initializer]),
+                 recurrent_weight_initializer: Utils.hash_to_obj(hash[:recurrent_weight_initializer]),
+                 bias_initializer: Utils.hash_to_obj(hash[:bias_initializer]),
+                 weight_regularizer: Utils.hash_to_obj(hash[:weight_regularizer]),
+                 recurrent_weight_regularizer: Utils.hash_to_obj(hash[:recurrent_weight_regularizer]),
+                 bias_regularizer: Utils.hash_to_obj(hash[:bias_regularizer]),
+                 use_bias: hash[:use_bias])
       end
 
+      # @param [DNN::Layers::Layer] activation Activation function to use in a recurrent network.
       def initialize(num_nodes,
                      stateful: false,
                      return_sequences: true,
-                     activation: Tanh.new,
-                     weight_initializer: RandomNormal.new,
-                     recurrent_weight_initializer: RandomNormal.new,
-                     bias_initializer: Zeros.new,
+                     activation: Activations::Tanh.new,
+                     weight_initializer: Initializers::RandomNormal.new,
+                     recurrent_weight_initializer: Initializers::RandomNormal.new,
+                     bias_initializer: Initializers::Zeros.new,
                      weight_regularizer: nil,
                      recurrent_weight_regularizer: nil,
                      bias_regularizer: nil,
@@ -197,29 +212,29 @@ module DNN
         @recurrent_weight.data = Xumo::SFloat.new(@num_nodes, @num_nodes)
         @bias.data = Xumo::SFloat.new(@num_nodes) if @bias
         init_weight_and_bias
-        @time_length.times do |t|
-          @layers << SimpleRNN_Dense.new(@weight, @recurrent_weight, @bias, @activation)
+        @time_length.times do
+          @layers << SimpleRNNDense.new(@weight, @recurrent_weight, @bias, @activation)
         end
       end
 
       def to_hash
-        super({activation: @activation.to_hash})
+        super(activation: @activation.to_hash)
       end
     end
 
 
-    class LSTM_Dense
+    class LSTMDense
       attr_accessor :trainable
 
       def initialize(weight, recurrent_weight, bias)
         @weight = weight
         @recurrent_weight = recurrent_weight
         @bias = bias
-        @tanh = Tanh.new
-        @g_tanh = Tanh.new
-        @forget_sigmoid = Sigmoid.new
-        @in_sigmoid = Sigmoid.new
-        @out_sigmoid = Sigmoid.new
+        @tanh = Activations::Tanh.new
+        @g_tanh = Activations::Tanh.new
+        @forget_sigmoid = Activations::Sigmoid.new
+        @in_sigmoid = Activations::Sigmoid.new
+        @out_sigmoid = Activations::Sigmoid.new
         @trainable = true
       end
 
@@ -267,32 +282,20 @@ module DNN
 
 
     class LSTM < RNN
-      def self.from_hash(hash)
-        lstm = self.new(hash[:num_nodes],
-                        stateful: hash[:stateful],
-                        return_sequences: hash[:return_sequences],
-                        weight_initializer: Utils.from_hash(hash[:weight_initializer]),
-                        recurrent_weight_initializer: Utils.from_hash(hash[:recurrent_weight_initializer]),
-                        bias_initializer: Utils.from_hash(hash[:bias_initializer]),
-                        weight_regularizer: Utils.from_hash(hash[:weight_regularizer]),
-                        recurrent_weight_regularizer: Utils.from_hash(hash[:recurrent_weight_regularizer]),
-                        bias_regularizer: Utils.from_hash(hash[:bias_regularizer]),
-                        use_bias: hash[:use_bias])
-        lstm
-      end
+      attr_reader :cell
 
       def initialize(num_nodes,
                      stateful: false,
                      return_sequences: true,
-                     weight_initializer: RandomNormal.new,
-                     recurrent_weight_initializer: RandomNormal.new,
-                     bias_initializer: Zeros.new,
+                     weight_initializer: Initializers::RandomNormal.new,
+                     recurrent_weight_initializer: Initializers::RandomNormal.new,
+                     bias_initializer: Initializers::Zeros.new,
                      weight_regularizer: nil,
                      recurrent_weight_regularizer: nil,
                      bias_regularizer: nil,
                      use_bias: true)
         super
-        @cell = @params[:cell] = Param.new
+        @cell = Param.new
       end
 
       def build(input_shape)
@@ -302,8 +305,8 @@ module DNN
         @recurrent_weight.data = Xumo::SFloat.new(@num_nodes, @num_nodes * 4)
         @bias.data = Xumo::SFloat.new(@num_nodes * 4) if @bias
         init_weight_and_bias
-        @time_length.times do |t|
-          @layers << LSTM_Dense.new(@weight, @recurrent_weight, @bias)
+        @time_length.times do
+          @layers << LSTMDense.new(@weight, @recurrent_weight, @bias)
         end
       end
 
@@ -350,19 +353,23 @@ module DNN
         super()
         @cell.data = @cell.data.fill(0) if @cell.data
       end
+
+      def get_params
+        { weight: @weight, recurrent_weight: @recurrent_weight, bias: @bias, hidden: @hidden, cell: @cell }
+      end
     end
 
 
-    class GRU_Dense
+    class GRUDense
       attr_accessor :trainable
 
       def initialize(weight, recurrent_weight, bias)
         @weight = weight
         @recurrent_weight = recurrent_weight
         @bias = bias
-        @update_sigmoid = Sigmoid.new
-        @reset_sigmoid = Sigmoid.new
-        @tanh = Tanh.new
+        @update_sigmoid = Activations::Sigmoid.new
+        @reset_sigmoid = Activations::Sigmoid.new
+        @tanh = Activations::Tanh.new
         @trainable = true
       end
 
@@ -423,33 +430,19 @@ module DNN
 
 
     class GRU < RNN
-      def self.from_hash(hash)
-        gru = self.new(hash[:num_nodes],
-                       stateful: hash[:stateful],
-                       return_sequences: hash[:return_sequences],
-                       weight_initializer: Utils.from_hash(hash[:weight_initializer]),
-                       recurrent_weight_initializer: Utils.from_hash(hash[:recurrent_weight_initializer]),
-                       bias_initializer: Utils.from_hash(hash[:bias_initializer]),
-                       weight_regularizer: Utils.from_hash(hash[:weight_regularizer]),
-                       recurrent_weight_regularizer: Utils.from_hash(hash[:recurrent_weight_regularizer]),
-                       bias_regularizer: Utils.from_hash(hash[:bias_regularizer]),
-                       use_bias: hash[:use_bias])
-        gru
-      end
-
       def initialize(num_nodes,
                      stateful: false,
                      return_sequences: true,
-                     weight_initializer: RandomNormal.new,
-                     recurrent_weight_initializer: RandomNormal.new,
-                     bias_initializer: Zeros.new,
+                     weight_initializer: Initializers::RandomNormal.new,
+                     recurrent_weight_initializer: Initializers::RandomNormal.new,
+                     bias_initializer: Initializers::Zeros.new,
                      weight_regularizer: nil,
                      recurrent_weight_regularizer: nil,
                      bias_regularizer: nil,
                      use_bias: true)
         super
       end
-    
+
       def build(input_shape)
         super
         num_prev_nodes = @input_shape[1]
@@ -457,8 +450,8 @@ module DNN
         @recurrent_weight.data = Xumo::SFloat.new(@num_nodes, @num_nodes * 3)
         @bias.data = Xumo::SFloat.new(@num_nodes * 3) if @bias
         init_weight_and_bias
-        @time_length.times do |t|
-          @layers << GRU_Dense.new(@weight, @recurrent_weight, @bias)
+        @time_length.times do
+          @layers << GRUDense.new(@weight, @recurrent_weight, @bias)
         end
       end
     end

data/lib/dnn/core/utils.rb

@@ -2,6 +2,9 @@ module DNN
   # This module provides utility functions.
   module Utils
     # Categorize labels into "num_classes" classes.
+    # @param [Numo::SFloat] y Label data.
+    # @param [Numo::SFloat] num_classes Number of classes to classify.
+    # @param [Class] narray_type Type of Numo::Narray data after classification.
     def self.to_categorical(y, num_classes, narray_type = nil)
       narray_type ||= y.class
       y2 = narray_type.zeros(y.shape[0], num_classes)
@@ -12,7 +15,7 @@ module DNN
     end
 
     # Convert hash to an object.
-    def self.from_hash(hash)
+    def self.hash_to_obj(hash)
       return nil if hash == nil
       dnn_class = DNN.const_get(hash[:class])
       if dnn_class.respond_to?(:from_hash)
@@ -23,12 +26,12 @@ module DNN
 
     # Return the result of the sigmoid function.
     def self.sigmoid(x)
-      Sigmoid.new.forward(x)
+      Activations::Sigmoid.new.forward(x)
     end
 
     # Return the result of the softmax function.
     def self.softmax(x)
-      SoftmaxCrossEntropy.softmax(x)
+      Losses::SoftmaxCrossEntropy.softmax(x)
     end
   end
 end

data/lib/dnn/downloader.rb

@@ -11,8 +11,8 @@ module DNN
         dir_path = "#{__dir__}/downloads"
       end
       Downloader.new(url).download(dir_path)
-    rescue => ex
-      raise DNN_DownloadError.new(ex.message)
+    rescue => e
+      raise DNN_DownloadError.new(e.message)
     end
 
     def initialize(url)
@@ -42,7 +42,7 @@ module DNN
         end
         puts ""
       end
-      file_name = @path.match(%r`.+/(.+)`)[1]
+      file_name = @path.match(%r`.*/(.+)`)[1]
       File.binwrite("#{dir_path}/#{file_name}", buf)
     end
   end

data/lib/dnn/fashion-mnist.rb

@@ -0,0 +1,89 @@
+require "zlib"
+require_relative "core/error"
+require_relative "downloader"
+require_relative "mnist"
+
+module DNN
+  module FashionMNIST
+    class DNN_MNIST_LoadError < DNN_Error; end
+
+    URL_BASE = "http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/"
+
+    TRAIN_IMAGES_FILE_NAME = "train-images-idx3-ubyte.gz"
+    TRAIN_LABELS_FILE_NAME = "train-labels-idx1-ubyte.gz"
+    TEST_IMAGES_FILE_NAME = "t10k-images-idx3-ubyte.gz"
+    TEST_LABELS_FILE_NAME = "t10k-labels-idx1-ubyte.gz"
+
+    URL_TRAIN_IMAGES = URL_BASE + TRAIN_IMAGES_FILE_NAME
+    URL_TRAIN_LABELS = URL_BASE + TRAIN_LABELS_FILE_NAME
+    URL_TEST_IMAGES = URL_BASE + TEST_IMAGES_FILE_NAME
+    URL_TEST_LABELS = URL_BASE + TEST_LABELS_FILE_NAME
+
+    def self.downloads
+      Dir.mkdir("#{__dir__}/downloads") unless Dir.exist?("#{__dir__}/downloads")
+      Dir.mkdir(mnist_dir) unless Dir.exist?(mnist_dir)
+      Downloader.download(URL_TRAIN_IMAGES, mnist_dir) unless File.exist?(get_file_path(TRAIN_IMAGES_FILE_NAME))
+      Downloader.download(URL_TRAIN_LABELS, mnist_dir) unless File.exist?(get_file_path(TRAIN_LABELS_FILE_NAME))
+      Downloader.download(URL_TEST_IMAGES, mnist_dir) unless File.exist?(get_file_path(TEST_IMAGES_FILE_NAME))
+      Downloader.download(URL_TEST_LABELS, mnist_dir) unless File.exist?(get_file_path(TEST_LABELS_FILE_NAME))
+    end
+
+    def self.load_train
+      downloads
+      train_images_file_path = get_file_path(TRAIN_IMAGES_FILE_NAME)
+      train_labels_file_path = get_file_path(TRAIN_LABELS_FILE_NAME)
+      unless File.exist?(train_images_file_path)
+        raise DNN_MNIST_LoadError.new(%`file "#{train_images_file_path}" is not found.`)
+      end
+      unless File.exist?(train_labels_file_path)
+        raise DNN_MNIST_LoadError.new(%`file "#{train_labels_file_path}" is not found.`)
+      end
+      images = load_images(train_images_file_path)
+      labels = load_labels(train_labels_file_path)
+      [images, labels]
+    end
+
+    def self.load_test
+      downloads
+      test_images_file_path = get_file_path(TEST_IMAGES_FILE_NAME)
+      test_labels_file_path = get_file_path(TEST_LABELS_FILE_NAME)
+      unless File.exist?(test_images_file_path)
+        raise DNN_MNIST_LoadError.new(%`file "#{test_images_file_path}" is not found.`)
+      end
+      unless File.exist?(test_labels_file_path)
+        raise DNN_MNIST_LoadError.new(%`file "#{test_labels_file_path}" is not found.`)
+      end
+      images = load_images(test_images_file_path)
+      labels = load_labels(test_labels_file_path)
+      [images, labels]
+    end
+
+    private_class_method def self.load_images(file_name)
+      images = nil
+      Zlib::GzipReader.open(file_name) do |f|
+        magic, num_images = f.read(8).unpack("N2")
+        rows, cols = f.read(8).unpack("N2")
+        images = Numo::UInt8.from_binary(f.read)
+        images = images.reshape(num_images, cols, rows, 1)
+      end
+      images
+    end
+
+    private_class_method def self.load_labels(file_name)
+      labels = nil
+      Zlib::GzipReader.open(file_name) do |f|
+        magic, num_labels = f.read(8).unpack("N2")
+        labels = Numo::UInt8.from_binary(f.read)
+      end
+      labels
+    end
+
+    private_class_method def self.mnist_dir
+      "#{__dir__}/downloads/fashion-mnist"
+    end
+
+    private_class_method def self.get_file_path(file_name)
+      mnist_dir + "/" + file_name
+    end
+  end
+end