rumale 0.23.3 → 0.24.0

data/lib/rumale/nearest_neighbors/vp_tree.rb

@@ -1,132 +0,0 @@
-# frozen_string_literal: true
-
-require 'rumale/validation'
-require 'rumale/pairwise_metric'
-require 'rumale/base/base_estimator'
-
-module Rumale
-  module NearestNeighbors
-    # VPTree is a class that implements the nearest neigbor searcher based on vantage point tree.
-    # This implementation, unlike the paper, does not perform random sampling with vantage point selection.
-    # This class is used internally for k-nearest neighbor estimators.
-    #
-    # @deprecated This class will be removed in ver. 0.24.0. The author recommends to use the annoy-rb gem instead.
-    #
-    # *Reference*
-    # - Yianilos, P N., "Data Structures and Algorithms for Nearest Neighbor Search in General Metric Spaces," Proc. SODA'93, pp. 311--321, 1993.
-    class VPTree
-      include Validation
-      include Base::BaseEstimator
-
-      # Return the training data.
-      # @return [Numo::DFloat] (shape: [n_samples, n_features])
-      attr_reader :data
-
-      # Create a search index with vantage point tree algorithm.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The data to used generating search index.
-      # @param min_samples_leaf [Integer] The minimum number of samples at a leaf node.
-      def initialize(x, min_samples_leaf: 1)
-        check_params_numeric(min_samples_leaf: min_samples_leaf)
-        check_params_positive(min_samples_leaf: min_samples_leaf)
-        @params = {}
-        @params[:min_samples_leaf] = min_samples_leaf
-        @data = x
-        @tree = build_tree(Numo::Int32.cast(Array(0...@data.shape[0])))
-      end
-
-      # Search k-nearest neighbors of given query point.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features])
-      # @param k [Integer] The samples to be query points.
-      # @return [Array<Array<Numo::Int32, Numo::DFloat>>] The indices and distances of retrieved k-nearest neighbors.
-      def query(x, k = 1)
-        x = check_convert_sample_array(x)
-        check_params_numeric(k: k)
-        check_params_positive(k: k)
-
-        n_samples = x.shape[0]
-        rel_ids = []
-        rel_dists = []
-
-        n_samples.times do |n|
-          q = x[n, true]
-          rel_node = search(q, @tree, k)
-          dist_arr = calc_distances(q, @data[rel_node.sample_ids, true])
-          rank_ids = dist_arr.sort_index[0...k]
-          rel_ids.push(rel_node.sample_ids[rank_ids].dup)
-          rel_dists.push(dist_arr[rank_ids].dup)
-        end
-
-        [Numo::Int32.cast(rel_ids), Numo::DFloat.cast(rel_dists)]
-      end
-
-      private
-
-      Node = Struct.new(:sample_ids, :n_samples, :vantage_point_id, :threshold, :left, :right) do
-        def leaf?
-          vantage_point_id.nil?
-        end
-      end
-
-      private_constant :Node
-
-      def search(q, node, k, tau = Float::INFINITY)
-        return node if node.leaf?
-
-        dist = Math.sqrt(((q - @data[node.vantage_point_id, true])**2).sum)
-        tau = dist if dist < tau
-
-        # :nocov:
-        if dist < node.threshold
-          if dist - tau <= node.threshold
-            node.left.n_samples < k ? node : search(q, node.left, k, tau)
-          elsif dist + tau >= node.threshold
-            node.right.n_samples < k ? node : search(q, node.right, k, tau)
-          else
-            node
-          end
-        elsif dist + tau >= node.threshold
-          node.right.n_samples < k ? node : search(q, node.right, k, tau)
-        elsif dist - tau <= node.threshold
-          node.left.n_samples < k ? node : search(q, node.left, k, tau)
-        else
-          node
-        end
-        # :nocov:
-      end
-
-      def build_tree(sample_ids)
-        n_samples = sample_ids.size
-        node = Node.new
-        node.n_samples = n_samples
-        node.sample_ids = sample_ids
-        return node if n_samples <= @params[:min_samples_leaf]
-
-        vantage_point_id = select_vantage_point_id(sample_ids)
-        distance_arr = calc_distances(@data[vantage_point_id, true], @data[sample_ids, true])
-        threshold = distance_arr.median
-        left_flgs = distance_arr.lt(threshold)
-        right_flgs = distance_arr.ge(threshold)
-        return node if left_flgs.count < @params[:min_samples_leaf] || right_flgs.count < @params[:min_samples_leaf]
-
-        node.left = build_tree(sample_ids[left_flgs])
-        node.right = build_tree(sample_ids[right_flgs])
-        node.vantage_point_id = vantage_point_id
-        node.threshold = threshold
-        node
-      end
-
-      def select_vantage_point_id(sample_ids)
-        dist_mat = Rumale::PairwiseMetric.euclidean_distance(@data[sample_ids, true])
-        means = dist_mat.mean(0)
-        vars = ((dist_mat - means)**2).mean(0)
-        sample_ids[vars.max_index]
-      end
-
-      def calc_distances(q, x)
-        Rumale::PairwiseMetric.euclidean_distance(q.expand_dims(0), x).flatten.dup
-      end
-    end
-  end
-end

data/lib/rumale/neural_network/adam.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require 'rumale/base/base_estimator'
-
-module Rumale
-  module NeuralNetwork
-    # @!visibility private
-    # This module consists of the classes that implement optimizers adaptively tuning learning rate.
-    module Optimizer
-      # @!visibility private
-      # Adam is a class that implements Adam optimizer.
-      #
-      # *Reference*
-      # - Kingma, D P., and Ba, J., "Adam: A Method for Stochastic Optimization," Proc. ICLR'15, 2015.
-      class Adam
-        include Base::BaseEstimator
-
-        # @!visibility private
-        # Create a new optimizer with Adam
-        #
-        # @param learning_rate [Float] The initial value of learning rate.
-        # @param decay1 [Float] The smoothing parameter for the first moment.
-        # @param decay2 [Float] The smoothing parameter for the second moment.
-        def initialize(learning_rate: 0.001, decay1: 0.9, decay2: 0.999)
-          @params = {}
-          @params[:learning_rate] = learning_rate
-          @params[:decay1] = decay1
-          @params[:decay2] = decay2
-          @fst_moment = nil
-          @sec_moment = nil
-          @iter = 0
-        end
-
-        # @!visibility private
-        # Calculate the updated weight with Adam adaptive learning rate.
-        #
-        # @param weight [Numo::DFloat] (shape: [n_features]) The weight to be updated.
-        # @param gradient [Numo::DFloat] (shape: [n_features]) The gradient for updating the weight.
-        # @return [Numo::DFloat] (shape: [n_feautres]) The updated weight.
-        def call(weight, gradient)
-          @fst_moment ||= Numo::DFloat.zeros(weight.shape)
-          @sec_moment ||= Numo::DFloat.zeros(weight.shape)
-
-          @iter += 1
-
-          @fst_moment = @params[:decay1] * @fst_moment + (1.0 - @params[:decay1]) * gradient
-          @sec_moment = @params[:decay2] * @sec_moment + (1.0 - @params[:decay2]) * gradient**2
-          nm_fst_moment = @fst_moment / (1.0 - @params[:decay1]**@iter)
-          nm_sec_moment = @sec_moment / (1.0 - @params[:decay2]**@iter)
-
-          weight - @params[:learning_rate] * nm_fst_moment / (nm_sec_moment**0.5 + 1e-8)
-        end
-      end
-    end
-  end
-end

data/lib/rumale/neural_network/base_mlp.rb

@@ -1,248 +0,0 @@
-# frozen_string_literal: true
-
-require 'rumale/base/base_estimator'
-
-module Rumale
-  # This module consists of the modules and classes for implementation multi-layer perceptron estimator.
-  module NeuralNetwork
-    # @!visibility private
-    # This module consists of the classes that implement layer functions of neural network.
-    module Layer
-      # @!visibility private
-      # Affine is a class that calculates the linear transform.
-      # This class is used internally.
-      class Affine
-        # @!visibility private
-        def initialize(n_inputs: nil, n_outputs: nil, optimizer: nil, rng: nil)
-          @weight = 0.01 * Rumale::Utils.rand_normal([n_inputs, n_outputs], rng)
-          @bias = Numo::DFloat.zeros(n_outputs)
-          @optimizer_weight = optimizer.dup
-          @optimizer_bias = optimizer.dup
-        end
-
-        # @!visibility private
-        def forward(x)
-          out = x.dot(@weight) + @bias
-
-          backward = proc do |dout|
-            dx = dout.dot(@weight.transpose)
-            dw = x.transpose.dot(dout)
-            db = dout.sum(0)
-
-            @weight = @optimizer_weight.call(@weight, dw)
-            @bias = @optimizer_bias.call(@bias, db)
-
-            dx
-          end
-
-          [out, backward]
-        end
-      end
-
-      # @!visibility private
-      # Dropout is a class that performs dropout regularization.
-      # This class is used internally.
-      class Dropout
-        # @!visibility private
-        def initialize(rate: 0.3, rng: nil)
-          @rate = rate
-          @rng = rng
-        end
-
-        # @!visibility private
-        def forward(x)
-          rand_mat = Rumale::Utils.rand_uniform(x.shape, @rng)
-          mask = rand_mat.ge(@rate)
-          out = x * mask
-          out *= 1.fdiv(1 - @rate) if @rate < 1.0
-
-          backward = proc { |dout| dout * mask }
-
-          [out, backward]
-        end
-      end
-
-      # @!visibility private
-      # ReLU is a class that calculates rectified linear function.
-      # This class is used internally.
-      class Relu
-        # @!visibility private
-        def forward(x)
-          mask = x.gt(0)
-          out = x * mask
-
-          backward = proc { |dout| dout * mask }
-
-          [out, backward]
-        end
-      end
-    end
-
-    # @!visibility private
-    # This module consists of the classes that implement loss function for neural network.
-    module Loss
-      # @!visibility private
-      # MeanSquaredError is a class that calculates mean squared error for regression task.
-      # This class is used internally.
-      class MeanSquaredError
-        # @!visibility private
-        def call(out, y)
-          sz_batch = y.shape[0]
-          diff = out - y
-          loss = (diff**2).sum.fdiv(sz_batch)
-          dout = 2.fdiv(sz_batch) * diff
-          [loss, dout]
-        end
-      end
-
-      # @!visibility private
-      # SoftmaxCrossEntropy is a class that calculates softmax cross-entropy for classification task.
-      # This class is used internally.
-      class SoftmaxCrossEntropy
-        # @!visibility private
-        def call(out, y)
-          sz_batch = y.shape[0]
-          z = softmax(out)
-          loss = -(y * Numo::NMath.log(z + 1e-8)).sum.fdiv(sz_batch)
-          dout = (z - y) / sz_batch
-          [loss, dout]
-        end
-
-        private
-
-        def softmax(x)
-          clip = x.max(-1).expand_dims(-1)
-          exp_x = Numo::NMath.exp(x - clip)
-          exp_x / exp_x.sum(-1).expand_dims(-1)
-        end
-      end
-    end
-
-    # @!visibility private
-    # This module consists of the classes for implementing neural network model.
-    module Model
-      # @!visibility private
-      attr_reader :layers
-
-      # @!visibility private
-      # Sequential is a class that implements linear stack model.
-      # This class is used internally.
-      class Sequential
-        # @!visibility private
-        def initialize
-          @layers = []
-        end
-
-        # @!visibility private
-        def push(ops)
-          @layers.push(ops)
-          self
-        end
-
-        # @!visibility private
-        def delete_dropout
-          @layers.delete_if { |node| node.is_a?(Layer::Dropout) }
-          self
-        end
-
-        # @!visibility private
-        def forward(x)
-          backprops = []
-          out = x.dup
-
-          @layers.each do |l|
-            out, bw = l.forward(out)
-            backprops.push(bw)
-          end
-
-          backward = proc do |dout|
-            backprops.reverse_each { |bw| dout = bw.call(dout) }
-            dout
-          end
-
-          [out, backward]
-        end
-      end
-    end
-
-    # BaseMLP is an abstract class for implementation of multi-layer peceptron estimator.
-    # This class is used internally.
-    class BaseMLP
-      include Base::BaseEstimator
-
-      # Create a multi-layer perceptron estimator.
-      #
-      # @param hidden_units [Array] The number of units in the i-th hidden layer.
-      # @param dropout_rate [Float] The rate of the units to drop.
-      # @param learning_rate [Float] The initial value of learning rate in Adam optimizer.
-      # @param decay1 [Float] The smoothing parameter for the first moment in Adam optimizer.
-      # @param decay2 [Float] The smoothing parameter for the second moment in Adam optimizer.
-      # @param max_iter [Integer] The maximum number of epochs that indicates
-      #   how many times the whole data is given to the training process.
-      # @param batch_size [Intger] The size of the mini batches.
-      # @param tol [Float] The tolerance of loss for terminating optimization.
-      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
-      # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(hidden_units: [128, 128], dropout_rate: 0.4, learning_rate: 0.001, decay1: 0.9, decay2: 0.999,
-                     max_iter: 200, batch_size: 50, tol: 1e-4, verbose: false, random_seed: nil)
-        @params = {}
-        @params[:hidden_units] = hidden_units
-        @params[:dropout_rate] = dropout_rate
-        @params[:learning_rate] = learning_rate
-        @params[:decay1] = decay1
-        @params[:decay2] = decay2
-        @params[:max_iter] = max_iter
-        @params[:batch_size] = batch_size
-        @params[:tol] = tol
-        @params[:verbose] = verbose
-        @params[:random_seed] = random_seed
-        @params[:random_seed] ||= srand
-        @n_iter = nil
-        @rng = Random.new(@params[:random_seed])
-      end
-
-      private
-
-      def buld_network(n_inputs, n_outputs, srng = nil)
-        adam = Rumale::NeuralNetwork::Optimizer::Adam.new(
-          learning_rate: @params[:learning_rate], decay1: @params[:decay1], decay2: @params[:decay2]
-        )
-        model = Model::Sequential.new
-        n_units = [n_inputs, *@params[:hidden_units]]
-        n_units.each_cons(2) do |n_in, n_out|
-          model.push(Layer::Affine.new(n_inputs: n_in, n_outputs: n_out, optimizer: adam, rng: srng))
-          model.push(Layer::Relu.new)
-          model.push(Layer::Dropout.new(rate: @params[:dropout_rate], rng: srng))
-        end
-        model.push(Layer::Affine.new(n_inputs: n_units[-1], n_outputs: n_outputs, optimizer: adam, rng: srng))
-      end
-
-      def train(x, y, network, loss_func, srng = nil)
-        class_name = self.class.to_s.split('::').last
-        n_samples = x.shape[0]
-
-        @params[:max_iter].times do |t|
-          sample_ids = Array(0...n_samples)
-          sample_ids.shuffle!(random: srng)
-          until (subset_ids = sample_ids.shift(@params[:batch_size])).empty?
-            # random sampling
-            sub_x = x[subset_ids, true].dup
-            sub_y = y[subset_ids, true].dup
-            # forward
-            out, backward = network.forward(sub_x)
-            # calc loss function
-            loss, dout = loss_func.call(out, sub_y)
-            break if loss < @params[:tol]
-
-            # backward
-            backward.call(dout)
-          end
-          @n_iter = t + 1
-          puts "[#{class_name}] Loss after #{@n_iter} epochs: #{loss}" if @params[:verbose]
-        end
-
-        network
-      end
-    end
-  end
-end

data/lib/rumale/neural_network/mlp_classifier.rb

@@ -1,120 +0,0 @@
-# frozen_string_literal: true
-
-require 'rumale/base/classifier'
-require 'rumale/neural_network/base_mlp'
-require 'rumale/preprocessing/label_binarizer'
-
-module Rumale
-  module NeuralNetwork
-    # MLPClassifier is a class that implements classifier based on multi-layer perceptron.
-    # MLPClassifier use ReLu as the activation function and Adam as the optimization method
-    # and softmax cross entropy as the loss function.
-    #
-    # @example
-    #   estimator = Rumale::NeuralNetwork::MLPClassifier.new(hidden_units: [100, 100], dropout_rate: 0.3)
-    #   estimator.fit(training_samples, traininig_labels)
-    #   results = estimator.predict(testing_samples)
-    class MLPClassifier < BaseMLP
-      include Base::Classifier
-
-      # Return the network.
-      # @return [Rumale::NeuralNetwork::Model::Sequential]
-      attr_reader :network
-
-      # Return the class labels.
-      # @return [Numo::Int32] (size: n_classes)
-      attr_reader :classes
-
-      # Return the number of iterations run for optimization
-      # @return [Integer]
-      attr_reader :n_iter
-
-      # Return the random generator.
-      # @return [Random]
-      attr_reader :rng
-
-      # Create a new classifier with multi-layer preceptron.
-      #
-      # @param hidden_units [Array] The number of units in the i-th hidden layer.
-      # @param dropout_rate [Float] The rate of the units to drop.
-      # @param learning_rate [Float] The initial value of learning rate in Adam optimizer.
-      # @param decay1 [Float] The smoothing parameter for the first moment in Adam optimizer.
-      # @param decay2 [Float] The smoothing parameter for the second moment in Adam optimizer.
-      # @param max_iter [Integer] The maximum number of epochs that indicates
-      #   how many times the whole data is given to the training process.
-      # @param batch_size [Intger] The size of the mini batches.
-      # @param tol [Float] The tolerance of loss for terminating optimization.
-      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
-      # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(hidden_units: [128, 128], dropout_rate: 0.4, learning_rate: 0.001, decay1: 0.9, decay2: 0.999,
-                     max_iter: 200, batch_size: 50, tol: 1e-4, verbose: false, random_seed: nil)
-        check_params_type(Array, hidden_units: hidden_units)
-        check_params_numeric(dropout_rate: dropout_rate, learning_rate: learning_rate, decay1: decay1, decay2: decay2,
-                             max_iter: max_iter, batch_size: batch_size, tol: tol)
-        check_params_boolean(verbose: verbose)
-        check_params_numeric_or_nil(random_seed: random_seed)
-        super
-        @classes = nil
-        @network = nil
-      end
-
-      # Fit the model with given training data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
-      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
-      # @return [MLPClassifier] The learned classifier itself.
-      def fit(x, y)
-        x = check_convert_sample_array(x)
-        y = check_convert_label_array(y)
-        check_sample_label_size(x, y)
-
-        @classes = Numo::Int32[*y.to_a.uniq.sort]
-        n_labels = @classes.size
-        n_features = x.shape[1]
-        sub_rng = @rng.dup
-
-        loss = Loss::SoftmaxCrossEntropy.new
-        @network = buld_network(n_features, n_labels, sub_rng)
-        @network = train(x, one_hot_encode(y), @network, loss, sub_rng)
-        @network.delete_dropout
-
-        self
-      end
-
-      # Predict class labels for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
-      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
-      def predict(x)
-        x = check_convert_sample_array(x)
-        n_samples = x.shape[0]
-        decision_values = predict_proba(x)
-        predicted = Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] }
-        Numo::Int32.asarray(predicted)
-      end
-
-      # Predict probability for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
-      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
-      def predict_proba(x)
-        x = check_convert_sample_array(x)
-        out, = @network.forward(x)
-        softmax(out)
-      end
-
-      private
-
-      def one_hot_encode(y)
-        encoder = Rumale::Preprocessing::LabelBinarizer.new
-        encoder.fit_transform(y)
-      end
-
-      def softmax(x)
-        clip = x.max(-1).expand_dims(-1)
-        exp_x = Numo::NMath.exp(x - clip)
-        exp_x / exp_x.sum(-1).expand_dims(-1)
-      end
-    end
-  end
-end

data/lib/rumale/neural_network/mlp_regressor.rb

@@ -1,90 +0,0 @@
-# frozen_string_literal: true
-
-require 'rumale/base/regressor'
-require 'rumale/neural_network/base_mlp'
-
-module Rumale
-  module NeuralNetwork
-    # MLPRegressor is a class that implements regressor based on multi-layer perceptron.
-    # MLPRegressor use ReLu as the activation function and Adam as the optimization method
-    # and mean squared error as the loss function.
-    #
-    # @example
-    #   estimator = Rumale::NeuralNetwork::MLPRegressor.new(hidden_units: [100, 100], dropout_rate: 0.3)
-    #   estimator.fit(training_samples, traininig_labels)
-    #   results = estimator.predict(testing_samples)
-    class MLPRegressor < BaseMLP
-      include Base::Regressor
-
-      # Return the network.
-      # @return [Rumale::NeuralNetwork::Model::Sequential]
-      attr_reader :network
-
-      # Return the number of iterations run for optimization
-      # @return [Integer]
-      attr_reader :n_iter
-
-      # Return the random generator.
-      # @return [Random]
-      attr_reader :rng
-
-      # Create a new regressor with multi-layer perceptron.
-      #
-      # @param hidden_units [Array] The number of units in the i-th hidden layer.
-      # @param dropout_rate [Float] The rate of the units to drop.
-      # @param learning_rate [Float] The initial value of learning rate in Adam optimizer.
-      # @param decay1 [Float] The smoothing parameter for the first moment in Adam optimizer.
-      # @param decay2 [Float] The smoothing parameter for the second moment in Adam optimizer.
-      # @param max_iter [Integer] The maximum number of epochs that indicates
-      #   how many times the whole data is given to the training process.
-      # @param batch_size [Intger] The size of the mini batches.
-      # @param tol [Float] The tolerance of loss for terminating optimization.
-      # @param verbose [Boolean] The flag indicating whether to output loss during iteration.
-      # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(hidden_units: [128, 128], dropout_rate: 0.4, learning_rate: 0.001, decay1: 0.9, decay2: 0.999,
-                     max_iter: 200, batch_size: 50, tol: 1e-4, verbose: false, random_seed: nil)
-        check_params_type(Array, hidden_units: hidden_units)
-        check_params_numeric(dropout_rate: dropout_rate, learning_rate: learning_rate, decay1: decay1, decay2: decay2,
-                             max_iter: max_iter, batch_size: batch_size, tol: tol)
-        check_params_boolean(verbose: verbose)
-        check_params_numeric_or_nil(random_seed: random_seed)
-        super
-        @network = nil
-      end
-
-      # Fit the model with given training data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
-      # @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The taget values to be used for fitting the model.
-      # @return [MLPRegressor] The learned regressor itself.
-      def fit(x, y)
-        x = check_convert_sample_array(x)
-        y = check_convert_tvalue_array(y)
-        check_sample_tvalue_size(x, y)
-
-        y = y.expand_dims(1) if y.ndim == 1
-        n_targets = y.shape[1]
-        n_features = x.shape[1]
-        sub_rng = @rng.dup
-
-        loss = Loss::MeanSquaredError.new
-        @network = buld_network(n_features, n_targets, sub_rng)
-        @network = train(x, y, @network, loss, sub_rng)
-        @network.delete_dropout
-
-        self
-      end
-
-      # Predict values for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
-      # @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted values per sample.
-      def predict(x)
-        x = check_convert_sample_array(x)
-        out, = @network.forward(x)
-        out = out[true, 0] if out.shape[1] == 1
-        out
-      end
-    end
-  end
-end