rumale 0.8.0

data/lib/rumale/nearest_neighbors/k_neighbors_classifier.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator'
+require 'rumale/base/classifier'
+
+module Rumale
+  # This module consists of the classes that implement estimators based on nearest neighbors rule.
+  module NearestNeighbors
+    # KNeighborsClassifier is a class that implements the classifier with the k-nearest neighbors rule.
+    # The current implementation uses the Euclidean distance for finding the neighbors.
+    #
+    # @example
+    #   estimator =
+    #     Rumale::NearestNeighbors::KNeighborsClassifier.new(n_neighbors = 5)
+    #   estimator.fit(training_samples, traininig_labels)
+    #   results = estimator.predict(testing_samples)
+    #
+    class KNeighborsClassifier
+      include Base::BaseEstimator
+      include Base::Classifier
+
+      # Return the prototypes for the nearest neighbor classifier.
+      # @return [Numo::DFloat] (shape: [n_samples, n_features])
+      attr_reader :prototypes
+
+      # Return the labels of the prototypes
+      # @return [Numo::Int32] (size: n_samples)
+      attr_reader :labels
+
+      # Return the class labels.
+      # @return [Numo::Int32] (size: n_classes)
+      attr_reader :classes
+
+      # Create a new classifier with the nearest neighbor rule.
+      #
+      # @param n_neighbors [Integer] The number of neighbors.
+      def initialize(n_neighbors: 5)
+        check_params_integer(n_neighbors: n_neighbors)
+        check_params_positive(n_neighbors: n_neighbors)
+        @params = {}
+        @params[:n_neighbors] = n_neighbors
+        @prototypes = nil
+        @labels = nil
+        @classes = nil
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [KNeighborsClassifier] The learned classifier itself.
+      def fit(x, y)
+        check_sample_array(x)
+        check_label_array(y)
+        check_sample_label_size(x, y)
+        @prototypes = Numo::DFloat.asarray(x.to_a)
+        @labels = Numo::Int32.asarray(y.to_a)
+        @classes = Numo::Int32.asarray(y.to_a.uniq.sort)
+        self
+      end
+
+      # Calculate confidence scores for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence scores per sample for each class.
+      def decision_function(x)
+        check_sample_array(x)
+        distance_matrix = PairwiseMetric.euclidean_distance(x, @prototypes)
+        n_samples, n_prototypes = distance_matrix.shape
+        n_classes = @classes.size
+        n_neighbors = [@params[:n_neighbors], n_prototypes].min
+        scores = Numo::DFloat.zeros(n_samples, n_classes)
+        n_samples.times do |m|
+          neighbor_ids = distance_matrix[m, true].to_a.each_with_index.sort.map(&:last)[0...n_neighbors]
+          neighbor_ids.each { |n| scores[m, @classes.to_a.index(@labels[n])] += 1.0 }
+        end
+        scores
+      end
+
+      # Predict class labels for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
+      def predict(x)
+        check_sample_array(x)
+        n_samples = x.shape.first
+        decision_values = decision_function(x)
+        Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data about KNeighborsClassifier.
+      def marshal_dump
+        { params: @params,
+          prototypes: @prototypes,
+          labels: @labels,
+          classes: @classes }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @prototypes = obj[:prototypes]
+        @labels = obj[:labels]
+        @classes = obj[:classes]
+        nil
+      end
+    end
+  end
+end

data/lib/rumale/nearest_neighbors/k_neighbors_regressor.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator'
+require 'rumale/base/regressor'
+
+module Rumale
+  module NearestNeighbors
+    # KNeighborsRegressor is a class that implements the regressor with the k-nearest neighbors rule.
+    # The current implementation uses the Euclidean distance for finding the neighbors.
+    #
+    # @example
+    #   estimator =
+    #     Rumale::NearestNeighbors::KNeighborsRegressor.new(n_neighbors = 5)
+    #   estimator.fit(training_samples, traininig_target_values)
+    #   results = estimator.predict(testing_samples)
+    #
+    class KNeighborsRegressor
+      include Base::BaseEstimator
+      include Base::Regressor
+
+      # Return the prototypes for the nearest neighbor regressor.
+      # @return [Numo::DFloat] (shape: [n_samples, n_features])
+      attr_reader :prototypes
+
+      # Return the values of the prototypes
+      # @return [Numo::DFloat] (shape: [n_samples, n_outputs])
+      attr_reader :values
+
+      # Create a new regressor with the nearest neighbor rule.
+      #
+      # @param n_neighbors [Integer] The number of neighbors.
+      def initialize(n_neighbors: 5)
+        check_params_integer(n_neighbors: n_neighbors)
+        check_params_positive(n_neighbors: n_neighbors)
+        @params = {}
+        @params[:n_neighbors] = n_neighbors
+        @prototypes = nil
+        @values = nil
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values to be used for fitting the model.
+      # @return [KNeighborsRegressor] The learned regressor itself.
+      def fit(x, y)
+        check_sample_array(x)
+        check_tvalue_array(y)
+        check_sample_tvalue_size(x, y)
+        @prototypes = x.dup
+        @values = y.dup
+        self
+      end
+
+      # Predict values for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
+      # @return [Numo::DFloat] (shape: [n_samples, n_outputs]) Predicted values per sample.
+      def predict(x)
+        check_sample_array(x)
+        # Initialize some variables.
+        n_samples, = x.shape
+        n_prototypes, n_outputs = @values.shape
+        n_neighbors = [@params[:n_neighbors], n_prototypes].min
+        # Calculate distance matrix.
+        distance_matrix = PairwiseMetric.euclidean_distance(x, @prototypes)
+        # Predict values for the given samples.
+        predicted_values = Array.new(n_samples) do |n|
+          neighbor_ids = distance_matrix[n, true].to_a.each_with_index.sort.map(&:last)[0...n_neighbors]
+          n_outputs.nil? ? @values[neighbor_ids].mean : @values[neighbor_ids, true].mean(0).to_a
+        end
+        Numo::DFloat[*predicted_values]
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data about KNeighborsRegressor.
+      def marshal_dump
+        { params: @params,
+          prototypes: @prototypes,
+          values: @values }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @prototypes = obj[:prototypes]
+        @values = obj[:values]
+        nil
+      end
+    end
+  end
+end

data/lib/rumale/optimizer/nadam.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'rumale/validation'
+require 'rumale/base/base_estimator'
+
+module Rumale
+  # This module consists of the classes that implement optimizers adaptively tuning hyperparameters.
+  module Optimizer
+    # Nadam is a class that implements Nadam optimizer.
+    #
+    # @example
+    #   optimizer = Rumale::Optimizer::Nadam.new(learning_rate: 0.01, momentum: 0.9, decay1: 0.9, decay2: 0.999)
+    #   estimator = Rumale::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
+    #   estimator.fit(samples, values)
+    #
+    # *Reference*
+    # - T. Dozat, "Incorporating Nesterov Momentum into Adam," Tech. Repo. Stanford University, 2015.
+    class Nadam
+      include Base::BaseEstimator
+      include Validation
+
+      # Create a new optimizer with Nadam
+      #
+      # @param learning_rate [Float] The initial value of learning rate.
+      # @param momentum [Float] The initial value of momentum.
+      # @param decay1 [Float] The smoothing parameter for the first moment.
+      # @param decay2 [Float] The smoothing parameter for the second moment.
+      def initialize(learning_rate: 0.01, momentum: 0.9, decay1: 0.9, decay2: 0.999)
+        check_params_float(learning_rate: learning_rate, momentum: momentum, decay1: decay1, decay2: decay2)
+        check_params_positive(learning_rate: learning_rate, momentum: momentum, decay1: decay1, decay2: decay2)
+        @params = {}
+        @params[:learning_rate] = learning_rate
+        @params[:momentum] = momentum
+        @params[:decay1] = decay1
+        @params[:decay2] = decay2
+        @fst_moment = nil
+        @sec_moment = nil
+        @decay1_prod = 1.0
+        @iter = 0
+      end
+
+      # Calculate the updated weight with Nadam adaptive learning rate.
+      #
+      # @param weight [Numo::DFloat] (shape: [n_features]) The weight to be updated.
+      # @param gradient [Numo::DFloat] (shape: [n_features]) The gradient for updating the weight.
+      # @return [Numo::DFloat] (shape: [n_feautres]) The updated weight.
+      def call(weight, gradient)
+        @fst_moment ||= Numo::DFloat.zeros(weight.shape[0])
+        @sec_moment ||= Numo::DFloat.zeros(weight.shape[0])
+
+        @iter += 1
+
+        decay1_curr = @params[:decay1] * (1.0 - 0.5 * 0.96**(@iter * 0.004))
+        decay1_next = @params[:decay1] * (1.0 - 0.5 * 0.96**((@iter + 1) * 0.004))
+        decay1_prod_curr = @decay1_prod * decay1_curr
+        decay1_prod_next = @decay1_prod * decay1_curr * decay1_next
+        @decay1_prod = decay1_prod_curr
+
+        @fst_moment = @params[:decay1] * @fst_moment + (1.0 - @params[:decay1]) * gradient
+        @sec_moment = @params[:decay2] * @sec_moment + (1.0 - @params[:decay2]) * gradient**2
+        nm_gradient = gradient / (1.0 - decay1_prod_curr)
+        nm_fst_moment = @fst_moment / (1.0 - decay1_prod_next)
+        nm_sec_moment = @sec_moment / (1.0 - @params[:decay2]**@iter)
+
+        weight - (@params[:learning_rate] / (nm_sec_moment**0.5 + 1e-8)) * ((1 - decay1_curr) * nm_gradient + decay1_next * nm_fst_moment)
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data.
+      def marshal_dump
+        { params: @params,
+          fst_moment: @fst_moment,
+          sec_moment: @sec_moment,
+          decay1_prod: @decay1_prod,
+          iter: @iter }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @fst_moment = obj[:fst_moment]
+        @sec_moment = obj[:sec_moment]
+        @decay1_prod = obj[:decay1_prod]
+        @iter = obj[:iter]
+        nil
+      end
+    end
+  end
+end

data/lib/rumale/optimizer/rmsprop.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'rumale/validation'
+require 'rumale/base/base_estimator'
+
+module Rumale
+  module Optimizer
+    # RMSProp is a class that implements RMSProp optimizer.
+    #
+    # @example
+    #   optimizer = Rumale::Optimizer::RMSProp.new(learning_rate: 0.01, momentum: 0.9, decay: 0.9)
+    #   estimator = Rumale::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
+    #   estimator.fit(samples, values)
+    #
+    # *Reference*
+    # - I. Sutskever, J. Martens, G. Dahl, and G. Hinton, "On the importance of initialization and momentum in deep learning," Proc. ICML' 13, pp. 1139--1147, 2013.
+    # - G. Hinton, N. Srivastava, and K. Swersky, "Lecture 6e rmsprop," Neural Networks for Machine Learning, 2012.
+    class RMSProp
+      include Base::BaseEstimator
+      include Validation
+
+      # Create a new optimizer with RMSProp.
+      #
+      # @param learning_rate [Float] The initial value of learning rate.
+      # @param momentum [Float] The initial value of momentum.
+      # @param decay [Float] The smooting parameter.
+      def initialize(learning_rate: 0.01, momentum: 0.9, decay: 0.9)
+        check_params_float(learning_rate: learning_rate, momentum: momentum, decay: decay)
+        check_params_positive(learning_rate: learning_rate, momentum: momentum, decay: decay)
+        @params = {}
+        @params[:learning_rate] = learning_rate
+        @params[:momentum] = momentum
+        @params[:decay] = decay
+        @moment = nil
+        @update = nil
+      end
+
+      # Calculate the updated weight with RMSProp adaptive learning rate.
+      #
+      # @param weight [Numo::DFloat] (shape: [n_features]) The weight to be updated.
+      # @param gradient [Numo::DFloat] (shape: [n_features]) The gradient for updating the weight.
+      # @return [Numo::DFloat] (shape: [n_feautres]) The updated weight.
+      def call(weight, gradient)
+        @moment ||= Numo::DFloat.zeros(weight.shape[0])
+        @update ||= Numo::DFloat.zeros(weight.shape[0])
+        @moment = @params[:decay] * @moment + (1.0 - @params[:decay]) * gradient**2
+        @update = @params[:momentum] * @update - (@params[:learning_rate] / (@moment**0.5 + 1.0e-8)) * gradient
+        weight + @update
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data.
+      def marshal_dump
+        { params: @params,
+          moment: @moment,
+          update: @update }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @moment = obj[:moment]
+        @update = obj[:update]
+        nil
+      end
+    end
+  end
+end

data/lib/rumale/optimizer/sgd.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'rumale/validation'
+require 'rumale/base/base_estimator'
+
+module Rumale
+  module Optimizer
+    # SGD is a class that implements SGD optimizer.
+    #
+    # @example
+    #   optimizer = Rumale::Optimizer::SGD.new(learning_rate: 0.01, momentum: 0.9, decay: 0.9)
+    #   estimator = Rumale::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
+    #   estimator.fit(samples, values)
+    class SGD
+      include Base::BaseEstimator
+      include Validation
+
+      # Create a new optimizer with SGD.
+      #
+      # @param learning_rate [Float] The initial value of learning rate.
+      # @param momentum [Float] The initial value of momentum.
+      # @param decay [Float] The smooting parameter.
+      def initialize(learning_rate: 0.01, momentum: 0.0, decay: 0.0)
+        check_params_float(learning_rate: learning_rate, momentum: momentum, decay: decay)
+        check_params_positive(learning_rate: learning_rate, momentum: momentum, decay: decay)
+        @params = {}
+        @params[:learning_rate] = learning_rate
+        @params[:momentum] = momentum
+        @params[:decay] = decay
+        @iter = 0
+        @update = nil
+      end
+
+      # Calculate the updated weight with SGD.
+      #
+      # @param weight [Numo::DFloat] (shape: [n_features]) The weight to be updated.
+      # @param gradient [Numo::DFloat] (shape: [n_features]) The gradient for updating the weight.
+      # @return [Numo::DFloat] (shape: [n_feautres]) The updated weight.
+      def call(weight, gradient)
+        @update ||= Numo::DFloat.zeros(weight.shape[0])
+        current_learning_rate = @params[:learning_rate] / (1.0 + @params[:decay] * @iter)
+        @iter += 1
+        @update = @params[:momentum] * @update - current_learning_rate * gradient
+        weight + @update
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data.
+      def marshal_dump
+        { params: @params,
+          iter: @iter,
+          update: @update }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @iter = obj[:iter]
+        @update = obj[:update]
+        nil
+      end
+    end
+  end
+end

data/lib/rumale/optimizer/yellow_fin.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require 'rumale/validation'
+require 'rumale/base/base_estimator'
+
+module Rumale
+  module Optimizer
+    # YellowFin is a class that implements YellowFin optimizer.
+    #
+    # @example
+    #   optimizer = Rumale::Optimizer::YellowFin.new(learning_rate: 0.01, momentum: 0.9, decay: 0.999, window_width: 20)
+    #   estimator = Rumale::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
+    #   estimator.fit(samples, values)
+    #
+    # *Reference*
+    # - J. Zhang and I. Mitliagkas, "YellowFin and the Art of Momentum Tuning," CoRR abs/1706.03471, 2017.
+    class YellowFin
+      include Base::BaseEstimator
+      include Validation
+
+      # Create a new optimizer with YellowFin.
+      #
+      # @param learning_rate [Float] The initial value of learning rate.
+      # @param momentum [Float] The initial value of momentum.
+      # @param decay [Float] The smooting parameter.
+      # @param window_width [Integer] The sliding window width for searching curvature range.
+      def initialize(learning_rate: 0.01, momentum: 0.9, decay: 0.999, window_width: 20)
+        check_params_float(learning_rate: learning_rate, momentum: momentum, decay: decay)
+        check_params_integer(window_width: window_width)
+        check_params_positive(learning_rate: learning_rate, momentum: momentum, decay: decay, window_width: window_width)
+        @params = {}
+        @params[:learning_rate] = learning_rate
+        @params[:momentum] = momentum
+        @params[:decay] = decay
+        @params[:window_width] = window_width
+        @smth_learning_rate = learning_rate
+        @smth_momentum = momentum
+        @grad_norms = nil
+        @grad_norm_min = 0.0
+        @grad_norm_max = 0.0
+        @grad_mean_sqr = 0.0
+        @grad_mean = 0.0
+        @grad_var = 0.0
+        @grad_norm_mean = 0.0
+        @curve_mean = 0.0
+        @distance_mean = 0.0
+        @update = nil
+      end
+
+      # Calculate the updated weight with adaptive momentum coefficient and learning rate.
+      #
+      # @param weight [Numo::DFloat] (shape: [n_features]) The weight to be updated.
+      # @param gradient [Numo::DFloat] (shape: [n_features]) The gradient for updating the weight.
+      # @return [Numo::DFloat] (shape: [n_feautres]) The updated weight.
+      def call(weight, gradient)
+        @update ||= Numo::DFloat.zeros(weight.shape[0])
+        curvature_range(gradient)
+        gradient_variance(gradient)
+        distance_to_optimum(gradient)
+        @smth_momentum = @params[:decay] * @smth_momentum + (1 - @params[:decay]) * current_momentum
+        @smth_learning_rate = @params[:decay] * @smth_learning_rate + (1 - @params[:decay]) * current_learning_rate
+        @update = @smth_momentum * @update - @smth_learning_rate * gradient
+        weight + @update
+      end
+
+      private
+
+      def current_momentum
+        dr = Math.sqrt(@grad_norm_max / @grad_norm_min + 1.0e-8)
+        [cubic_root**2, ((dr - 1) / (dr + 1))**2].max
+      end
+
+      def current_learning_rate
+        (1.0 - Math.sqrt(@params[:momentum]))**2 / (@grad_norm_min + 1.0e-8)
+      end
+
+      def cubic_root
+        p = (@distance_mean**2 * @grad_norm_min**2) / (2 * @grad_var + 1.0e-8)
+        w3 = (-Math.sqrt(p**2 + 4.fdiv(27) * p**3) - p).fdiv(2)
+        w = (w3 >= 0.0 ? 1 : -1) * w3.abs**1.fdiv(3)
+        y = w - p / (3 * w + 1.0e-8)
+        y + 1
+      end
+
+      def curvature_range(gradient)
+        @grad_norms ||= []
+        @grad_norms.push((gradient**2).sum)
+        @grad_norms.shift(@grad_norms.size - @params[:window_width]) if @grad_norms.size > @params[:window_width]
+        @grad_norm_min = @params[:decay] * @grad_norm_min + (1 - @params[:decay]) * @grad_norms.min
+        @grad_norm_max = @params[:decay] * @grad_norm_max + (1 - @params[:decay]) * @grad_norms.max
+      end
+
+      def gradient_variance(gradient)
+        @grad_mean_sqr = @params[:decay] * @grad_mean_sqr + (1 - @params[:decay]) * gradient**2
+        @grad_mean = @params[:decay] * @grad_mean + (1 - @params[:decay]) * gradient
+        @grad_var = (@grad_mean_sqr - @grad_mean**2).sum
+      end
+
+      def distance_to_optimum(gradient)
+        grad_sqr = (gradient**2).sum
+        @grad_norm_mean = @params[:decay] * @grad_norm_mean + (1 - @params[:decay]) * Math.sqrt(grad_sqr + 1.0e-8)
+        @curve_mean = @params[:decay] * @curve_mean + (1 - @params[:decay]) * grad_sqr
+        @distance_mean = @params[:decay] * @distance_mean + (1 - @params[:decay]) * (@grad_norm_mean / @curve_mean)
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data.
+      def marshal_dump
+        { params: @params,
+          smth_learning_rate: @smth_learning_rate,
+          smth_momentum: @smth_momentum,
+          grad_norms: @grad_norms,
+          grad_norm_min: @grad_norm_min,
+          grad_norm_max: @grad_norm_max,
+          grad_mean_sqr: @grad_mean_sqr,
+          grad_mean: @grad_mean,
+          grad_var: @grad_var,
+          grad_norm_mean: @grad_norm_mean,
+          curve_mean: @curve_mean,
+          distance_mean: @distance_mean,
+          update: @update }
+      end
+
+      # Load marshal data.
+      # @return [nis]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @smth_learning_rate = obj[:smth_learning_rate]
+        @smth_momentum = obj[:smth_momentum]
+        @grad_norms = obj[:grad_norms]
+        @grad_norm_min = obj[:grad_norm_min]
+        @grad_norm_max = obj[:grad_norm_max]
+        @grad_mean_sqr = obj[:grad_mean_sqr]
+        @grad_mean = obj[:grad_mean]
+        @grad_var = obj[:grad_var]
+        @grad_norm_mean = obj[:grad_norm_mean]
+        @curve_mean = obj[:curve_mean]
+        @distance_mean = obj[:distance_mean]
+        @update = obj[:update]
+        nil
+      end
+    end
+  end
+end