svmkit 0.7.3 → 0.8.1

data/lib/svmkit/optimizer/nadam.rb

@@ -1,90 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
-
-module SVMKit
-  # This module consists of the classes that implement optimizers adaptively tuning hyperparameters.
-  module Optimizer
-    # Nadam is a class that implements Nadam optimizer.
-    #
-    # @example
-    #   optimizer = SVMKit::Optimizer::Nadam.new(learning_rate: 0.01, momentum: 0.9, decay1: 0.9, decay2: 0.999)
-    #   estimator = SVMKit::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
-    #   estimator.fit(samples, values)
-    #
-    # *Reference*
-    # - T. Dozat, "Incorporating Nesterov Momentum into Adam," Tech. Repo. Stanford University, 2015.
-    class Nadam
-      include Base::BaseEstimator
-      include Validation
-
-      # Create a new optimizer with Nadam
-      #
-      # @param learning_rate [Float] The initial value of learning rate.
-      # @param momentum [Float] The initial value of momentum.
-      # @param decay1 [Float] The smoothing parameter for the first moment.
-      # @param decay2 [Float] The smoothing parameter for the second moment.
-      def initialize(learning_rate: 0.01, momentum: 0.9, decay1: 0.9, decay2: 0.999)
-        check_params_float(learning_rate: learning_rate, momentum: momentum, decay1: decay1, decay2: decay2)
-        check_params_positive(learning_rate: learning_rate, momentum: momentum, decay1: decay1, decay2: decay2)
-        @params = {}
-        @params[:learning_rate] = learning_rate
-        @params[:momentum] = momentum
-        @params[:decay1] = decay1
-        @params[:decay2] = decay2
-        @fst_moment = nil
-        @sec_moment = nil
-        @decay1_prod = 1.0
-        @iter = 0
-      end
-
-      # Calculate the updated weight with Nadam adaptive learning rate.
-      #
-      # @param weight [Numo::DFloat] (shape: [n_features]) The weight to be updated.
-      # @param gradient [Numo::DFloat] (shape: [n_features]) The gradient for updating the weight.
-      # @return [Numo::DFloat] (shape: [n_feautres]) The updated weight.
-      def call(weight, gradient)
-        @fst_moment ||= Numo::DFloat.zeros(weight.shape[0])
-        @sec_moment ||= Numo::DFloat.zeros(weight.shape[0])
-
-        @iter += 1
-
-        decay1_curr = @params[:decay1] * (1.0 - 0.5 * 0.96**(@iter * 0.004))
-        decay1_next = @params[:decay1] * (1.0 - 0.5 * 0.96**((@iter + 1) * 0.004))
-        decay1_prod_curr = @decay1_prod * decay1_curr
-        decay1_prod_next = @decay1_prod * decay1_curr * decay1_next
-        @decay1_prod = decay1_prod_curr
-
-        @fst_moment = @params[:decay1] * @fst_moment + (1.0 - @params[:decay1]) * gradient
-        @sec_moment = @params[:decay2] * @sec_moment + (1.0 - @params[:decay2]) * gradient**2
-        nm_gradient = gradient / (1.0 - decay1_prod_curr)
-        nm_fst_moment = @fst_moment / (1.0 - decay1_prod_next)
-        nm_sec_moment = @sec_moment / (1.0 - @params[:decay2]**@iter)
-
-        weight - (@params[:learning_rate] / (nm_sec_moment**0.5 + 1e-8)) * ((1 - decay1_curr) * nm_gradient + decay1_next * nm_fst_moment)
-      end
-
-      # Dump marshal data.
-      # @return [Hash] The marshal data.
-      def marshal_dump
-        { params: @params,
-          fst_moment: @fst_moment,
-          sec_moment: @sec_moment,
-          decay1_prod: @decay1_prod,
-          iter: @iter }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @fst_moment = obj[:fst_moment]
-        @sec_moment = obj[:sec_moment]
-        @decay1_prod = obj[:decay1_prod]
-        @iter = obj[:iter]
-        nil
-      end
-    end
-  end
-end

data/lib/svmkit/optimizer/rmsprop.rb

@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
-
-module SVMKit
-  module Optimizer
-    # RMSProp is a class that implements RMSProp optimizer.
-    #
-    # @example
-    #   optimizer = SVMKit::Optimizer::RMSProp.new(learning_rate: 0.01, momentum: 0.9, decay: 0.9)
-    #   estimator = SVMKit::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
-    #   estimator.fit(samples, values)
-    #
-    # *Reference*
-    # - I. Sutskever, J. Martens, G. Dahl, and G. Hinton, "On the importance of initialization and momentum in deep learning," Proc. ICML' 13, pp. 1139--1147, 2013.
-    # - G. Hinton, N. Srivastava, and K. Swersky, "Lecture 6e rmsprop," Neural Networks for Machine Learning, 2012.
-    class RMSProp
-      include Base::BaseEstimator
-      include Validation
-
-      # Create a new optimizer with RMSProp.
-      #
-      # @param learning_rate [Float] The initial value of learning rate.
-      # @param momentum [Float] The initial value of momentum.
-      # @param decay [Float] The smooting parameter.
-      def initialize(learning_rate: 0.01, momentum: 0.9, decay: 0.9)
-        check_params_float(learning_rate: learning_rate, momentum: momentum, decay: decay)
-        check_params_positive(learning_rate: learning_rate, momentum: momentum, decay: decay)
-        @params = {}
-        @params[:learning_rate] = learning_rate
-        @params[:momentum] = momentum
-        @params[:decay] = decay
-        @moment = nil
-        @update = nil
-      end
-
-      # Calculate the updated weight with RMSProp adaptive learning rate.
-      #
-      # @param weight [Numo::DFloat] (shape: [n_features]) The weight to be updated.
-      # @param gradient [Numo::DFloat] (shape: [n_features]) The gradient for updating the weight.
-      # @return [Numo::DFloat] (shape: [n_feautres]) The updated weight.
-      def call(weight, gradient)
-        @moment ||= Numo::DFloat.zeros(weight.shape[0])
-        @update ||= Numo::DFloat.zeros(weight.shape[0])
-        @moment = @params[:decay] * @moment + (1.0 - @params[:decay]) * gradient**2
-        @update = @params[:momentum] * @update - (@params[:learning_rate] / (@moment**0.5 + 1.0e-8)) * gradient
-        weight + @update
-      end
-
-      # Dump marshal data.
-      # @return [Hash] The marshal data.
-      def marshal_dump
-        { params: @params,
-          moment: @moment,
-          update: @update }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @moment = obj[:moment]
-        @update = obj[:update]
-        nil
-      end
-    end
-  end
-end

data/lib/svmkit/optimizer/sgd.rb

@@ -1,65 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
-
-module SVMKit
-  module Optimizer
-    # SGD is a class that implements SGD optimizer.
-    #
-    # @example
-    #   optimizer = SVMKit::Optimizer::SGD.new(learning_rate: 0.01, momentum: 0.9, decay: 0.9)
-    #   estimator = SVMKit::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
-    #   estimator.fit(samples, values)
-    class SGD
-      include Base::BaseEstimator
-      include Validation
-
-      # Create a new optimizer with SGD.
-      #
-      # @param learning_rate [Float] The initial value of learning rate.
-      # @param momentum [Float] The initial value of momentum.
-      # @param decay [Float] The smooting parameter.
-      def initialize(learning_rate: 0.01, momentum: 0.0, decay: 0.0)
-        check_params_float(learning_rate: learning_rate, momentum: momentum, decay: decay)
-        check_params_positive(learning_rate: learning_rate, momentum: momentum, decay: decay)
-        @params = {}
-        @params[:learning_rate] = learning_rate
-        @params[:momentum] = momentum
-        @params[:decay] = decay
-        @iter = 0
-        @update = nil
-      end
-
-      # Calculate the updated weight with SGD.
-      #
-      # @param weight [Numo::DFloat] (shape: [n_features]) The weight to be updated.
-      # @param gradient [Numo::DFloat] (shape: [n_features]) The gradient for updating the weight.
-      # @return [Numo::DFloat] (shape: [n_feautres]) The updated weight.
-      def call(weight, gradient)
-        @update ||= Numo::DFloat.zeros(weight.shape[0])
-        current_learning_rate = @params[:learning_rate] / (1.0 + @params[:decay] * @iter)
-        @iter += 1
-        @update = @params[:momentum] * @update - current_learning_rate * gradient
-        weight + @update
-      end
-
-      # Dump marshal data.
-      # @return [Hash] The marshal data.
-      def marshal_dump
-        { params: @params,
-          iter: @iter,
-          update: @update }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @iter = obj[:iter]
-        @update = obj[:update]
-        nil
-      end
-    end
-  end
-end

data/lib/svmkit/optimizer/yellow_fin.rb

@@ -1,144 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
-
-module SVMKit
-  module Optimizer
-    # YellowFin is a class that implements YellowFin optimizer.
-    #
-    # @example
-    #   optimizer = SVMKit::Optimizer::YellowFin.new(learning_rate: 0.01, momentum: 0.9, decay: 0.999, window_width: 20)
-    #   estimator = SVMKit::LinearModel::LinearRegression.new(optimizer: optimizer, random_seed: 1)
-    #   estimator.fit(samples, values)
-    #
-    # *Reference*
-    # - J. Zhang and I. Mitliagkas, "YellowFin and the Art of Momentum Tuning," CoRR abs/1706.03471, 2017.
-    class YellowFin
-      include Base::BaseEstimator
-      include Validation
-
-      # Create a new optimizer with YellowFin.
-      #
-      # @param learning_rate [Float] The initial value of learning rate.
-      # @param momentum [Float] The initial value of momentum.
-      # @param decay [Float] The smooting parameter.
-      # @param window_width [Integer] The sliding window width for searching curvature range.
-      def initialize(learning_rate: 0.01, momentum: 0.9, decay: 0.999, window_width: 20)
-        check_params_float(learning_rate: learning_rate, momentum: momentum, decay: decay)
-        check_params_integer(window_width: window_width)
-        check_params_positive(learning_rate: learning_rate, momentum: momentum, decay: decay, window_width: window_width)
-        @params = {}
-        @params[:learning_rate] = learning_rate
-        @params[:momentum] = momentum
-        @params[:decay] = decay
-        @params[:window_width] = window_width
-        @smth_learning_rate = learning_rate
-        @smth_momentum = momentum
-        @grad_norms = nil
-        @grad_norm_min = 0.0
-        @grad_norm_max = 0.0
-        @grad_mean_sqr = 0.0
-        @grad_mean = 0.0
-        @grad_var = 0.0
-        @grad_norm_mean = 0.0
-        @curve_mean = 0.0
-        @distance_mean = 0.0
-        @update = nil
-      end
-
-      # Calculate the updated weight with adaptive momentum coefficient and learning rate.
-      #
-      # @param weight [Numo::DFloat] (shape: [n_features]) The weight to be updated.
-      # @param gradient [Numo::DFloat] (shape: [n_features]) The gradient for updating the weight.
-      # @return [Numo::DFloat] (shape: [n_feautres]) The updated weight.
-      def call(weight, gradient)
-        @update ||= Numo::DFloat.zeros(weight.shape[0])
-        curvature_range(gradient)
-        gradient_variance(gradient)
-        distance_to_optimum(gradient)
-        @smth_momentum = @params[:decay] * @smth_momentum + (1 - @params[:decay]) * current_momentum
-        @smth_learning_rate = @params[:decay] * @smth_learning_rate + (1 - @params[:decay]) * current_learning_rate
-        @update = @smth_momentum * @update - @smth_learning_rate * gradient
-        weight + @update
-      end
-
-      private
-
-      def current_momentum
-        dr = Math.sqrt(@grad_norm_max / @grad_norm_min + 1.0e-8)
-        [cubic_root**2, ((dr - 1) / (dr + 1))**2].max
-      end
-
-      def current_learning_rate
-        (1.0 - Math.sqrt(@params[:momentum]))**2 / (@grad_norm_min + 1.0e-8)
-      end
-
-      def cubic_root
-        p = (@distance_mean**2 * @grad_norm_min**2) / (2 * @grad_var + 1.0e-8)
-        w3 = (-Math.sqrt(p**2 + 4.fdiv(27) * p**3) - p).fdiv(2)
-        w = (w3 >= 0.0 ? 1 : -1) * w3.abs**1.fdiv(3)
-        y = w - p / (3 * w + 1.0e-8)
-        y + 1
-      end
-
-      def curvature_range(gradient)
-        @grad_norms ||= []
-        @grad_norms.push((gradient**2).sum)
-        @grad_norms.shift(@grad_norms.size - @params[:window_width]) if @grad_norms.size > @params[:window_width]
-        @grad_norm_min = @params[:decay] * @grad_norm_min + (1 - @params[:decay]) * @grad_norms.min
-        @grad_norm_max = @params[:decay] * @grad_norm_max + (1 - @params[:decay]) * @grad_norms.max
-      end
-
-      def gradient_variance(gradient)
-        @grad_mean_sqr = @params[:decay] * @grad_mean_sqr + (1 - @params[:decay]) * gradient**2
-        @grad_mean = @params[:decay] * @grad_mean + (1 - @params[:decay]) * gradient
-        @grad_var = (@grad_mean_sqr - @grad_mean**2).sum
-      end
-
-      def distance_to_optimum(gradient)
-        grad_sqr = (gradient**2).sum
-        @grad_norm_mean = @params[:decay] * @grad_norm_mean + (1 - @params[:decay]) * Math.sqrt(grad_sqr + 1.0e-8)
-        @curve_mean = @params[:decay] * @curve_mean + (1 - @params[:decay]) * grad_sqr
-        @distance_mean = @params[:decay] * @distance_mean + (1 - @params[:decay]) * (@grad_norm_mean / @curve_mean)
-      end
-
-      # Dump marshal data.
-      # @return [Hash] The marshal data.
-      def marshal_dump
-        { params: @params,
-          smth_learning_rate: @smth_learning_rate,
-          smth_momentum: @smth_momentum,
-          grad_norms: @grad_norms,
-          grad_norm_min: @grad_norm_min,
-          grad_norm_max: @grad_norm_max,
-          grad_mean_sqr: @grad_mean_sqr,
-          grad_mean: @grad_mean,
-          grad_var: @grad_var,
-          grad_norm_mean: @grad_norm_mean,
-          curve_mean: @curve_mean,
-          distance_mean: @distance_mean,
-          update: @update }
-      end
-
-      # Load marshal data.
-      # @return [nis]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @smth_learning_rate = obj[:smth_learning_rate]
-        @smth_momentum = obj[:smth_momentum]
-        @grad_norms = obj[:grad_norms]
-        @grad_norm_min = obj[:grad_norm_min]
-        @grad_norm_max = obj[:grad_norm_max]
-        @grad_mean_sqr = obj[:grad_mean_sqr]
-        @grad_mean = obj[:grad_mean]
-        @grad_var = obj[:grad_var]
-        @grad_norm_mean = obj[:grad_norm_mean]
-        @curve_mean = obj[:curve_mean]
-        @distance_mean = obj[:distance_mean]
-        @update = obj[:update]
-        nil
-      end
-    end
-  end
-end

data/lib/svmkit/pairwise_metric.rb

@@ -1,91 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-
-module SVMKit
-  # Module for calculating pairwise distances, similarities, and kernels.
-  module PairwiseMetric
-    class << self
-      # Calculate the pairwise euclidean distances between x and y.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
-      # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
-      # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
-      def euclidean_distance(x, y = nil)
-        y = x if y.nil?
-        SVMKit::Validation.check_sample_array(x)
-        SVMKit::Validation.check_sample_array(y)
-        sum_x_vec = (x**2).sum(1)
-        sum_y_vec = (y**2).sum(1)
-        dot_xy_mat = x.dot(y.transpose)
-        distance_matrix = dot_xy_mat * -2.0 +
-                          sum_x_vec.tile(y.shape[0], 1).transpose +
-                          sum_y_vec.tile(x.shape[0], 1)
-        Numo::NMath.sqrt(distance_matrix.abs)
-      end
-
-      # Calculate the rbf kernel between x and y.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
-      # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
-      # @param gamma [Float] The parameter of rbf kernel, if nil it is 1 / n_features.
-      # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
-      def rbf_kernel(x, y = nil, gamma = nil)
-        y = x if y.nil?
-        gamma ||= 1.0 / x.shape[1]
-        SVMKit::Validation.check_sample_array(x)
-        SVMKit::Validation.check_sample_array(y)
-        SVMKit::Validation.check_params_float(gamma: gamma)
-        distance_matrix = euclidean_distance(x, y)
-        Numo::NMath.exp((distance_matrix**2) * -gamma)
-      end
-
-      # Calculate the linear kernel between x and y.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
-      # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
-      # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
-      def linear_kernel(x, y = nil)
-        y = x if y.nil?
-        SVMKit::Validation.check_sample_array(x)
-        SVMKit::Validation.check_sample_array(y)
-        x.dot(y.transpose)
-      end
-
-      # Calculate the polynomial kernel between x and y.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
-      # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
-      # @param degree [Integer] The parameter of polynomial kernel.
-      # @param gamma [Float] The parameter of polynomial kernel, if nil it is 1 / n_features.
-      # @param coef [Integer] The parameter of polynomial kernel.
-      # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
-      def polynomial_kernel(x, y = nil, degree = 3, gamma = nil, coef = 1)
-        y = x if y.nil?
-        gamma ||= 1.0 / x.shape[1]
-        SVMKit::Validation.check_sample_array(x)
-        SVMKit::Validation.check_sample_array(y)
-        SVMKit::Validation.check_params_float(gamma: gamma)
-        SVMKit::Validation.check_params_integer(degree: degree, coef: coef)
-        (x.dot(y.transpose) * gamma + coef)**degree
-      end
-
-      # Calculate the sigmoid kernel between x and y.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
-      # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
-      # @param gamma [Float] The parameter of polynomial kernel, if nil it is 1 / n_features.
-      # @param coef [Integer] The parameter of polynomial kernel.
-      # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
-      def sigmoid_kernel(x, y = nil, gamma = nil, coef = 1)
-        y = x if y.nil?
-        gamma ||= 1.0 / x.shape[1]
-        SVMKit::Validation.check_sample_array(x)
-        SVMKit::Validation.check_sample_array(y)
-        SVMKit::Validation.check_params_float(gamma: gamma)
-        SVMKit::Validation.check_params_integer(coef: coef)
-        Numo::NMath.tanh(x.dot(y.transpose) * gamma + coef)
-      end
-    end
-  end
-end