rumale 0.8.0

data/lib/rumale/pairwise_metric.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'rumale/validation'
+
+module Rumale
+  # Module for calculating pairwise distances, similarities, and kernels.
+  module PairwiseMetric
+    class << self
+      # Calculate the pairwise euclidean distances between x and y.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
+      # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
+      # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
+      def euclidean_distance(x, y = nil)
+        y = x if y.nil?
+        Rumale::Validation.check_sample_array(x)
+        Rumale::Validation.check_sample_array(y)
+        sum_x_vec = (x**2).sum(1)
+        sum_y_vec = (y**2).sum(1)
+        dot_xy_mat = x.dot(y.transpose)
+        distance_matrix = dot_xy_mat * -2.0 +
+                          sum_x_vec.tile(y.shape[0], 1).transpose +
+                          sum_y_vec.tile(x.shape[0], 1)
+        Numo::NMath.sqrt(distance_matrix.abs)
+      end
+
+      # Calculate the rbf kernel between x and y.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
+      # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
+      # @param gamma [Float] The parameter of rbf kernel, if nil it is 1 / n_features.
+      # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
+      def rbf_kernel(x, y = nil, gamma = nil)
+        y = x if y.nil?
+        gamma ||= 1.0 / x.shape[1]
+        Rumale::Validation.check_sample_array(x)
+        Rumale::Validation.check_sample_array(y)
+        Rumale::Validation.check_params_float(gamma: gamma)
+        distance_matrix = euclidean_distance(x, y)
+        Numo::NMath.exp((distance_matrix**2) * -gamma)
+      end
+
+      # Calculate the linear kernel between x and y.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
+      # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
+      # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
+      def linear_kernel(x, y = nil)
+        y = x if y.nil?
+        Rumale::Validation.check_sample_array(x)
+        Rumale::Validation.check_sample_array(y)
+        x.dot(y.transpose)
+      end
+
+      # Calculate the polynomial kernel between x and y.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
+      # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
+      # @param degree [Integer] The parameter of polynomial kernel.
+      # @param gamma [Float] The parameter of polynomial kernel, if nil it is 1 / n_features.
+      # @param coef [Integer] The parameter of polynomial kernel.
+      # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
+      def polynomial_kernel(x, y = nil, degree = 3, gamma = nil, coef = 1)
+        y = x if y.nil?
+        gamma ||= 1.0 / x.shape[1]
+        Rumale::Validation.check_sample_array(x)
+        Rumale::Validation.check_sample_array(y)
+        Rumale::Validation.check_params_float(gamma: gamma)
+        Rumale::Validation.check_params_integer(degree: degree, coef: coef)
+        (x.dot(y.transpose) * gamma + coef)**degree
+      end
+
+      # Calculate the sigmoid kernel between x and y.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples_x, n_features])
+      # @param y [Numo::DFloat] (shape: [n_samples_y, n_features])
+      # @param gamma [Float] The parameter of polynomial kernel, if nil it is 1 / n_features.
+      # @param coef [Integer] The parameter of polynomial kernel.
+      # @return [Numo::DFloat] (shape: [n_samples_x, n_samples_x] or [n_samples_x, n_samples_y] if y is given)
+      def sigmoid_kernel(x, y = nil, gamma = nil, coef = 1)
+        y = x if y.nil?
+        gamma ||= 1.0 / x.shape[1]
+        Rumale::Validation.check_sample_array(x)
+        Rumale::Validation.check_sample_array(y)
+        Rumale::Validation.check_params_float(gamma: gamma)
+        Rumale::Validation.check_params_integer(coef: coef)
+        Numo::NMath.tanh(x.dot(y.transpose) * gamma + coef)
+      end
+    end
+  end
+end

data/lib/rumale/pipeline/pipeline.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require 'rumale/validation'
+require 'rumale/base/base_estimator'
+
+module Rumale
+  # Module implements utilities of pipeline that cosists of a chain of transfomers and estimators.
+  module Pipeline
+    # Pipeline is a class that implements the function to perform the transformers and estimators sequencially.
+    #
+    # @example
+    #   rbf = Rumale::KernelApproximation::RBF.new(gamma: 1.0, n_coponents: 128, random_seed: 1)
+    #   svc = Rumale::LinearModel::SVC.new(reg_param: 1.0, fit_bias: true, max_iter: 5000, random_seed: 1)
+    #   pipeline = Rumale::Pipeline::Pipeline.new(steps: { trs: rbf, est: svc })
+    #   pipeline.fit(training_samples, traininig_labels)
+    #   results = pipeline.predict(testing_samples)
+    #
+    class Pipeline
+      include Base::BaseEstimator
+      include Validation
+
+      # Return the steps.
+      # @return [Hash]
+      attr_reader :steps
+
+      # Create a new pipeline.
+      #
+      # @param steps [Hash] List of transformers and estimators. The order of transforms follows the insertion order of hash keys.
+      #   The last entry is considered an estimator.
+      def initialize(steps:)
+        check_params_type(Hash, steps: steps)
+        validate_steps(steps)
+        @params = {}
+        @steps = steps
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be transformed and used for fitting the model.
+      # @param y [Numo::NArray] (shape: [n_samples, n_outputs]) The target values or labels to be used for fitting the model.
+      # @return [Pipeline] The learned pipeline itself.
+      def fit(x, y)
+        check_sample_array(x)
+        trans_x = apply_transforms(x, y, fit: true)
+        last_estimator&.fit(trans_x, y)
+        self
+      end
+
+      # Call the fit_predict method of last estimator after applying all transforms.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be transformed and used for fitting the model.
+      # @param y [Numo::NArray] (shape: [n_samples, n_outputs], default: nil) The target values or labels to be used for fitting the model.
+      # @return [Numo::NArray] The predicted results by last estimator.
+      def fit_predict(x, y = nil)
+        check_sample_array(x)
+        trans_x = apply_transforms(x, y, fit: true)
+        last_estimator.fit_predict(trans_x)
+      end
+
+      # Call the fit_transform method of last estimator after applying all transforms.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be transformed and used for fitting the model.
+      # @param y [Numo::NArray] (shape: [n_samples, n_outputs], default: nil) The target values or labels to be used for fitting the model.
+      # @return [Numo::NArray] The predicted results by last estimator.
+      def fit_transform(x, y = nil)
+        check_sample_array(x)
+        trans_x = apply_transforms(x, y, fit: true)
+        last_estimator.fit_transform(trans_x, y)
+      end
+
+      # Call the decision_function method of last estimator after applying all transforms.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
+      # @return [Numo::DFloat] (shape: [n_samples]) Confidence score per sample.
+      def decision_function(x)
+        check_sample_array(x)
+        trans_x = apply_transforms(x)
+        last_estimator.decision_function(trans_x)
+      end
+
+      # Call the predict method of last estimator after applying all transforms.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to obtain prediction result.
+      # @return [Numo::NArray] The predicted results by last estimator.
+      def predict(x)
+        check_sample_array(x)
+        trans_x = apply_transforms(x)
+        last_estimator.predict(trans_x)
+      end
+
+      # Call the predict_log_proba method of last estimator after applying all transforms.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the log-probailities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted log-probability of each class per sample.
+      def predict_log_proba(x)
+        check_sample_array(x)
+        trans_x = apply_transforms(x)
+        last_estimator.predict_log_proba(trans_x)
+      end
+
+      # Call the predict_proba method of last estimator after applying all transforms.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
+      def predict_proba(x)
+        check_sample_array(x)
+        trans_x = apply_transforms(x)
+        last_estimator.predict_proba(trans_x)
+      end
+
+      # Call the transform method of last estimator after applying all transforms.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to be transformed.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The transformed samples.
+      def transform(x)
+        check_sample_array(x)
+        trans_x = apply_transforms(x)
+        last_estimator.nil? ? trans_x : last_estimator.transform(trans_x)
+      end
+
+      # Call the inverse_transform method in reverse order.
+      #
+      # @param z [Numo::DFloat] (shape: [n_samples, n_components]) The transformed samples to be restored into original space.
+      # @return [Numo::DFloat] (shape: [n_samples, n_featuress]) The restored samples.
+      def inverse_transform(z)
+        check_sample_array(z)
+        itrans_z = z
+        @steps.keys.reverse_each do |name|
+          transformer = @steps[name]
+          next if transformer.nil?
+          itrans_z = transformer.inverse_transform(itrans_z)
+        end
+        itrans_z
+      end
+
+      # Call the score method of last estimator after applying all transforms.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) Testing data.
+      # @param y [Numo::NArray] (shape: [n_samples, n_outputs]) True target values or labels for testing data.
+      # @return [Float] The score of last estimator
+      def score(x, y)
+        check_sample_array(x)
+        trans_x = apply_transforms(x)
+        last_estimator.score(trans_x, y)
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data about Pipeline.
+      def marshal_dump
+        { params: @params,
+          steps: @steps }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @steps = obj[:steps]
+        nil
+      end
+
+      private
+
+      def validate_steps(steps)
+        steps.keys[0...-1].each do |name|
+          transformer = steps[name]
+          next if transformer.nil? || %i[fit transform].all? { |m| transformer.class.method_defined?(m) }
+          raise TypeError,
+                'Class of intermediate step in pipeline should be implemented fit and transform methods: ' \
+                "#{name} => #{transformer.class}"
+        end
+
+        estimator = steps[steps.keys.last]
+        unless estimator.nil? || estimator.class.method_defined?(:fit) # rubocop:disable Style/GuardClause
+          raise TypeError,
+                'Class of last step in pipeline should be implemented fit method: ' \
+                "#{steps.keys.last} => #{estimator.class}"
+        end
+      end
+
+      def apply_transforms(x, y = nil, fit: false)
+        trans_x = x
+        @steps.keys[0...-1].each do |name|
+          transformer = @steps[name]
+          next if transformer.nil?
+          transformer.fit(trans_x, y) if fit
+          trans_x = transformer.transform(trans_x)
+        end
+        trans_x
+      end
+
+      def last_estimator
+        @steps[@steps.keys.last]
+      end
+    end
+  end
+end

data/lib/rumale/polynomial_model/base_factorization_machine.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator'
+require 'rumale/optimizer/nadam'
+
+module Rumale
+  # This module consists of the classes that implement polynomial models.
+  module PolynomialModel
+    # BaseFactorizationMachine is an abstract class for implementation of Factorization Machine-based estimators.
+    # This class is used internally.
+    class BaseFactorizationMachine
+      include Base::BaseEstimator
+
+      # Initialize a Factorization Machine-based estimator.
+      #
+      # @param n_factors [Integer] The maximum number of iterations.
+      # @param loss [String] The loss function ('hinge' or 'logistic' or nil).
+      # @param reg_param_linear [Float] The regularization parameter for linear model.
+      # @param reg_param_factor [Float] The regularization parameter for factor matrix.
+      # @param max_iter [Integer] The maximum number of iterations.
+      # @param batch_size [Integer] The size of the mini batches.
+      # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
+      #   If nil is given, Nadam is used.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(n_factors: 2, loss: nil, reg_param_linear: 1.0, reg_param_factor: 1.0,
+                     max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
+        @params = {}
+        @params[:n_factors] = n_factors
+        @params[:loss] = loss unless loss.nil?
+        @params[:reg_param_linear] = reg_param_linear
+        @params[:reg_param_factor] = reg_param_factor
+        @params[:max_iter] = max_iter
+        @params[:batch_size] = batch_size
+        @params[:optimizer] = optimizer
+        @params[:optimizer] ||= Optimizer::Nadam.new
+        @params[:random_seed] = random_seed
+        @params[:random_seed] ||= srand
+        @factor_mat = nil
+        @weight_vec = nil
+        @bias_term = nil
+        @rng = Random.new(@params[:random_seed])
+      end
+
+      private
+
+      def partial_fit(x, y)
+        # Initialize some variables.
+        n_samples, n_features = x.shape
+        rand_ids = [*0...n_samples].shuffle(random: @rng)
+        weight_vec = Numo::DFloat.zeros(n_features + 1)
+        factor_mat = Numo::DFloat.zeros(@params[:n_factors], n_features)
+        weight_optimizer = @params[:optimizer].dup
+        factor_optimizers = Array.new(@params[:n_factors]) { @params[:optimizer].dup }
+        # Start optimization.
+        @params[:max_iter].times do |_t|
+          # Random sampling.
+          subset_ids = rand_ids.shift(@params[:batch_size])
+          rand_ids.concat(subset_ids)
+          data = x[subset_ids, true]
+          ex_data = expand_feature(data)
+          targets = y[subset_ids]
+          # Calculate gradients for loss function.
+          loss_grad = loss_gradient(data, ex_data, targets, factor_mat, weight_vec)
+          next if loss_grad.ne(0.0).count.zero?
+          # Update each parameter.
+          weight_vec = weight_optimizer.call(weight_vec, weight_gradient(loss_grad, ex_data, weight_vec))
+          @params[:n_factors].times do |n|
+            factor_mat[n, true] = factor_optimizers[n].call(factor_mat[n, true],
+                                                            factor_gradient(loss_grad, data, factor_mat[n, true]))
+          end
+        end
+        [factor_mat, *split_weight_vec_bias(weight_vec)]
+      end
+
+      def loss_gradient(_x, _expanded_x, _y, _factor, _weight)
+        raise NotImplementedError, "#{__method__} has to be implemented in #{self.class}."
+      end
+
+      def weight_gradient(loss_grad, data, weight)
+        (loss_grad.expand_dims(1) * data).mean(0) + @params[:reg_param_linear] * weight
+      end
+
+      def factor_gradient(loss_grad, data, factor)
+        (loss_grad.expand_dims(1) * (data * data.dot(factor).expand_dims(1) - factor * (data**2))).mean(0) +
+          @params[:reg_param_factor] * factor
+      end
+
+      def expand_feature(x)
+        Numo::NArray.hstack([x, Numo::DFloat.ones([x.shape[0], 1])])
+      end
+
+      def split_weight_vec_bias(weight_vec)
+        weights = weight_vec[0...-1].dup
+        bias = weight_vec[-1]
+        [weights, bias]
+      end
+    end
+  end
+end

data/lib/rumale/polynomial_model/factorization_machine_classifier.rb

@@ -0,0 +1,197 @@
+# frozen_string_literal: true
+
+require 'rumale/base/classifier'
+require 'rumale/polynomial_model/base_factorization_machine'
+
+module Rumale
+  # This module consists of the classes that implement polynomial models.
+  module PolynomialModel
+    # FactorizationMachineClassifier is a class that implements Factorization Machine
+    # with stochastic gradient descent (SGD) optimization.
+    # For multiclass classification problem, it uses one-vs-the-rest strategy.
+    #
+    # @example
+    #   estimator =
+    #     Rumale::PolynomialModel::FactorizationMachineClassifier.new(
+    #      n_factors: 10, loss: 'hinge', reg_param_linear: 0.001, reg_param_factor: 0.001,
+    #      max_iter: 5000, batch_size: 50, random_seed: 1)
+    #   estimator.fit(training_samples, traininig_labels)
+    #   results = estimator.predict(testing_samples)
+    #
+    # *Reference*
+    # - S. Rendle, "Factorization Machines with libFM," ACM TIST, vol. 3 (3), pp. 57:1--57:22, 2012.
+    # - S. Rendle, "Factorization Machines," Proc. ICDM'10, pp. 995--1000, 2010.
+    class FactorizationMachineClassifier < BaseFactorizationMachine
+      include Base::Classifier
+
+      # Return the factor matrix for Factorization Machine.
+      # @return [Numo::DFloat] (shape: [n_classes, n_factors, n_features])
+      attr_reader :factor_mat
+
+      # Return the weight vector for Factorization Machine.
+      # @return [Numo::DFloat] (shape: [n_classes, n_features])
+      attr_reader :weight_vec
+
+      # Return the bias term for Factoriazation Machine.
+      # @return [Numo::DFloat] (shape: [n_classes])
+      attr_reader :bias_term
+
+      # Return the class labels.
+      # @return [Numo::Int32] (shape: [n_classes])
+      attr_reader :classes
+
+      # Return the random generator for random sampling.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new classifier with Factorization Machine.
+      #
+      # @param n_factors [Integer] The maximum number of iterations.
+      # @param loss [String] The loss function ('hinge' or 'logistic').
+      # @param reg_param_linear [Float] The regularization parameter for linear model.
+      # @param reg_param_factor [Float] The regularization parameter for factor matrix.
+      # @param max_iter [Integer] The maximum number of iterations.
+      # @param batch_size [Integer] The size of the mini batches.
+      # @param optimizer [Optimizer] The optimizer to calculate adaptive learning rate.
+      #   If nil is given, Nadam is used.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(n_factors: 2, loss: 'hinge', reg_param_linear: 1.0, reg_param_factor: 1.0,
+                     max_iter: 1000, batch_size: 10, optimizer: nil, random_seed: nil)
+        check_params_float(reg_param_linear: reg_param_linear, reg_param_factor: reg_param_factor)
+        check_params_integer(n_factors: n_factors, max_iter: max_iter, batch_size: batch_size)
+        check_params_string(loss: loss)
+        check_params_type_or_nil(Integer, random_seed: random_seed)
+        check_params_positive(n_factors: n_factors,
+                              reg_param_linear: reg_param_linear, reg_param_factor: reg_param_factor,
+                              max_iter: max_iter, batch_size: batch_size)
+        super
+        @classes = nil
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [FactorizationMachineClassifier] The learned classifier itself.
+      def fit(x, y)
+        check_sample_array(x)
+        check_label_array(y)
+        check_sample_label_size(x, y)
+
+        @classes = Numo::Int32[*y.to_a.uniq.sort]
+        n_classes = @classes.size
+        _n_samples, n_features = x.shape
+
+        if n_classes > 2
+          @factor_mat = Numo::DFloat.zeros(n_classes, @params[:n_factors], n_features)
+          @weight_vec = Numo::DFloat.zeros(n_classes, n_features)
+          @bias_term = Numo::DFloat.zeros(n_classes)
+          n_classes.times do |n|
+            bin_y = Numo::Int32.cast(y.eq(@classes[n])) * 2 - 1
+            @factor_mat[n, true, true], @weight_vec[n, true], @bias_term[n] = partial_fit(x, bin_y)
+          end
+        else
+          negative_label = y.to_a.uniq.min
+          bin_y = Numo::Int32.cast(y.ne(negative_label)) * 2 - 1
+          @factor_mat, @weight_vec, @bias_term = partial_fit(x, bin_y)
+        end
+
+        self
+      end
+
+      # Calculate confidence scores for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
+      # @return [Numo::DFloat] (shape: [n_samples]) Confidence score per sample.
+      def decision_function(x)
+        check_sample_array(x)
+        linear_term = @bias_term + x.dot(@weight_vec.transpose)
+        factor_term = if @classes.size <= 2
+                        0.5 * (@factor_mat.dot(x.transpose)**2 - (@factor_mat**2).dot(x.transpose**2)).sum(0)
+                      else
+                        0.5 * (@factor_mat.dot(x.transpose)**2 - (@factor_mat**2).dot(x.transpose**2)).sum(1).transpose
+                      end
+        linear_term + factor_term
+      end
+
+      # Predict class labels for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
+      def predict(x)
+        check_sample_array(x)
+        return Numo::Int32.cast(decision_function(x).ge(0.0)) * 2 - 1 if @classes.size <= 2
+
+        n_samples, = x.shape
+        decision_values = decision_function(x)
+        Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })
+      end
+
+      # Predict probability for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
+      def predict_proba(x)
+        check_sample_array(x)
+        proba = 1.0 / (Numo::NMath.exp(-decision_function(x)) + 1.0)
+        return (proba.transpose / proba.sum(axis: 1)).transpose if @classes.size > 2
+
+        n_samples, = x.shape
+        probs = Numo::DFloat.zeros(n_samples, 2)
+        probs[true, 1] = proba
+        probs[true, 0] = 1.0 - proba
+        probs
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data about FactorizationMachineClassifier.
+      def marshal_dump
+        { params: @params,
+          factor_mat: @factor_mat,
+          weight_vec: @weight_vec,
+          bias_term: @bias_term,
+          classes: @classes,
+          rng: @rng }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @factor_mat = obj[:factor_mat]
+        @weight_vec = obj[:weight_vec]
+        @bias_term = obj[:bias_term]
+        @classes = obj[:classes]
+        @rng = obj[:rng]
+        nil
+      end
+
+      private
+
+      def bin_decision_function(x, ex_x, factor, weight)
+        ex_x.dot(weight) + 0.5 * (factor.dot(x.transpose)**2 - (factor**2).dot(x.transpose**2)).sum(0)
+      end
+
+      def hinge_loss_gradient(x, ex_x, y, factor, weight)
+        evaluated = y * bin_decision_function(x, ex_x, factor, weight)
+        gradient = Numo::DFloat.zeros(evaluated.size)
+        gradient[evaluated < 1.0] = -y[evaluated < 1.0]
+        gradient
+      end
+
+      def logistic_loss_gradient(x, ex_x, y, factor, weight)
+        evaluated = y * bin_decision_function(x, ex_x, factor, weight)
+        sigmoid_func = 1.0 / (Numo::NMath.exp(-evaluated) + 1.0)
+        (sigmoid_func - 1.0) * y
+      end
+
+      def loss_gradient(x, ex_x, y, factor, weight)
+        if @params[:loss] == 'hinge'
+          hinge_loss_gradient(x, ex_x, y, factor, weight)
+        else
+          logistic_loss_gradient(x, ex_x, y, factor, weight)
+        end
+      end
+    end
+  end
+end