rumale 0.8.0

data/lib/rumale/preprocessing/standard_scaler.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator'
+require 'rumale/base/transformer'
+
+module Rumale
+  # This module consists of the classes that perform preprocessings.
+  module Preprocessing
+    # Normalize samples by centering and scaling to unit variance.
+    #
+    # @example
+    #   normalizer = Rumale::Preprocessing::StandardScaler.new
+    #   new_training_samples = normalizer.fit_transform(training_samples)
+    #   new_testing_samples = normalizer.transform(testing_samples)
+    class StandardScaler
+      include Base::BaseEstimator
+      include Base::Transformer
+
+      # Return the vector consists of the mean value for each feature.
+      # @return [Numo::DFloat] (shape: [n_features])
+      attr_reader :mean_vec
+
+      # Return the vector consists of the standard deviation for each feature.
+      # @return [Numo::DFloat] (shape: [n_features])
+      attr_reader :std_vec
+
+      # Create a new normalizer for centering and scaling to unit variance.
+      def initialize
+        @params = {}
+        @mean_vec = nil
+        @std_vec = nil
+      end
+
+      # Calculate the mean value and standard deviation of each feature for scaling.
+      #
+      # @overload fit(x) -> StandardScaler
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features])
+      #   The samples to calculate the mean values and standard deviations.
+      # @return [StandardScaler]
+      def fit(x, _y = nil)
+        check_sample_array(x)
+        @mean_vec = x.mean(0)
+        @std_vec = x.stddev(0)
+        self
+      end
+
+      # Calculate the mean values and standard deviations, and then normalize samples using them.
+      #
+      # @overload fit_transform(x) -> Numo::DFloat
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features])
+      #   The samples to calculate the mean values and standard deviations.
+      # @return [Numo::DFloat] The scaled samples.
+      def fit_transform(x, _y = nil)
+        check_sample_array(x)
+        fit(x).transform(x)
+      end
+
+      # Perform standardization the given samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to be scaled.
+      # @return [Numo::DFloat] The scaled samples.
+      def transform(x)
+        check_sample_array(x)
+        n_samples, = x.shape
+        (x - @mean_vec.tile(n_samples, 1)) / @std_vec.tile(n_samples, 1)
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data about StandardScaler.
+      def marshal_dump
+        { mean_vec: @mean_vec,
+          std_vec: @std_vec }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @mean_vec = obj[:mean_vec]
+        @std_vec = obj[:std_vec]
+        nil
+      end
+    end
+  end
+end

data/lib/rumale/probabilistic_output.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Rumale
+  # Module for calculating posterior class probabilities with SVM outputs.
+  # This module is used for internal processes.
+  #
+  # @example
+  #   estimator = Rumale::LinearModel::SVC.new
+  #   estimator.fit(x, bin_y)
+  #   df = estimator.decision_function(x)
+  #   params = Rumale::ProbabilisticOutput.fit_sigmoid(df, bin_y)
+  #   probs = 1 / (Numo::NMath.exp(params[0] * df + params[1]) + 1)
+  #
+  # *Reference*
+  # 1. J C. Platt, "Probabilistic Outputs for Support Vector Machines and Comparisons to Regularized Likelihood Methods," Adv. Large Margin Classifiers, pp. 61--74, 2000.
+  # 1. H-T Lin, C-J Lin, and R C.Weng, "A Note on Platt's Probabilistic Outputs for Support Vector Machines," J. Machine Learning, Vol. 63 (3), pp. 267--276, 2007.
+  module ProbabilisticOutput
+    class << self
+      # Fit the probabilistic model for binary SVM outputs.
+      #
+      # @param df [Numo::DFloat] (shape: [n_samples]) The outputs of decision function to be used for fitting the model.
+      # @param bin_y [Numo::Int32] (shape: [n_samples]) The binary labels to be used for fitting the model.
+      # @param max_iter [Integer] The maximum number of iterations.
+      # @param min_step [Float] The minimum step of Newton's method.
+      # @param sigma [Float] The parameter to avoid hessian matrix from becoming singular matrix.
+      # @return [Numo::DFloat] (shape: 2) The parameters of the model.
+      def fit_sigmoid(df, bin_y, max_iter = 100, min_step = 1e-10, sigma = 1e-12)
+        # Initialize some variables.
+        n_samples = bin_y.size
+        negative_label = bin_y.to_a.uniq.min
+        pos = bin_y.ne(negative_label)
+        neg = bin_y.eq(negative_label)
+        n_pos_samples = pos.count
+        n_neg_samples = neg.count
+        target_probs = Numo::DFloat.zeros(n_samples)
+        target_probs[pos] = (n_pos_samples + 1) / (n_pos_samples + 2.0)
+        target_probs[neg] = 1 / (n_neg_samples + 2.0)
+        alpha = 0.0
+        beta = Math.log((n_neg_samples + 1) / (n_pos_samples + 1.0))
+        err = error_function(target_probs, df, alpha, beta)
+        # Optimize parameters for class porbability calculation.
+        old_grad_vec = Numo::DFloat.zeros(2)
+        max_iter.times do
+          # Calculate gradient and hessian matrix.
+          probs = predicted_probs(df, alpha, beta)
+          grad_vec = gradient(target_probs, probs, df)
+          hess_mat = hessian_matrix(probs, df, sigma)
+          break if grad_vec.abs.lt(1e-5).count == 2
+          break if (old_grad_vec - grad_vec).abs.sum < 1e-5
+          old_grad_vec = grad_vec
+          # Calculate Newton directions.
+          dirs_vec = directions(grad_vec, hess_mat)
+          grad_dir = grad_vec.dot(dirs_vec)
+          stepsize = 2.0
+          while stepsize >= min_step
+            stepsize *= 0.5
+            new_alpha = alpha + stepsize * dirs_vec[0]
+            new_beta = beta + stepsize * dirs_vec[1]
+            new_err = error_function(target_probs, df, new_alpha, new_beta)
+            next unless new_err < err + 0.0001 * stepsize * grad_dir
+            alpha = new_alpha
+            beta = new_beta
+            err = new_err
+            break
+          end
+        end
+        Numo::DFloat[alpha, beta]
+      end
+
+      private
+
+      def error_function(target_probs, df, alpha, beta)
+        fn = alpha * df + beta
+        pos = fn.ge(0.0)
+        neg = fn.lt(0.0)
+        err = 0.0
+        err += (target_probs[pos] * fn[pos] + Numo::NMath.log(1 + Numo::NMath.exp(-fn[pos]))).sum if pos.count.positive?
+        err += ((target_probs[neg] - 1) * fn[neg] + Numo::NMath.log(1 + Numo::NMath.exp(fn[neg]))).sum if neg.count.positive?
+        err
+      end
+
+      def predicted_probs(df, alpha, beta)
+        fn = alpha * df + beta
+        pos = fn.ge(0.0)
+        neg = fn.lt(0.0)
+        probs = Numo::DFloat.zeros(df.shape[0])
+        probs[pos] = Numo::NMath.exp(-fn[pos]) / (1 + Numo::NMath.exp(-fn[pos])) if pos.count.positive?
+        probs[neg] = 1 / (1 + Numo::NMath.exp(fn[neg])) if neg.count.positive?
+        probs
+      end
+
+      def gradient(target_probs, probs, df)
+        sub = target_probs - probs
+        Numo::DFloat[(df * sub).sum, sub.sum]
+      end
+
+      def hessian_matrix(probs, df, sigma)
+        sub = probs * (1 - probs)
+        h11 = (df * df * sub).sum + sigma
+        h22 = sub.sum + sigma
+        h21 = (df * sub).sum
+        Numo::DFloat[[h11, h21], [h21, h22]]
+      end
+
+      def directions(grad_vec, hess_mat)
+        det = hess_mat[0, 0] * hess_mat[1, 1] - hess_mat[0, 1] * hess_mat[1, 0]
+        inv_hess_mat = Numo::DFloat[[hess_mat[1, 1], -hess_mat[0, 1]], [-hess_mat[1, 0], hess_mat[0, 0]]] / det
+        -inv_hess_mat.dot(grad_vec)
+      end
+    end
+  end
+end

data/lib/rumale/tree/base_decision_tree.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator'
+require 'rumale/tree/node'
+
+module Rumale
+  # This module consists of the classes that implement tree models.
+  module Tree
+    # BaseDecisionTree is an abstract class for implementation of decision tree-based estimator.
+    # This class is used internally.
+    class BaseDecisionTree
+      include Base::BaseEstimator
+
+      # Initialize a decision tree-based estimator.
+      #
+      # @param criterion [String] The function to evalue spliting point.
+      # @param max_depth [Integer] The maximum depth of the tree.
+      #   If nil is given, decision tree grows without concern for depth.
+      # @param max_leaf_nodes [Integer] The maximum number of leaves on decision tree.
+      #   If nil is given, number of leaves is not limited.
+      # @param min_samples_leaf [Integer] The minimum number of samples at a leaf node.
+      # @param max_features [Integer] The number of features to consider when searching optimal split point.
+      #   If nil is given, split process considers all features.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      #   It is used to randomly determine the order of features when deciding spliting point.
+      def initialize(criterion: nil, max_depth: nil, max_leaf_nodes: nil, min_samples_leaf: 1, max_features: nil, random_seed: nil)
+        @params = {}
+        @params[:criterion] = criterion
+        @params[:max_depth] = max_depth
+        @params[:max_leaf_nodes] = max_leaf_nodes
+        @params[:min_samples_leaf] = min_samples_leaf
+        @params[:max_features] = max_features
+        @params[:random_seed] = random_seed
+        @params[:random_seed] ||= srand
+        @tree = nil
+        @feature_importances = nil
+        @n_leaves = nil
+        @rng = Random.new(@params[:random_seed])
+      end
+
+      # Return the index of the leaf that each sample reached.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Leaf index for sample.
+      def apply(x)
+        check_sample_array(x)
+        Numo::Int32[*(Array.new(x.shape[0]) { |n| apply_at_node(@tree, x[n, true]) })]
+      end
+
+      private
+
+      def apply_at_node(node, sample)
+        return node.leaf_id if node.leaf
+        return apply_at_node(node.left, sample) if node.right.nil?
+        return apply_at_node(node.right, sample) if node.left.nil?
+        if sample[node.feature_id] <= node.threshold
+          apply_at_node(node.left, sample)
+        else
+          apply_at_node(node.right, sample)
+        end
+      end
+
+      def build_tree(x, y)
+        y = y.expand_dims(1).dup if y.shape[1].nil?
+        @tree = grow_node(0, x, y, impurity(y))
+        nil
+      end
+
+      def grow_node(depth, x, y, whole_impurity)
+        unless @params[:max_leaf_nodes].nil?
+          return nil if @n_leaves >= @params[:max_leaf_nodes]
+        end
+
+        n_samples, n_features = x.shape
+        return nil if n_samples <= @params[:min_samples_leaf]
+
+        node = Node.new(depth: depth, impurity: whole_impurity, n_samples: n_samples)
+
+        return put_leaf(node, y) if stop_growing?(y)
+
+        unless @params[:max_depth].nil?
+          return put_leaf(node, y) if depth == @params[:max_depth]
+        end
+
+        feature_id, threshold, left_ids, right_ids, left_impurity, right_impurity, gain =
+          rand_ids(n_features).map { |f_id| [f_id, *best_split(x[true, f_id], y, whole_impurity)] }.max_by(&:last)
+
+        return put_leaf(node, y) if gain.nil? || gain.zero?
+
+        node.left = grow_node(depth + 1, x[left_ids, true], y[left_ids, true], left_impurity)
+        node.right = grow_node(depth + 1, x[right_ids, true], y[right_ids, true], right_impurity)
+
+        return put_leaf(node, y) if node.left.nil? && node.right.nil?
+
+        node.feature_id = feature_id
+        node.threshold = threshold
+        node.leaf = false
+        node
+      end
+
+      def stop_growing?(_y)
+        raise NotImplementedError, "#{__method__} has to be implemented in #{self.class}."
+      end
+
+      def put_leaf(_node, _y)
+        raise NotImplementedError, "#{__method__} has to be implemented in #{self.class}."
+      end
+
+      def rand_ids(n)
+        [*0...n].sample(@params[:max_features], random: @rng)
+      end
+
+      def best_split(features, targets, whole_impurity)
+        n_samples = targets.shape[0]
+        features.to_a.uniq.sort.each_cons(2).map do |l, r|
+          threshold = 0.5 * (l + r)
+          left_ids = features.le(threshold).where
+          right_ids = features.gt(threshold).where
+          left_impurity = impurity(targets[left_ids, true])
+          right_impurity = impurity(targets[right_ids, true])
+          gain = whole_impurity -
+                 left_impurity * left_ids.size.fdiv(n_samples) -
+                 right_impurity * right_ids.size.fdiv(n_samples)
+          [threshold, left_ids, right_ids, left_impurity, right_impurity, gain]
+        end.max_by(&:last)
+      end
+
+      def impurity(_targets)
+        raise NotImplementedError, "#{__method__} has to be implemented in #{self.class}."
+      end
+
+      def eval_importance(n_samples, n_features)
+        @feature_importances = Numo::DFloat.zeros(n_features)
+        eval_importance_at_node(@tree)
+        @feature_importances /= n_samples
+        normalizer = @feature_importances.sum
+        @feature_importances /= normalizer if normalizer > 0.0
+        nil
+      end
+
+      def eval_importance_at_node(node)
+        return nil if node.leaf
+        return nil if node.left.nil? || node.right.nil?
+        gain = node.n_samples * node.impurity -
+               node.left.n_samples * node.left.impurity -
+               node.right.n_samples * node.right.impurity
+        @feature_importances[node.feature_id] += gain
+        eval_importance_at_node(node.left)
+        eval_importance_at_node(node.right)
+      end
+    end
+  end
+end

data/lib/rumale/tree/decision_tree_classifier.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'rumale/tree/base_decision_tree'
+require 'rumale/base/classifier'
+
+module Rumale
+  module Tree
+    # DecisionTreeClassifier is a class that implements decision tree for classification.
+    #
+    # @example
+    #   estimator =
+    #     Rumale::Tree::DecisionTreeClassifier.new(
+    #       criterion: 'gini', max_depth: 3, max_leaf_nodes: 10, min_samples_leaf: 5, random_seed: 1)
+    #   estimator.fit(training_samples, traininig_labels)
+    #   results = estimator.predict(testing_samples)
+    #
+    class DecisionTreeClassifier < BaseDecisionTree
+      include Base::Classifier
+
+      # Return the class labels.
+      # @return [Numo::Int32] (size: n_classes)
+      attr_reader :classes
+
+      # Return the importance for each feature.
+      # @return [Numo::DFloat] (size: n_features)
+      attr_reader :feature_importances
+
+      # Return the learned tree.
+      # @return [Node]
+      attr_reader :tree
+
+      # Return the random generator for random selection of feature index.
+      # @return [Random]
+      attr_reader :rng
+
+      # Return the labels assigned each leaf.
+      # @return [Numo::Int32] (size: n_leafs)
+      attr_reader :leaf_labels
+
+      # Create a new classifier with decision tree algorithm.
+      #
+      # @param criterion [String] The function to evalue spliting point. Supported criteria are 'gini' and 'entropy'.
+      # @param max_depth [Integer] The maximum depth of the tree.
+      #   If nil is given, decision tree grows without concern for depth.
+      # @param max_leaf_nodes [Integer] The maximum number of leaves on decision tree.
+      #   If nil is given, number of leaves is not limited.
+      # @param min_samples_leaf [Integer] The minimum number of samples at a leaf node.
+      # @param max_features [Integer] The number of features to consider when searching optimal split point.
+      #   If nil is given, split process considers all features.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      #   It is used to randomly determine the order of features when deciding spliting point.
+      def initialize(criterion: 'gini', max_depth: nil, max_leaf_nodes: nil, min_samples_leaf: 1, max_features: nil,
+                     random_seed: nil)
+        check_params_type_or_nil(Integer, max_depth: max_depth, max_leaf_nodes: max_leaf_nodes,
+                                          max_features: max_features, random_seed: random_seed)
+        check_params_integer(min_samples_leaf: min_samples_leaf)
+        check_params_string(criterion: criterion)
+        check_params_positive(max_depth: max_depth, max_leaf_nodes: max_leaf_nodes,
+                              min_samples_leaf: min_samples_leaf, max_features: max_features)
+        super
+        @leaf_labels = nil
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [DecisionTreeClassifier] The learned classifier itself.
+      def fit(x, y)
+        check_sample_array(x)
+        check_label_array(y)
+        check_sample_label_size(x, y)
+        n_samples, n_features = x.shape
+        @params[:max_features] = n_features if @params[:max_features].nil?
+        @params[:max_features] = [@params[:max_features], n_features].min
+        uniq_y = y.to_a.uniq.sort
+        @classes = Numo::Int32.asarray(uniq_y)
+        @n_leaves = 0
+        @leaf_labels = []
+        build_tree(x, y.map { |v| uniq_y.index(v) })
+        eval_importance(n_samples, n_features)
+        @leaf_labels = Numo::Int32[*@leaf_labels]
+        self
+      end
+
+      # Predict class labels for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
+      def predict(x)
+        check_sample_array(x)
+        @leaf_labels[apply(x)]
+      end
+
+      # Predict probability for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
+      def predict_proba(x)
+        check_sample_array(x)
+        Numo::DFloat[*(Array.new(x.shape[0]) { |n| predict_proba_at_node(@tree, x[n, true]) })]
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data about DecisionTreeClassifier
+      def marshal_dump
+        { params: @params,
+          classes: @classes,
+          tree: @tree,
+          feature_importances: @feature_importances,
+          leaf_labels: @leaf_labels,
+          rng: @rng }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @classes = obj[:classes]
+        @tree = obj[:tree]
+        @feature_importances = obj[:feature_importances]
+        @leaf_labels = obj[:leaf_labels]
+        @rng = obj[:rng]
+        nil
+      end
+
+      private
+
+      def predict_proba_at_node(node, sample)
+        return node.probs if node.leaf
+        return predict_proba_at_node(node.left, sample) if node.right.nil?
+        return predict_proba_at_node(node.right, sample) if node.left.nil?
+        if sample[node.feature_id] <= node.threshold
+          predict_proba_at_node(node.left, sample)
+        else
+          predict_proba_at_node(node.right, sample)
+        end
+      end
+
+      def stop_growing?(y)
+        y.flatten.to_a.uniq.size == 1
+      end
+
+      def put_leaf(node, y)
+        node.probs = y.flatten.bincount(minlength: @classes.size) / node.n_samples.to_f
+        node.leaf = true
+        node.leaf_id = @n_leaves
+        @n_leaves += 1
+        @leaf_labels.push(@classes[node.probs.max_index])
+        node
+      end
+
+      def impurity(y)
+        posterior_probs = y.flatten.bincount / y.size.to_f
+        if @params[:criterion] == 'entropy'
+          -(posterior_probs * Numo::NMath.log(posterior_probs + 1)).sum
+        else
+          1.0 - (posterior_probs * posterior_probs).sum
+        end
+      end
+    end
+  end
+end