svmkit 0.7.3 → 0.8.1

data/lib/svmkit/preprocessing/standard_scaler.rb

@@ -1,87 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
-require 'svmkit/base/transformer'
-
-module SVMKit
-  # This module consists of the classes that perform preprocessings.
-  module Preprocessing
-    # Normalize samples by centering and scaling to unit variance.
-    #
-    # @example
-    #   normalizer = SVMKit::Preprocessing::StandardScaler.new
-    #   new_training_samples = normalizer.fit_transform(training_samples)
-    #   new_testing_samples = normalizer.transform(testing_samples)
-    class StandardScaler
-      include Base::BaseEstimator
-      include Base::Transformer
-
-      # Return the vector consists of the mean value for each feature.
-      # @return [Numo::DFloat] (shape: [n_features])
-      attr_reader :mean_vec
-
-      # Return the vector consists of the standard deviation for each feature.
-      # @return [Numo::DFloat] (shape: [n_features])
-      attr_reader :std_vec
-
-      # Create a new normalizer for centering and scaling to unit variance.
-      def initialize
-        @params = {}
-        @mean_vec = nil
-        @std_vec = nil
-      end
-
-      # Calculate the mean value and standard deviation of each feature for scaling.
-      #
-      # @overload fit(x) -> StandardScaler
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features])
-      #   The samples to calculate the mean values and standard deviations.
-      # @return [StandardScaler]
-      def fit(x, _y = nil)
-        SVMKit::Validation.check_sample_array(x)
-        @mean_vec = x.mean(0)
-        @std_vec = x.stddev(0)
-        self
-      end
-
-      # Calculate the mean values and standard deviations, and then normalize samples using them.
-      #
-      # @overload fit_transform(x) -> Numo::DFloat
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features])
-      #   The samples to calculate the mean values and standard deviations.
-      # @return [Numo::DFloat] The scaled samples.
-      def fit_transform(x, _y = nil)
-        SVMKit::Validation.check_sample_array(x)
-        fit(x).transform(x)
-      end
-
-      # Perform standardization the given samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to be scaled.
-      # @return [Numo::DFloat] The scaled samples.
-      def transform(x)
-        SVMKit::Validation.check_sample_array(x)
-        n_samples, = x.shape
-        (x - @mean_vec.tile(n_samples, 1)) / @std_vec.tile(n_samples, 1)
-      end
-
-      # Dump marshal data.
-      # @return [Hash] The marshal data about StandardScaler.
-      def marshal_dump
-        { mean_vec: @mean_vec,
-          std_vec: @std_vec }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @mean_vec = obj[:mean_vec]
-        @std_vec = obj[:std_vec]
-        nil
-      end
-    end
-  end
-end

data/lib/svmkit/probabilistic_output.rb

@@ -1,112 +0,0 @@
-# frozen_string_literal: true
-
-module SVMKit
-  # Module for calculating posterior class probabilities with SVM outputs.
-  # This module is used for internal processes.
-  #
-  # @example
-  #   estimator = SVMKit::LinearModel::SVC.new
-  #   estimator.fit(x, bin_y)
-  #   df = estimator.decision_function(x)
-  #   params = SVMKit::ProbabilisticOutput.fit_sigmoid(df, bin_y)
-  #   probs = 1 / (Numo::NMath.exp(params[0] * df + params[1]) + 1)
-  #
-  # *Reference*
-  # 1. J C. Platt, "Probabilistic Outputs for Support Vector Machines and Comparisons to Regularized Likelihood Methods," Adv. Large Margin Classifiers, pp. 61--74, 2000.
-  # 1. H-T Lin, C-J Lin, and R C.Weng, "A Note on Platt's Probabilistic Outputs for Support Vector Machines," J. Machine Learning, Vol. 63 (3), pp. 267--276, 2007.
-  module ProbabilisticOutput
-    class << self
-      # Fit the probabilistic model for binary SVM outputs.
-      #
-      # @param df [Numo::DFloat] (shape: [n_samples]) The outputs of decision function to be used for fitting the model.
-      # @param bin_y [Numo::Int32] (shape: [n_samples]) The binary labels to be used for fitting the model.
-      # @param max_iter [Integer] The maximum number of iterations.
-      # @param min_step [Float] The minimum step of Newton's method.
-      # @param sigma [Float] The parameter to avoid hessian matrix from becoming singular matrix.
-      # @return [Numo::DFloat] (shape: 2) The parameters of the model.
-      def fit_sigmoid(df, bin_y, max_iter = 100, min_step = 1e-10, sigma = 1e-12)
-        # Initialize some variables.
-        n_samples = bin_y.size
-        negative_label = bin_y.to_a.uniq.min
-        pos = bin_y.ne(negative_label)
-        neg = bin_y.eq(negative_label)
-        n_pos_samples = pos.count
-        n_neg_samples = neg.count
-        target_probs = Numo::DFloat.zeros(n_samples)
-        target_probs[pos] = (n_pos_samples + 1) / (n_pos_samples + 2.0)
-        target_probs[neg] = 1 / (n_neg_samples + 2.0)
-        alpha = 0.0
-        beta = Math.log((n_neg_samples + 1) / (n_pos_samples + 1.0))
-        err = error_function(target_probs, df, alpha, beta)
-        # Optimize parameters for class porbability calculation.
-        old_grad_vec = Numo::DFloat.zeros(2)
-        max_iter.times do
-          # Calculate gradient and hessian matrix.
-          probs = predicted_probs(df, alpha, beta)
-          grad_vec = gradient(target_probs, probs, df)
-          hess_mat = hessian_matrix(probs, df, sigma)
-          break if grad_vec.abs.lt(1e-5).count == 2
-          break if (old_grad_vec - grad_vec).abs.sum < 1e-5
-          old_grad_vec = grad_vec
-          # Calculate Newton directions.
-          dirs_vec = directions(grad_vec, hess_mat)
-          grad_dir = grad_vec.dot(dirs_vec)
-          stepsize = 2.0
-          while stepsize >= min_step
-            stepsize *= 0.5
-            new_alpha = alpha + stepsize * dirs_vec[0]
-            new_beta = beta + stepsize * dirs_vec[1]
-            new_err = error_function(target_probs, df, new_alpha, new_beta)
-            next unless new_err < err + 0.0001 * stepsize * grad_dir
-            alpha = new_alpha
-            beta = new_beta
-            err = new_err
-            break
-          end
-        end
-        Numo::DFloat[alpha, beta]
-      end
-
-      private
-
-      def error_function(target_probs, df, alpha, beta)
-        fn = alpha * df + beta
-        pos = fn.ge(0.0)
-        neg = fn.lt(0.0)
-        err = 0.0
-        err += (target_probs[pos] * fn[pos] + Numo::NMath.log(1 + Numo::NMath.exp(-fn[pos]))).sum if pos.count > 0
-        err += ((target_probs[neg] - 1) * fn[neg] + Numo::NMath.log(1 + Numo::NMath.exp(fn[neg]))).sum if neg.count > 0
-        err
-      end
-
-      def predicted_probs(df, alpha, beta)
-        fn = alpha * df + beta
-        pos = fn.ge(0.0)
-        neg = fn.lt(0.0)
-        probs = Numo::DFloat.zeros(df.shape[0])
-        probs[pos] = Numo::NMath.exp(-fn[pos]) / (1 + Numo::NMath.exp(-fn[pos])) if pos.count > 0
-        probs[neg] = 1 / (1 + Numo::NMath.exp(fn[neg])) if neg.count > 0
-        probs
-      end
-
-      def gradient(target_probs, probs, df)
-        sub = target_probs - probs
-        Numo::DFloat[(df * sub).sum, sub.sum]
-      end
-
-      def hessian_matrix(probs, df, sigma)
-        sub = probs * (1 - probs)
-        h11 = (df * df * sub).sum + sigma
-        h22 = sub.sum + sigma
-        h21 = (df * sub).sum
-        Numo::DFloat[[h11, h21], [h21, h22]]
-      end
-
-      def directions(grad_vec, hess_mat)
-        det = hess_mat[0, 0] * hess_mat[1, 1] - hess_mat[0, 1] * hess_mat[1, 0]
-        inv_hess_mat = Numo::DFloat[[hess_mat[1, 1], -hess_mat[0, 1]], [-hess_mat[1, 0], hess_mat[0, 0]]] / det
-        -inv_hess_mat.dot(grad_vec)
-      end
-    end
-  end
-end

data/lib/svmkit/tree/decision_tree_classifier.rb

@@ -1,276 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
-require 'svmkit/base/classifier'
-require 'svmkit/tree/node'
-
-module SVMKit
-  # This module consists of the classes that implement tree models.
-  module Tree
-    # DecisionTreeClassifier is a class that implements decision tree for classification.
-    #
-    # @example
-    #   estimator =
-    #     SVMKit::Tree::DecisionTreeClassifier.new(
-    #       criterion: 'gini', max_depth: 3, max_leaf_nodes: 10, min_samples_leaf: 5, random_seed: 1)
-    #   estimator.fit(training_samples, traininig_labels)
-    #   results = estimator.predict(testing_samples)
-    #
-    class DecisionTreeClassifier
-      include Base::BaseEstimator
-      include Base::Classifier
-
-      # Return the class labels.
-      # @return [Numo::Int32] (size: n_classes)
-      attr_reader :classes
-
-      # Return the importance for each feature.
-      # @return [Numo::DFloat] (size: n_features)
-      attr_reader :feature_importances
-
-      # Return the learned tree.
-      # @return [Node]
-      attr_reader :tree
-
-      # Return the random generator for random selection of feature index.
-      # @return [Random]
-      attr_reader :rng
-
-      # Return the labels assigned each leaf.
-      # @return [Numo::Int32] (size: n_leafs)
-      attr_reader :leaf_labels
-
-      # Create a new classifier with decision tree algorithm.
-      #
-      # @param criterion [String] The function to evalue spliting point. Supported criteria are 'gini' and 'entropy'.
-      # @param max_depth [Integer] The maximum depth of the tree.
-      #   If nil is given, decision tree grows without concern for depth.
-      # @param max_leaf_nodes [Integer] The maximum number of leaves on decision tree.
-      #   If nil is given, number of leaves is not limited.
-      # @param min_samples_leaf [Integer] The minimum number of samples at a leaf node.
-      # @param max_features [Integer] The number of features to consider when searching optimal split point.
-      #   If nil is given, split process considers all features.
-      # @param random_seed [Integer] The seed value using to initialize the random generator.
-      #   It is used to randomly determine the order of features when deciding spliting point.
-      def initialize(criterion: 'gini', max_depth: nil, max_leaf_nodes: nil, min_samples_leaf: 1, max_features: nil,
-                     random_seed: nil)
-        SVMKit::Validation.check_params_type_or_nil(Integer, max_depth: max_depth, max_leaf_nodes: max_leaf_nodes,
-                                                             max_features: max_features, random_seed: random_seed)
-        SVMKit::Validation.check_params_integer(min_samples_leaf: min_samples_leaf)
-        SVMKit::Validation.check_params_string(criterion: criterion)
-        SVMKit::Validation.check_params_positive(max_depth: max_depth, max_leaf_nodes: max_leaf_nodes,
-                                                 min_samples_leaf: min_samples_leaf, max_features: max_features)
-        @params = {}
-        @params[:criterion] = criterion
-        @params[:max_depth] = max_depth
-        @params[:max_leaf_nodes] = max_leaf_nodes
-        @params[:min_samples_leaf] = min_samples_leaf
-        @params[:max_features] = max_features
-        @params[:random_seed] = random_seed
-        @params[:random_seed] ||= srand
-        @criterion = :gini
-        @criterion = :entropy if @params[:criterion] == 'entropy'
-        @tree = nil
-        @classes = nil
-        @feature_importances = nil
-        @n_leaves = nil
-        @leaf_labels = nil
-        @rng = Random.new(@params[:random_seed])
-      end
-
-      # Fit the model with given training data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
-      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
-      # @return [DecisionTreeClassifier] The learned classifier itself.
-      def fit(x, y)
-        SVMKit::Validation.check_sample_array(x)
-        SVMKit::Validation.check_label_array(y)
-        SVMKit::Validation.check_sample_label_size(x, y)
-        n_samples, n_features = x.shape
-        @params[:max_features] = n_features if @params[:max_features].nil?
-        @params[:max_features] = [@params[:max_features], n_features].min
-        uniq_y = y.to_a.uniq.sort
-        @classes = Numo::Int32.asarray(uniq_y)
-        build_tree(x, y.map { |v| uniq_y.index(v) })
-        eval_importance(n_samples, n_features)
-        self
-      end
-
-      # Predict class labels for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
-      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
-      def predict(x)
-        SVMKit::Validation.check_sample_array(x)
-        @leaf_labels[apply(x)]
-      end
-
-      # Predict probability for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
-      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
-      def predict_proba(x)
-        SVMKit::Validation.check_sample_array(x)
-        Numo::DFloat[*(Array.new(x.shape[0]) { |n| predict_at_node(@tree, x[n, true]) })]
-      end
-
-      # Return the index of the leaf that each sample reached.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
-      # @return [Numo::Int32] (shape: [n_samples]) Leaf index for sample.
-      def apply(x)
-        SVMKit::Validation.check_sample_array(x)
-        Numo::Int32[*(Array.new(x.shape[0]) { |n| apply_at_node(@tree, x[n, true]) })]
-      end
-
-      # Dump marshal data.
-      # @return [Hash] The marshal data about DecisionTreeClassifier
-      def marshal_dump
-        { params: @params,
-          classes: @classes,
-          criterion: @criterion,
-          tree: @tree,
-          feature_importances: @feature_importances,
-          leaf_labels: @leaf_labels,
-          rng: @rng }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @classes = obj[:classes]
-        @criterion = obj[:criterion]
-        @tree = obj[:tree]
-        @feature_importances = obj[:feature_importances]
-        @leaf_labels = obj[:leaf_labels]
-        @rng = obj[:rng]
-        nil
-      end
-
-      private
-
-      def predict_at_node(node, sample)
-        return node.probs if node.leaf
-        branch_at_node('predict', node, sample)
-      end
-
-      def apply_at_node(node, sample)
-        return node.leaf_id if node.leaf
-        branch_at_node('apply', node, sample)
-      end
-
-      def branch_at_node(action, node, sample)
-        return send("#{action}_at_node", node.left, sample) if node.right.nil?
-        return send("#{action}_at_node", node.right, sample) if node.left.nil?
-        if sample[node.feature_id] <= node.threshold
-          send("#{action}_at_node", node.left, sample)
-        else
-          send("#{action}_at_node", node.right, sample)
-        end
-      end
-
-      def build_tree(x, y)
-        @n_leaves = 0
-        @leaf_labels = []
-        @tree = grow_node(0, x, y, impurity(y))
-        @leaf_labels = Numo::Int32[*@leaf_labels]
-        nil
-      end
-
-      def grow_node(depth, x, y, whole_impurity)
-        unless @params[:max_leaf_nodes].nil?
-          return nil if @n_leaves >= @params[:max_leaf_nodes]
-        end
-
-        n_samples, n_features = x.shape
-        return nil if n_samples <= @params[:min_samples_leaf]
-
-        node = Node.new(depth: depth, impurity: whole_impurity, n_samples: n_samples)
-
-        return put_leaf(node, y) if y.to_a.uniq.size == 1
-
-        unless @params[:max_depth].nil?
-          return put_leaf(node, y) if depth == @params[:max_depth]
-        end
-
-        feature_id, threshold, left_ids, right_ids, left_impurity, right_impurity, gain =
-          rand_ids(n_features).map { |f_id| [f_id, *best_split(x[true, f_id], y, whole_impurity)] }.max_by(&:last)
-
-        return put_leaf(node, y) if gain.nil? || gain.zero?
-
-        node.left = grow_node(depth + 1, x[left_ids, true], y[left_ids], left_impurity)
-        node.right = grow_node(depth + 1, x[right_ids, true], y[right_ids], right_impurity)
-
-        return put_leaf(node, y) if node.left.nil? && node.right.nil?
-
-        node.feature_id = feature_id
-        node.threshold = threshold
-        node.leaf = false
-        node
-      end
-
-      def put_leaf(node, y)
-        node.probs = y.bincount(minlength: @classes.size) / node.n_samples.to_f
-        node.leaf = true
-        node.leaf_id = @n_leaves
-        @n_leaves += 1
-        @leaf_labels.push(@classes[node.probs.max_index])
-        node
-      end
-
-      def rand_ids(n)
-        [*0...n].sample(@params[:max_features], random: @rng)
-      end
-
-      def best_split(features, labels, whole_impurity)
-        n_samples = labels.size
-        features.to_a.uniq.sort.each_cons(2).map do |l, r|
-          threshold = 0.5 * (l + r)
-          left_ids = features.le(threshold).where
-          right_ids = features.gt(threshold).where
-          left_impurity = impurity(labels[left_ids])
-          right_impurity = impurity(labels[right_ids])
-          gain = whole_impurity -
-                 left_impurity * left_ids.size.fdiv(n_samples) -
-                 right_impurity * right_ids.size.fdiv(n_samples)
-          [threshold, left_ids, right_ids, left_impurity, right_impurity, gain]
-        end.max_by(&:last)
-      end
-
-      def impurity(labels)
-        send(@criterion, labels.bincount / labels.size.to_f)
-      end
-
-      def gini(posterior_probs)
-        1.0 - (posterior_probs * posterior_probs).sum
-      end
-
-      def entropy(posterior_probs)
-        -(posterior_probs * Numo::NMath.log(posterior_probs + 1)).sum
-      end
-
-      def eval_importance(n_samples, n_features)
-        @feature_importances = Numo::DFloat.zeros(n_features)
-        eval_importance_at_node(@tree)
-        @feature_importances /= n_samples
-        normalizer = @feature_importances.sum
-        @feature_importances /= normalizer if normalizer > 0.0
-        nil
-      end
-
-      def eval_importance_at_node(node)
-        return nil if node.leaf
-        return nil if node.left.nil? || node.right.nil?
-        gain = node.n_samples * node.impurity -
-               node.left.n_samples * node.left.impurity -
-               node.right.n_samples * node.right.impurity
-        @feature_importances[node.feature_id] += gain
-        eval_importance_at_node(node.left)
-        eval_importance_at_node(node.right)
-      end
-    end
-  end
-end