svmkit 0.7.3 → 0.8.1

data/lib/svmkit/model_selection/grid_search_cv.rb

@@ -1,247 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/base/base_estimator'
-require 'svmkit/base/evaluator'
-require 'svmkit/base/splitter'
-require 'svmkit/pipeline/pipeline'
-
-module SVMKit
-  module ModelSelection
-    # GridSearchCV is a class that performs hyperparameter optimization with grid search method.
-    #
-    # @example
-    #   rfc = SVMKit::Ensemble::RandomForestClassifier.new(random_seed: 1)
-    #   pg = { n_estimators: [5, 10], max_depth: [3, 5], max_leaf_nodes: [15, 31] }
-    #   kf = SVMKit::ModelSelection::StratifiedKFold.new(n_splits: 5)
-    #   gs = SVMKit::ModelSelection::GridSearchCV.new(estimator: rfc, param_grid: pg, splitter: kf)
-    #   gs.fit(samples, labels)
-    #   p gs.cv_results
-    #   p gs.best_params
-    #
-    # @example
-    #   rbf = SVMKit::KernelApproximation::RBF.new(random_seed: 1)
-    #   svc = SVMKit::LinearModel::SVC.new(random_seed: 1)
-    #   pipe = SVMKit::Pipeline::Pipeline.new(steps: { rbf: rbf, svc: svc })
-    #   pg = { rbf__gamma: [32.0, 1.0], rbf__n_components: [4, 128], svc__reg_param: [16.0, 0.1] }
-    #   kf = SVMKit::ModelSelection::StratifiedKFold.new(n_splits: 5)
-    #   gs = SVMKit::ModelSelection::GridSearchCV.new(estimator: pipe, param_grid: pg, splitter: kf)
-    #   gs.fit(samples, labels)
-    #   p gs.cv_results
-    #   p gs.best_params
-    #
-    class GridSearchCV
-      include Base::BaseEstimator
-      include Validation
-
-      # Return the result of cross validation for each parameter.
-      # @return [Hash]
-      attr_reader :cv_results
-
-      # Return the score of the estimator learned with the best parameter.
-      # @return [Float]
-      attr_reader :best_score
-
-      # Return the best parameter set.
-      # @return [Hash]
-      attr_reader :best_params
-
-      # Return the index of the best parameter.
-      # @return [Integer]
-      attr_reader :best_index
-
-      # Return the estimator learned with the best parameter.
-      # @return [Estimator]
-      attr_reader :best_estimator
-
-      # Create a new grid search method.
-      #
-      # @param estimator [Classifier/Regresor] The estimator to be searched for optimal parameters with grid search method.
-      # @param param_grid [Array<Hash>] The parameter sets is represented with array of hash that
-      #   consists of parameter names as keys and array of parameter values as values.
-      # @param splitter [Splitter] The splitter that divides dataset to training and testing dataset on cross validation.
-      # @param evaluator [Evaluator] The evaluator that calculates score of estimator results on cross validation.
-      #   If nil is given, the score method of estimator is used to evaluation.
-      # @param greater_is_better [Boolean] The flag that indicates whether the estimator is better as
-      #   evaluation score is larger.
-      def initialize(estimator: nil, param_grid: nil, splitter: nil, evaluator: nil, greater_is_better: true)
-        check_params_type(SVMKit::Base::BaseEstimator, estimator: estimator)
-        check_params_type(SVMKit::Base::Splitter, splitter: splitter)
-        check_params_type_or_nil(SVMKit::Base::Evaluator, evaluator: evaluator)
-        check_params_boolean(greater_is_better: greater_is_better)
-        @params = {}
-        @params[:param_grid] = valid_param_grid(param_grid)
-        @params[:estimator] = Marshal.load(Marshal.dump(estimator))
-        @params[:splitter] = Marshal.load(Marshal.dump(splitter))
-        @params[:evaluator] = Marshal.load(Marshal.dump(evaluator))
-        @params[:greater_is_better] = greater_is_better
-        @cv_results = nil
-        @best_score = nil
-        @best_params = nil
-        @best_index = nil
-        @best_estimator = nil
-      end
-
-      # Fit the model with given training data and all sets of parameters.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
-      # @param y [Numo::NArray] (shape: [n_samples, n_outputs]) The target values or labels to be used for fitting the model.
-      # @return [GridSearchCV] The learned estimator with grid search.
-      def fit(x, y)
-        check_sample_array(x)
-
-        init_attrs
-
-        param_combinations.each do |prm_set|
-          prm_set.each do |prms|
-            report = perform_cross_validation(x, y, prms)
-            store_cv_result(prms, report)
-          end
-        end
-
-        find_best_params
-
-        @best_estimator = configurated_estimator(@best_params)
-        @best_estimator.fit(x, y)
-        self
-      end
-
-      # Call the decision_function method of learned estimator with the best parameter.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
-      # @return [Numo::DFloat] (shape: [n_samples]) Confidence score per sample.
-      def decision_function(x)
-        check_sample_array(x)
-        @best_estimator.decision_function(x)
-      end
-
-      # Call the predict method of learned estimator with the best parameter.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to obtain prediction result.
-      # @return [Numo::NArray] Predicted results.
-      def predict(x)
-        check_sample_array(x)
-        @best_estimator.predict(x)
-      end
-
-      # Call the predict_log_proba method of learned estimator with the best parameter.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the log-probailities.
-      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted log-probability of each class per sample.
-      def predict_log_proba(x)
-        check_sample_array(x)
-        @best_estimator.predict_log_proba(x)
-      end
-
-      # Call the predict_proba method of learned estimator with the best parameter.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
-      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
-      def predict_proba(x)
-        check_sample_array(x)
-        @best_estimator.predict_proba(x)
-      end
-
-      # Call the score method of learned estimator with the best parameter.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) Testing data.
-      # @param y [Numo::NArray] (shape: [n_samples, n_outputs]) True target values or labels for testing data.
-      # @return [Float] The score of estimator.
-      def score(x, y)
-        check_sample_array(x)
-        @best_estimator.score(x, y)
-      end
-
-      # Dump marshal data.
-      # @return [Hash] The marshal data about GridSearchCV.
-      def marshal_dump
-        { params: @params,
-          cv_results: @cv_results,
-          best_score: @best_score,
-          best_params: @best_params,
-          best_index: @best_index,
-          best_estimator: @best_estimator }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @cv_results = obj[:cv_results]
-        @best_score = obj[:best_score]
-        @best_params = obj[:best_params]
-        @best_index = obj[:best_index]
-        @best_estimator = obj[:best_estimator]
-        nil
-      end
-
-      private
-
-      def valid_param_grid(grid)
-        raise TypeError, 'Expect class of param_grid to be Hash or Array' unless grid.is_a?(Hash) || grid.is_a?(Array)
-        grid = [grid] if grid.is_a?(Hash)
-        grid.each do |h|
-          raise TypeError, 'Expect class of elements in param_grid to be Hash' unless h.is_a?(Hash)
-          raise TypeError, 'Expect class of parameter values in param_grid to be Array' unless h.values.all? { |v| v.is_a?(Array) }
-        end
-        grid
-      end
-
-      def param_combinations
-        @param_combinations ||= @params[:param_grid].map do |prm|
-          x = Hash[prm.sort].map { |k, v| [k].product(v) }
-          x[0].product(*x[1...x.size]).map { |v| Hash[v] }
-        end
-      end
-
-      def perform_cross_validation(x, y, prms)
-        est = configurated_estimator(prms)
-        cv = CrossValidation.new(estimator: est, splitter: @params[:splitter],
-                                 evaluator: @params[:evaluator], return_train_score: true)
-        cv.perform(x, y)
-      end
-
-      def configurated_estimator(prms)
-        estimator = Marshal.load(Marshal.dump(@params[:estimator]))
-        if @params[:estimator].is_a?(SVMKit::Pipeline::Pipeline)
-          prms.each do |k, v|
-            est_name, prm_name = k.to_s.split('__')
-            estimator.steps[est_name.to_sym].params[prm_name.to_sym] = v
-          end
-        else
-          prms.each { |k, v| estimator.params[k] = v }
-        end
-        estimator
-      end
-
-      def init_attrs
-        @cv_results = %i[mean_test_score std_test_score
-                         mean_train_score std_train_score
-                         mean_fit_time std_fit_time params].map { |v| [v, []] }.to_h
-        @best_score = nil
-        @best_params = nil
-        @best_index = nil
-        @best_estimator = nil
-      end
-
-      def store_cv_result(prms, report)
-        test_scores = Numo::DFloat[*report[:test_score]]
-        train_scores = Numo::DFloat[*report[:train_score]]
-        fit_times = Numo::DFloat[*report[:fit_time]]
-        @cv_results[:mean_test_score].push(test_scores.mean)
-        @cv_results[:std_test_score].push(test_scores.stddev)
-        @cv_results[:mean_train_score].push(train_scores.mean)
-        @cv_results[:std_train_score].push(train_scores.stddev)
-        @cv_results[:mean_fit_time].push(fit_times.mean)
-        @cv_results[:std_fit_time].push(fit_times.stddev)
-        @cv_results[:params].push(prms)
-      end
-
-      def find_best_params
-        @best_score = @params[:greater_is_better] ? @cv_results[:mean_test_score].max : @cv_results[:mean_test_score].min
-        @best_index = @cv_results[:mean_test_score].index(@best_score)
-        @best_params = @cv_results[:params][@best_index]
-      end
-    end
-  end
-end

data/lib/svmkit/model_selection/k_fold.rb

@@ -1,77 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/base/splitter'
-
-module SVMKit
-  # This module consists of the classes for model validation techniques.
-  module ModelSelection
-    # KFold is a class that generates the set of data indices for K-fold cross-validation.
-    #
-    # @example
-    #   kf = SVMKit::ModelSelection::KFold.new(n_splits: 3, shuffle: true, random_seed: 1)
-    #   kf.split(samples, labels).each do |train_ids, test_ids|
-    #     train_samples = samples[train_ids, true]
-    #     test_samples = samples[test_ids, true]
-    #     ...
-    #   end
-    #
-    class KFold
-      include Base::Splitter
-
-      # Return the flag indicating whether to shuffle the dataset.
-      # @return [Boolean]
-      attr_reader :shuffle
-
-      # Return the random generator for shuffling the dataset.
-      # @return [Random]
-      attr_reader :rng
-
-      # Create a new data splitter for K-fold cross validation.
-      #
-      # @param n_splits [Integer] The number of folds.
-      # @param shuffle [Boolean] The flag indicating whether to shuffle the dataset.
-      # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(n_splits: 3, shuffle: false, random_seed: nil)
-        SVMKit::Validation.check_params_integer(n_splits: n_splits)
-        SVMKit::Validation.check_params_boolean(shuffle: shuffle)
-        SVMKit::Validation.check_params_type_or_nil(Integer, random_seed: random_seed)
-        SVMKit::Validation.check_params_positive(n_splits: n_splits)
-        @n_splits = n_splits
-        @shuffle = shuffle
-        @random_seed = random_seed
-        @random_seed ||= srand
-        @rng = Random.new(@random_seed)
-      end
-
-      # Generate data indices for K-fold cross validation.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features])
-      #   The dataset to be used to generate data indices for K-fold cross validation.
-      # @return [Array] The set of data indices for constructing the training and testing dataset in each fold.
-      def split(x, _y = nil)
-        SVMKit::Validation.check_sample_array(x)
-        # Initialize and check some variables.
-        n_samples, = x.shape
-        unless @n_splits.between?(2, n_samples)
-          raise ArgumentError,
-                'The value of n_splits must be not less than 2 and not more than the number of samples.'
-        end
-        # Splits dataset ids to each fold.
-        dataset_ids = [*0...n_samples]
-        dataset_ids.shuffle!(random: @rng) if @shuffle
-        fold_sets = Array.new(@n_splits) do |n|
-          n_fold_samples = n_samples / @n_splits
-          n_fold_samples += 1 if n < n_samples % @n_splits
-          dataset_ids.shift(n_fold_samples)
-        end
-        # Returns array consisting of the training and testing ids for each fold.
-        Array.new(@n_splits) do |n|
-          train_ids = fold_sets.select.with_index { |_, id| id != n }.flatten
-          test_ids = fold_sets[n]
-          [train_ids, test_ids]
-        end
-      end
-    end
-  end
-end

data/lib/svmkit/model_selection/stratified_k_fold.rb

@@ -1,95 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/base/splitter'
-
-module SVMKit
-  module ModelSelection
-    # StratifiedKFold is a class that generates the set of data indices for K-fold cross-validation.
-    # The proportion of the number of samples in each class will be almost equal for each fold.
-    #
-    # @example
-    #   kf = SVMKit::ModelSelection::StratifiedKFold.new(n_splits: 3, shuffle: true, random_seed: 1)
-    #   kf.split(samples, labels).each do |train_ids, test_ids|
-    #     train_samples = samples[train_ids, true]
-    #     test_samples = samples[test_ids, true]
-    #     ...
-    #   end
-    #
-    class StratifiedKFold
-      include Base::Splitter
-
-      # Return the flag indicating whether to shuffle the dataset.
-      # @return [Boolean]
-      attr_reader :shuffle
-
-      # Return the random generator for shuffling the dataset.
-      # @return [Random]
-      attr_reader :rng
-
-      # Create a new data splitter for K-fold cross validation.
-      #
-      # @param n_splits [Integer] The number of folds.
-      # @param shuffle [Boolean] The flag indicating whether to shuffle the dataset.
-      # @param random_seed [Integer] The seed value using to initialize the random generator.
-      def initialize(n_splits: 3, shuffle: false, random_seed: nil)
-        SVMKit::Validation.check_params_integer(n_splits: n_splits)
-        SVMKit::Validation.check_params_boolean(shuffle: shuffle)
-        SVMKit::Validation.check_params_type_or_nil(Integer, random_seed: random_seed)
-        SVMKit::Validation.check_params_positive(n_splits: n_splits)
-        @n_splits = n_splits
-        @shuffle = shuffle
-        @random_seed = random_seed
-        @random_seed ||= srand
-        @rng = Random.new(@random_seed)
-      end
-
-      # Generate data indices for stratified K-fold cross validation.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features])
-      #   The dataset to be used to generate data indices for stratified K-fold cross validation.
-      #   This argument exists to unify the interface between the K-fold methods, it is not used in the method.
-      # @param y [Numo::Int32] (shape: [n_samples])
-      #   The labels to be used to generate data indices for stratified K-fold cross validation.
-      # @return [Array] The set of data indices for constructing the training and testing dataset in each fold.
-      def split(x, y)
-        SVMKit::Validation.check_sample_array(x)
-        SVMKit::Validation.check_label_array(y)
-        SVMKit::Validation.check_sample_label_size(x, y)
-        # Check the number of samples in each class.
-        unless valid_n_splits?(y)
-          raise ArgumentError,
-                'The value of n_splits must be not less than 2 and not more than the number of samples in each class.'
-        end
-        # Splits dataset ids of each class to each fold.
-        fold_sets_each_class = y.to_a.uniq.map { |label| fold_sets(y, label) }
-        # Returns array consisting of the training and testing ids for each fold.
-        Array.new(@n_splits) { |fold_id| train_test_sets(fold_sets_each_class, fold_id) }
-      end
-
-      private
-
-      def valid_n_splits?(y)
-        y.to_a.uniq.map { |label| y.eq(label).where.size }.all? { |n_samples| @n_splits.between?(2, n_samples) }
-      end
-
-      def fold_sets(y, label)
-        sample_ids = y.eq(label).where.to_a
-        sample_ids.shuffle!(random: @rng) if @shuffle
-        n_samples = sample_ids.size
-        Array.new(@n_splits) do |n|
-          n_fold_samples = n_samples / @n_splits
-          n_fold_samples += 1 if n < n_samples % @n_splits
-          sample_ids.shift(n_fold_samples)
-        end
-      end
-
-      def train_test_sets(fold_sets_each_class, fold_id)
-        train_test_sets_each_class = fold_sets_each_class.map do |folds|
-          folds.partition.with_index { |_, id| id != fold_id }.map(&:flatten)
-        end
-        train_test_sets_each_class.transpose.map(&:flatten)
-      end
-    end
-  end
-end

data/lib/svmkit/multiclass/one_vs_rest_classifier.rb

@@ -1,101 +0,0 @@
-# frozen_string_literal: true
-
-require 'svmkit/validation'
-require 'svmkit/base/base_estimator.rb'
-require 'svmkit/base/classifier.rb'
-
-module SVMKit
-  # This module consists of the classes that implement multi-class classification strategy.
-  module Multiclass
-    # @note
-    #   All classifier in SVMKit support multi-class classifiction since version 0.2.7.
-    #   There is no need to explicitly use this class for multiclass classifiction.
-    #
-    # OneVsRestClassifier is a class that implements One-vs-Rest (OvR) strategy for multi-class classification.
-    #
-    # @example
-    #   base_estimator = SVMKit::LinearModel::LogisticRegression.new
-    #   estimator = SVMKit::Multiclass::OneVsRestClassifier.new(estimator: base_estimator)
-    #   estimator.fit(training_samples, training_labels)
-    #   results = estimator.predict(testing_samples)
-    class OneVsRestClassifier
-      include Base::BaseEstimator
-      include Base::Classifier
-
-      # Return the set of estimators.
-      # @return [Array<Classifier>]
-      attr_reader :estimators
-
-      # Return the class labels.
-      # @return [Numo::Int32] (shape: [n_classes])
-      attr_reader :classes
-
-      # Create a new multi-class classifier with the one-vs-rest startegy.
-      #
-      # @param estimator [Classifier] The (binary) classifier for construction a multi-class classifier.
-      def initialize(estimator: nil)
-        SVMKit::Validation.check_params_type(SVMKit::Base::BaseEstimator, estimator: estimator)
-        @params = {}
-        @params[:estimator] = estimator
-        @estimators = nil
-        @classes = nil
-      end
-
-      # Fit the model with given training data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
-      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
-      # @return [OneVsRestClassifier] The learned classifier itself.
-      def fit(x, y)
-        SVMKit::Validation.check_sample_array(x)
-        SVMKit::Validation.check_label_array(y)
-        SVMKit::Validation.check_sample_label_size(x, y)
-        y_arr = y.to_a
-        @classes = Numo::Int32.asarray(y_arr.uniq.sort)
-        @estimators = @classes.to_a.map do |label|
-          bin_y = Numo::Int32.asarray(y_arr.map { |l| l == label ? 1 : -1 })
-          @params[:estimator].dup.fit(x, bin_y)
-        end
-        self
-      end
-
-      # Calculate confidence scores for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
-      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence scores per sample for each class.
-      def decision_function(x)
-        SVMKit::Validation.check_sample_array(x)
-        n_classes = @classes.size
-        Numo::DFloat.asarray(Array.new(n_classes) { |m| @estimators[m].decision_function(x).to_a }).transpose
-      end
-
-      # Predict class labels for samples.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
-      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
-      def predict(x)
-        SVMKit::Validation.check_sample_array(x)
-        n_samples, = x.shape
-        decision_values = decision_function(x)
-        Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })
-      end
-
-      # Dump marshal data.
-      # @return [Hash] The marshal data about OneVsRestClassifier.
-      def marshal_dump
-        { params: @params,
-          classes: @classes,
-          estimators: @estimators.map { |e| Marshal.dump(e) } }
-      end
-
-      # Load marshal data.
-      # @return [nil]
-      def marshal_load(obj)
-        @params = obj[:params]
-        @classes = obj[:classes]
-        @estimators = obj[:estimators].map { |e| Marshal.load(e) }
-        nil
-      end
-    end
-  end
-end