rumale 0.8.0

data/lib/rumale/dataset.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module Rumale
+  # Module for loading and saving a dataset file.
+  module Dataset
+    class << self
+      # Load a dataset with the libsvm file format into Numo::NArray.
+      #
+      # @param filename [String] A path to a dataset file.
+      # @param zero_based [Boolean] Whether the column index starts from 0 (true) or 1 (false).
+      # @param dtype [Numo::NArray] Data type of Numo::NArray for features to be loaded.
+      #
+      # @return [Array<Numo::NArray>]
+      #   Returns array containing the (n_samples x n_features) matrix for feature vectors
+      #   and (n_samples) vector for labels or target values.
+      def load_libsvm_file(filename, zero_based: false, dtype: Numo::DFloat)
+        ftvecs = []
+        labels = []
+        n_features = 0
+        CSV.foreach(filename, col_sep: "\s", headers: false) do |line|
+          label, ftvec, max_idx = parse_libsvm_line(line, zero_based)
+          labels.push(label)
+          ftvecs.push(ftvec)
+          n_features = max_idx if n_features < max_idx
+        end
+        [convert_to_matrix(ftvecs, n_features, dtype), Numo::NArray.asarray(labels)]
+      end
+
+      # Dump the dataset with the libsvm file format.
+      #
+      # @param data [Numo::NArray] (shape: [n_samples, n_features]) matrix consisting of feature vectors.
+      # @param labels [Numo::NArray] (shape: [n_samples]) matrix consisting of labels or target values.
+      # @param filename [String] A path to the output libsvm file.
+      # @param zero_based [Boolean] Whether the column index starts from 0 (true) or 1 (false).
+      def dump_libsvm_file(data, labels, filename, zero_based: false)
+        n_samples = [data.shape[0], labels.shape[0]].min
+        single_label = labels.shape[1].nil?
+        label_type = detect_dtype(labels)
+        value_type = detect_dtype(data)
+        File.open(filename, 'w') do |file|
+          n_samples.times do |n|
+            label = single_label ? labels[n] : labels[n, true].to_a
+            file.puts(dump_libsvm_line(label, data[n, true],
+                                       label_type, value_type, zero_based))
+          end
+        end
+      end
+
+      private
+
+      def parse_libsvm_line(line, zero_based)
+        label = parse_label(line.shift)
+        adj_idx = zero_based == false ? 1 : 0
+        max_idx = -1
+        ftvec = []
+        while (el = line.shift)
+          idx, val = el.split(':')
+          idx = idx.to_i - adj_idx
+          val = val.to_i.to_s == val ? val.to_i : val.to_f
+          max_idx = idx if max_idx < idx
+          ftvec.push([idx, val])
+        end
+        [label, ftvec, max_idx]
+      end
+
+      def parse_label(label)
+        lbl_arr = label.split(',').map { |lbl| lbl.to_i.to_s == lbl ? lbl.to_i : lbl.to_f }
+        lbl_arr.size > 1 ? lbl_arr : lbl_arr[0]
+      end
+
+      def convert_to_matrix(data, n_features, dtype)
+        mat = []
+        data.each do |ft|
+          vec = Array.new(n_features) { 0 }
+          ft.each { |el| vec[el[0]] = el[1] }
+          mat.push(vec)
+        end
+        dtype.asarray(mat)
+      end
+
+      def detect_dtype(data)
+        arr_type_str = Numo::NArray.array_type(data).to_s
+        type = '%s'
+        type = '%d' if ['Numo::Int8', 'Numo::Int16', 'Numo::Int32', 'Numo::Int64'].include?(arr_type_str)
+        type = '%d' if ['Numo::UInt8', 'Numo::UInt16', 'Numo::UInt32', 'Numo::UInt64'].include?(arr_type_str)
+        type = '%.10g' if ['Numo::SFloat', 'Numo::DFloat'].include?(arr_type_str)
+        type
+      end
+
+      def dump_libsvm_line(label, ftvec, label_type, value_type, zero_based)
+        line = dump_label(label, label_type.to_s)
+        ftvec.to_a.each_with_index do |val, n|
+          idx = n + (zero_based == false ? 1 : 0)
+          line += format(" %d:#{value_type}", idx, val) if val != 0.0
+        end
+        line
+      end
+
+      def dump_label(label, label_type_str)
+        if label.is_a?(Array)
+          label.map { |lbl| format(label_type_str, lbl) }.join(',')
+        else
+          format(label_type_str, label)
+        end
+      end
+    end
+  end
+end

data/lib/rumale/decomposition/nmf.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require 'rumale/utils'
+require 'rumale/base/base_estimator'
+require 'rumale/base/transformer'
+
+module Rumale
+  module Decomposition
+    # NMF is a class that implements Non-negative Matrix Factorization.
+    #
+    # @example
+    #   decomposer = Rumale::Decomposition::NMF.new(n_components: 2)
+    #   representaion = decomposer.fit_transform(samples)
+    #
+    # *Reference*
+    # - W. Xu, X. Liu, and Y.Gong, "Document Clustering Based On Non-negative Matrix Factorization," Proc. SIGIR' 03 , pp. 267--273, 2003.
+    class NMF
+      include Base::BaseEstimator
+      include Base::Transformer
+
+      # Returns the factorization matrix.
+      # @return [Numo::DFloat] (shape: [n_components, n_features])
+      attr_reader :components
+
+      # Return the random generator.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new transformer with NMF.
+      #
+      # @param n_components [Integer] The number of components.
+      # @param max_iter [Integer] The maximum number of iterations.
+      # @param tol [Float] The tolerance of termination criterion.
+      # @param eps [Float] A small value close to zero to avoid zero division error.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(n_components: 2, max_iter: 500, tol: 1.0e-4, eps: 1.0e-16, random_seed: nil)
+        check_params_integer(n_components: n_components, max_iter: max_iter)
+        check_params_float(tol: tol, eps: eps)
+        check_params_type_or_nil(Integer, random_seed: random_seed)
+        check_params_positive(n_components: n_components, max_iter: max_iter, tol: tol, eps: eps)
+        @params = {}
+        @params[:n_components] = n_components
+        @params[:max_iter] = max_iter
+        @params[:tol] = tol
+        @params[:eps] = eps
+        @params[:random_seed] = random_seed
+        @params[:random_seed] ||= srand
+        @components = nil
+        @rng = Random.new(@params[:random_seed])
+      end
+
+      # Fit the model with given training data.
+      #
+      # @overload fit(x) -> NMF
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @return [NMF] The learned transformer itself.
+      def fit(x, _y = nil)
+        check_sample_array(x)
+        partial_fit(x)
+        self
+      end
+
+      # Fit the model with training data, and then transform them with the learned model.
+      #
+      # @overload fit_transform(x) -> Numo::DFloat
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The transformed data
+      def fit_transform(x, _y = nil)
+        check_sample_array(x)
+        partial_fit(x)
+      end
+
+      # Transform the given data with the learned model.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The data to be transformed with the learned model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The transformed data.
+      def transform(x)
+        check_sample_array(x)
+        partial_fit(x, false)
+      end
+
+      # Inverse transform the given transformed data with the learned model.
+      #
+      # @param z [Numo::DFloat] (shape: [n_samples, n_components]) The data to be restored into original space with the learned model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_featuress]) The restored data.
+      def inverse_transform(z)
+        check_sample_array(z)
+        z.dot(@components)
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data.
+      def marshal_dump
+        { params: @params,
+          components: @components,
+          rng: @rng }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @components = obj[:components]
+        @rng = obj[:rng]
+        nil
+      end
+
+      private
+
+      def partial_fit(x, update_comps = true)
+        # initialize some variables.
+        n_samples, n_features = x.shape
+        scale = Math.sqrt(x.mean / @params[:n_components])
+        @components = Rumale::Utils.rand_uniform([@params[:n_components], n_features], @rng) * scale if update_comps
+        coefficients = Rumale::Utils.rand_uniform([n_samples, @params[:n_components]], @rng) * scale
+        # optimization.
+        @params[:max_iter].times do
+          # update
+          if update_comps
+            nume = coefficients.transpose.dot(x)
+            deno = coefficients.transpose.dot(coefficients).dot(@components) + @params[:eps]
+            @components *= (nume / deno)
+          end
+          nume = x.dot(@components.transpose)
+          deno = coefficients.dot(@components).dot(@components.transpose) + @params[:eps]
+          coefficients *= (nume / deno)
+          # normalize
+          norm = Numo::NMath.sqrt((@components**2).sum(1)) + @params[:eps]
+          @components /= norm.expand_dims(1) if update_comps
+          coefficients *= norm
+          # check convergence
+          err = ((x - coefficients.dot(@components))**2).sum(1).mean
+          break if err < @params[:tol]
+        end
+        coefficients
+      end
+    end
+  end
+end

data/lib/rumale/decomposition/pca.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require 'rumale/base/base_estimator'
+require 'rumale/base/transformer'
+
+module Rumale
+  # Module for matrix decomposition algorithms.
+  module Decomposition
+    # PCA is a class that implements Principal Component Analysis.
+    #
+    # @example
+    #   decomposer = Rumale::Decomposition::PCA.new(n_components: 2)
+    #   representaion = decomposer.fit_transform(samples)
+    #
+    # *Reference*
+    # - A. Sharma and K K. Paliwal, "Fast principal component analysis using fixed-point algorithm," Pattern Recognition Letters, 28, pp. 1151--1155, 2007.
+    class PCA
+      include Base::BaseEstimator
+      include Base::Transformer
+
+      # Returns the principal components.
+      # @return [Numo::DFloat] (shape: [n_components, n_features])
+      attr_reader :components
+
+      # Returns the mean vector.
+      # @return [Numo::DFloat] (shape: [n_features]
+      attr_reader :mean
+
+      # Return the random generator.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new transformer with PCA.
+      #
+      # @param n_components [Integer] The number of principal components.
+      # @param max_iter [Integer] The maximum number of iterations.
+      # @param tol [Float] The tolerance of termination criterion.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      def initialize(n_components: 2, max_iter: 100, tol: 1.0e-4, random_seed: nil)
+        check_params_integer(n_components: n_components, max_iter: max_iter)
+        check_params_float(tol: tol)
+        check_params_type_or_nil(Integer, random_seed: random_seed)
+        check_params_positive(n_components: n_components, max_iter: max_iter, tol: tol)
+        @params = {}
+        @params[:n_components] = n_components
+        @params[:max_iter] = max_iter
+        @params[:tol] = tol
+        @params[:random_seed] = random_seed
+        @params[:random_seed] ||= srand
+        @components = nil
+        @mean = nil
+        @rng = Random.new(@params[:random_seed])
+      end
+
+      # Fit the model with given training data.
+      #
+      # @overload fit(x) -> PCA
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @return [PCA] The learned transformer itself.
+      def fit(x, _y = nil)
+        check_sample_array(x)
+        # initialize some variables.
+        @components = nil
+        n_samples, n_features = x.shape
+        # centering.
+        @mean = x.mean(0)
+        centered_x = x - @mean
+        # optimization.
+        covariance_mat = centered_x.transpose.dot(centered_x) / (n_samples - 1)
+        @params[:n_components].times do
+          comp_vec = random_vec(n_features)
+          @params[:max_iter].times do
+            updated = orthogonalize(covariance_mat.dot(comp_vec))
+            break if (updated.dot(comp_vec) - 1).abs < @params[:tol]
+            comp_vec = updated
+          end
+          @components = @components.nil? ? comp_vec : Numo::NArray.vstack([@components, comp_vec])
+        end
+        self
+      end
+
+      # Fit the model with training data, and then transform them with the learned model.
+      #
+      # @overload fit_transform(x) -> Numo::DFloat
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The transformed data
+      def fit_transform(x, _y = nil)
+        check_sample_array(x)
+        fit(x).transform(x)
+      end
+
+      # Transform the given data with the learned model.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The data to be transformed with the learned model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_components]) The transformed data.
+      def transform(x)
+        check_sample_array(x)
+        (x - @mean).dot(@components.transpose)
+      end
+
+      # Inverse transform the given transformed data with the learned model.
+      #
+      # @param z [Numo::DFloat] (shape: [n_samples, n_components]) The data to be restored into original space with the learned model.
+      # @return [Numo::DFloat] (shape: [n_samples, n_featuress]) The restored data.
+      def inverse_transform(z)
+        check_sample_array(z)
+        c = @components.shape[1].nil? ? @components.expand_dims(0) : @components
+        z.dot(c) + @mean
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data.
+      def marshal_dump
+        { params: @params,
+          components: @components,
+          mean: @mean,
+          rng: @rng }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @components = obj[:components]
+        @mean = obj[:mean]
+        @rng = obj[:rng]
+        nil
+      end
+
+      private
+
+      def orthogonalize(pcvec)
+        unless @components.nil?
+          delta = @components.dot(pcvec) * @components.transpose
+          delta = delta.sum(1) unless delta.shape[1].nil?
+          pcvec -= delta
+        end
+        pcvec / Math.sqrt((pcvec**2).sum.abs) + 1.0e-12
+      end
+
+      def random_vec(n_features)
+        Numo::DFloat[*(Array.new(n_features) { @rng.rand })]
+      end
+    end
+  end
+end

data/lib/rumale/ensemble/ada_boost_classifier.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+require 'rumale/values'
+require 'rumale/utils'
+require 'rumale/base/base_estimator'
+require 'rumale/base/classifier'
+require 'rumale/tree/decision_tree_classifier'
+
+module Rumale
+  module Ensemble
+    # AdaBoostClassifier is a class that implements AdaBoost (SAMME.R) for classification.
+    # This class uses decision tree for a weak learner.
+    #
+    # @example
+    #   estimator =
+    #     Rumale::Ensemble::AdaBoostClassifier.new(
+    #       n_estimators: 10, criterion: 'gini', max_depth: 3, max_leaf_nodes: 10, min_samples_leaf: 5, random_seed: 1)
+    #   estimator.fit(training_samples, traininig_labels)
+    #   results = estimator.predict(testing_samples)
+    #
+    # *Reference*
+    # - J. Zhu, S. Rosset, H. Zou, and T.Hashie, "Multi-class AdaBoost," Technical Report No. 430, Department of Statistics, University of Michigan, 2005.
+    class AdaBoostClassifier
+      include Base::BaseEstimator
+      include Base::Classifier
+
+      # Return the set of estimators.
+      # @return [Array<DecisionTreeClassifier>]
+      attr_reader :estimators
+
+      # Return the class labels.
+      # @return [Numo::Int32] (size: n_classes)
+      attr_reader :classes
+
+      # Return the importance for each feature.
+      # @return [Numo::DFloat] (size: n_features)
+      attr_reader :feature_importances
+
+      # Return the random generator for random selection of feature index.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new classifier with AdaBoost.
+      #
+      # @param n_estimators [Integer] The numeber of decision trees for contructing random forest.
+      # @param criterion [String] The function to evalue spliting point. Supported criteria are 'gini' and 'entropy'.
+      # @param max_depth [Integer] The maximum depth of the tree.
+      #   If nil is given, decision tree grows without concern for depth.
+      # @param max_leaf_nodes [Integer] The maximum number of leaves on decision tree.
+      #   If nil is given, number of leaves is not limited.
+      # @param min_samples_leaf [Integer] The minimum number of samples at a leaf node.
+      # @param max_features [Integer] The number of features to consider when searching optimal split point.
+      #   If nil is given, split process considers all features.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      #   It is used to randomly determine the order of features when deciding spliting point.
+      def initialize(n_estimators: 50,
+                     criterion: 'gini', max_depth: nil, max_leaf_nodes: nil, min_samples_leaf: 1,
+                     max_features: nil, random_seed: nil)
+        check_params_type_or_nil(Integer, max_depth: max_depth, max_leaf_nodes: max_leaf_nodes,
+                                          max_features: max_features, random_seed: random_seed)
+        check_params_integer(n_estimators: n_estimators, min_samples_leaf: min_samples_leaf)
+        check_params_string(criterion: criterion)
+        check_params_positive(n_estimators: n_estimators, max_depth: max_depth,
+                              max_leaf_nodes: max_leaf_nodes, min_samples_leaf: min_samples_leaf,
+                              max_features: max_features)
+        @params = {}
+        @params[:n_estimators] = n_estimators
+        @params[:criterion] = criterion
+        @params[:max_depth] = max_depth
+        @params[:max_leaf_nodes] = max_leaf_nodes
+        @params[:min_samples_leaf] = min_samples_leaf
+        @params[:max_features] = max_features
+        @params[:random_seed] = random_seed
+        @params[:random_seed] ||= srand
+        @estimators = nil
+        @classes = nil
+        @feature_importances = nil
+        @rng = Random.new(@params[:random_seed])
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [AdaBoostClassifier] The learned classifier itself.
+      def fit(x, y) # rubocop:disable Metrics/AbcSize
+        check_sample_array(x)
+        check_label_array(y)
+        check_sample_label_size(x, y)
+        ## Initialize some variables.
+        n_samples, n_features = x.shape
+        @estimators = []
+        @feature_importances = Numo::DFloat.zeros(n_features)
+        @params[:max_features] = n_features unless @params[:max_features].is_a?(Integer)
+        @params[:max_features] = [[1, @params[:max_features]].max, n_features].min
+        @classes = Numo::Int32.asarray(y.to_a.uniq.sort)
+        n_classes = @classes.shape[0]
+        ## Boosting.
+        classes_arr = @classes.to_a
+        y_codes = Numo::DFloat.zeros(n_samples, n_classes) - 1.fdiv(n_classes - 1)
+        n_samples.times { |n| y_codes[n, classes_arr.index(y[n])] = 1.0 }
+        observation_weights = Numo::DFloat.zeros(n_samples) + 1.fdiv(n_samples)
+        @params[:n_estimators].times do |_t|
+          # Fit classfier.
+          ids = Rumale::Utils.choice_ids(n_samples, observation_weights, @rng)
+          break if y[ids].to_a.uniq.size != n_classes
+          tree = Tree::DecisionTreeClassifier.new(
+            criterion: @params[:criterion], max_depth: @params[:max_depth],
+            max_leaf_nodes: @params[:max_leaf_nodes], min_samples_leaf: @params[:min_samples_leaf],
+            max_features: @params[:max_features], random_seed: @rng.rand(Rumale::Values.int_max)
+          )
+          tree.fit(x[ids, true], y[ids])
+          # Calculate estimator error.
+          proba = tree.predict_proba(x).clip(1.0e-15, nil)
+          p = Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[proba[n, true].max_index] })
+          inds = p.ne(y)
+          error = (observation_weights * inds).sum / observation_weights.sum
+          # Store model.
+          @estimators.push(tree)
+          @feature_importances += tree.feature_importances
+          break if error.zero?
+          # Update observation weights.
+          log_proba = Numo::NMath.log(proba)
+          observation_weights *= Numo::NMath.exp(-1.0 * (n_classes - 1).fdiv(n_classes) * (y_codes * log_proba).sum(1))
+          observation_weights = observation_weights.clip(1.0e-15, nil)
+          sum_observation_weights = observation_weights.sum
+          break if sum_observation_weights.zero?
+          observation_weights /= sum_observation_weights
+        end
+        @feature_importances /= @feature_importances.sum
+        self
+      end
+
+      # Calculate confidence scores for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to compute the scores.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Confidence score per sample.
+      def decision_function(x)
+        check_sample_array(x)
+        n_samples, = x.shape
+        n_classes = @classes.size
+        sum_probs = Numo::DFloat.zeros(n_samples, n_classes)
+        @estimators.each do |tree|
+          log_proba = Numo::NMath.log(tree.predict_proba(x).clip(1.0e-15, nil))
+          sum_probs += (n_classes - 1) * (log_proba - 1.fdiv(n_classes) * Numo::DFloat[log_proba.sum(1)].transpose)
+        end
+        sum_probs /= @estimators.size
+      end
+
+      # Predict class labels for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
+      def predict(x)
+        check_sample_array(x)
+        n_samples, = x.shape
+        probs = decision_function(x)
+        Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[probs[n, true].max_index] })
+      end
+
+      # Predict probability for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
+      def predict_proba(x)
+        check_sample_array(x)
+        n_classes = @classes.size
+        probs = Numo::NMath.exp(1.fdiv(n_classes - 1) * decision_function(x))
+        sum_probs = probs.sum(1)
+        probs /= Numo::DFloat[sum_probs].transpose
+        probs
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data about AdaBoostClassifier.
+      def marshal_dump
+        { params: @params,
+          estimators: @estimators,
+          classes: @classes,
+          feature_importances: @feature_importances,
+          rng: @rng }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @estimators = obj[:estimators]
+        @classes = obj[:classes]
+        @feature_importances = obj[:feature_importances]
+        @rng = obj[:rng]
+        nil
+      end
+    end
+  end
+end