svmkit 0.2.5 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 42b8bbee820defc7646b422fa160ade5dd0ffddd
-  data.tar.gz: 5ba44c2c18a02231646456ab1ff73fee409c50c0
+  metadata.gz: b36f6b299c47d1107d587aafeb7bb66531f1208c
+  data.tar.gz: 1bd382f3339b8fb08454493a45a2338020791b6c
 SHA512:
-  metadata.gz: 23bdab14e55581e61b9050c88167dd74ebd5c086dfa2c37f57aaebe2110e8d7bac70df659ae34615c788d36cd03fac36663267ed07cf950564da7b1e496e9b59
-  data.tar.gz: c36b3565bd731e1613ee63f83ad83f2021e8c2c255267df00a0872ec8f1d9f2e9ab73edd10d5f2a5bef6ca1af8712aef4220434bc680f1f5228bf9727c94b6e4
+  metadata.gz: 0676f2e9b3ef4ac9786f10ca976721e73d2cd918a9c939900281e36267ab14a3413c3a719d9504415f527e5d6163d5640ea2af186e023c3980327cb7c476afba
+  data.tar.gz: e8dcf72f7d1641903a4625bb23399deabf9a19931a5c00bd2c5077b525a2d0361b194b8c43dee1b7a365b25eafdabb460e0a4ae21f17afd5b401379671379463

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/HISTORY.md

@@ -1,3 +1,10 @@
+# 0.2.6
+- Added class for Decision Tree classifier.
+- Added class for Random Forest classifier.
+- Fixed to use frozen string literal.
+- Refactored marshal dump method on some classes.
+- Introduced Coveralls to confirm test coverage.
+
 # 0.2.5
 - Added classes for Naive Bayes classifier.
 - Fixed decision function method on Logistic Regression class.

data/README.md

@@ -1,13 +1,14 @@
 # SVMKit
 
 [![Build Status](https://travis-ci.org/yoshoku/SVMKit.svg?branch=master)](https://travis-ci.org/yoshoku/SVMKit)
+[![Coverage Status](https://coveralls.io/repos/github/yoshoku/SVMKit/badge.svg?branch=master)](https://coveralls.io/github/yoshoku/SVMKit?branch=master)
 [![Gem Version](https://badge.fury.io/rb/svmkit.svg)](https://badge.fury.io/rb/svmkit)
 [![BSD 2-Clause License](https://img.shields.io/badge/License-BSD%202--Clause-orange.svg)](https://github.com/yoshoku/SVMKit/blob/master/LICENSE.txt)
 
 SVMKit is a machine learninig library in Ruby.
 SVMKit provides machine learning algorithms with interfaces similar to Scikit-Learn in Python.
 SVMKit currently supports Linear / Kernel Support Vector Machine,
-Logistic Regression, Factorization Machine, Naive Bayes,
+Logistic Regression, Factorization Machine, Naive Bayes, Decision Tree, Random Forest,
 K-nearest neighbor classifier, and cross-validation.
 
 ## Installation

data/lib/svmkit.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 require 'numo/narray'
 
@@ -17,6 +18,8 @@ require 'svmkit/polynomial_model/factorization_machine_classifier'
 require 'svmkit/multiclass/one_vs_rest_classifier'
 require 'svmkit/nearest_neighbors/k_neighbors_classifier'
 require 'svmkit/naive_bayes/naive_bayes'
+require 'svmkit/tree/decision_tree_classifier'
+require 'svmkit/ensemble/random_forest_classifier'
 require 'svmkit/preprocessing/l2_normalizer'
 require 'svmkit/preprocessing/min_max_scaler'
 require 'svmkit/preprocessing/standard_scaler'

data/lib/svmkit/base/base_estimator.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module SVMKit
   # This module consists of basic mix-in classes.

data/lib/svmkit/base/classifier.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module SVMKit
   module Base
@@ -13,9 +14,14 @@ module SVMKit
         raise NotImplementedError, "#{__method__} has to be implemented in #{self.class}."
       end
 
-      # An abstract method for calculating classification accuracy.
-      def score
-        raise NotImplementedError, "#{__method__} has to be implemented in #{self.class}."
+      # Claculate the mean accuracy of the given testing data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) Testing data.
+      # @param y [Numo::Int32] (shape: [n_samples]) True labels for testing data.
+      # @return [Float] Mean accuracy
+      def score(x, y)
+        evaluator = SVMKit::EvaluationMeasure::Accuracy.new
+        evaluator.score(y, predict(x))
       end
     end
   end

data/lib/svmkit/base/evaluator.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module SVMKit
   module Base

data/lib/svmkit/base/splitter.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module SVMKit
   module Base

data/lib/svmkit/base/transformer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 
 module SVMKit
   module Base

data/lib/svmkit/dataset.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SVMKit
   # Module for loading and saving a dataset file.
   module Dataset

data/lib/svmkit/ensemble/random_forest_classifier.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require 'svmkit/base/base_estimator'
+require 'svmkit/base/classifier'
+
+module SVMKit
+  # This module consists of the classes that implement ensemble-based methods.
+  module Ensemble
+    # RandomForestClassifier is a class that implements random forest for classification.
+    #
+    # @example
+    #   estimator =
+    #     SVMKit::Ensemble::RandomForestClassifier.new(
+    #       n_estimators: 10, criterion: 'gini', max_depth: 3, max_leaf_nodes: 10, min_samples_leaf: 5, random_seed: 1)
+    #   estimator.fit(training_samples, traininig_labels)
+    #   results = estimator.predict(testing_samples)
+    #
+    class RandomForestClassifier
+      include Base::BaseEstimator
+      include Base::Classifier
+
+      # Return the set of estimators.
+      # @return [Array<DecisionTreeClassifier>]
+      attr_reader :estimators
+
+      # Return the class labels.
+      # @return [Numo::Int32] (size: n_classes)
+      attr_reader :classes
+
+      # Return the importance for each feature.
+      # @return [Numo::DFloat] (size: n_features)
+      attr_reader :feature_importances
+
+      # Return the random generator for performing random sampling in the Pegasos algorithm.
+      # @return [Random]
+      attr_reader :rng
+
+      # Create a new classifier with random forest.
+      #
+      # @param n_estimators [Integer] The numeber of decision trees for contructing random forest.
+      # @param criterion [String] The function to evalue spliting point. Supported criteria are 'gini' and 'entropy'.
+      # @param max_depth [Integer] The maximum depth of the tree.
+      #   If nil is given, decision tree grows without concern for depth.
+      # @param max_leaf_nodes [Integer] The maximum number of leaves on decision tree.
+      #   If nil is given, number of leaves is not limited.
+      # @param min_samples_leaf [Integer] The minimum number of samples at a leaf node.
+      # @param max_features [Integer] The number of features to consider when searching optimal split point.
+      #   If nil is given, split process considers all features.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      #   It is used to randomly determine the order of features when deciding spliting point.
+      def initialize(n_estimators: 10, criterion: 'gini', max_depth: nil, max_leaf_nodes: nil, min_samples_leaf: 1,
+                     max_features: nil, random_seed: nil)
+        @params = {}
+        @params[:n_estimators] = n_estimators
+        @params[:criterion] = criterion
+        @params[:max_depth] = max_depth
+        @params[:max_leaf_nodes] = max_leaf_nodes
+        @params[:min_samples_leaf] = min_samples_leaf
+        @params[:max_features] = max_features
+        @params[:random_seed] = random_seed
+        @params[:random_seed] ||= srand
+        @rng = Random.new(@params[:random_seed])
+        @estimators = nil
+        @classes = nil
+        @feature_importances = nil
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [RandomForestClassifier] The learned classifier itself.
+      def fit(x, y)
+        # Initialize some variables.
+        n_samples, n_features = x.shape
+        @params[:max_features] = n_features unless @params[:max_features].is_a?(Integer)
+        @params[:max_features] = [[1, @params[:max_features]].max, Math.sqrt(n_features).to_i].min
+        @classes = Numo::Int32.asarray(y.to_a.uniq.sort)
+        # Construct forest.
+        @estimators = Array.new(@params[:n_estimators]) do |_n|
+          tree = Tree::DecisionTreeClassifier.new(
+            criterion: @params[:criterion], max_depth: @params[:max_depth],
+            max_leaf_nodes: @params[:max_leaf_nodes], min_samples_leaf: @params[:min_samples_leaf],
+            max_features: @params[:max_features], random_seed: @params[:random_seed]
+          )
+          bootstrap_ids = Array.new(n_samples) { @rng.rand(0...n_samples) }
+          tree.fit(x[bootstrap_ids, true], y[bootstrap_ids])
+        end
+        # Calculate feature importances.
+        @feature_importances = Numo::DFloat.zeros(n_features)
+        @estimators.each { |tree| @feature_importances += tree.feature_importances }
+        @feature_importances /= @feature_importances.sum
+        self
+      end
+
+      # Predict class labels for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
+      def predict(x)
+        n_samples, = x.shape
+        n_classes = @classes.size
+        classes_arr = @classes.to_a
+        ballot_box = Numo::DFloat.zeros(n_samples, n_classes)
+        @estimators.each do |tree|
+          predicted = tree.predict(x)
+          n_samples.times do |n|
+            class_id = classes_arr.index(predicted[n])
+            ballot_box[n, class_id] += 1.0 unless class_id.nil?
+          end
+        end
+        Numo::Int32[*Array.new(n_samples) { |n| @classes[ballot_box[n, true].max_index] }]
+      end
+
+      # Predict probability for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
+      def predict_proba(x)
+        n_samples, = x.shape
+        n_classes = @classes.size
+        classes_arr = @classes.to_a
+        ballot_box = Numo::DFloat.zeros(n_samples, n_classes)
+        @estimators.each do |tree|
+          probs = tree.predict_proba(x)
+          tree.classes.size.times do |n|
+            class_id = classes_arr.index(tree.classes[n])
+            ballot_box[true, class_id] += probs[true, n] unless class_id.nil?
+          end
+        end
+        (ballot_box.transpose / ballot_box.sum(axis: 1)).transpose
+      end
+
+      # Return the index of the leaf that each sample reached.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples, n_estimators]) Leaf index for sample.
+      def apply(x)
+        Numo::Int32[*Array.new(@params[:n_estimators]) { |n| @estimators[n].apply(x) }].transpose
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data about RandomForestClassifier
+      def marshal_dump
+        { params: @params, estimators: @estimators, classes: @classes,
+          feature_importances: @feature_importances, rng: @rng }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @estimators = obj[:estimators]
+        @classes = obj[:classes]
+        @feature_importances = obj[:feature_importances]
+        @rng = obj[:rng]
+        nil
+      end
+    end
+  end
+end

data/lib/svmkit/evaluation_measure/accuracy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/evaluator'
 
 module SVMKit

data/lib/svmkit/evaluation_measure/f_score.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/evaluator'
 require 'svmkit/evaluation_measure/precision_recall'
 

data/lib/svmkit/evaluation_measure/precision.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/evaluator'
 require 'svmkit/evaluation_measure/precision_recall'
 

data/lib/svmkit/evaluation_measure/precision_recall.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/evaluator'
 
 module SVMKit

data/lib/svmkit/evaluation_measure/recall.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/evaluator'
 require 'svmkit/evaluation_measure/precision_recall'
 

data/lib/svmkit/kernel_approximation/rbf.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/base_estimator'
 require 'svmkit/base/transformer'
 

data/lib/svmkit/kernel_machine/kernel_svc.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/base_estimator'
 require 'svmkit/base/classifier'
 
@@ -97,9 +99,7 @@ module SVMKit
       # @param y [Numo::Int32] (shape: [n_testing_samples]) True labels for testing data.
       # @return [Float] Mean accuracy
       def score(x, y)
-        p = predict(x)
-        n_hits = (y.to_a.map.with_index { |l, n| l == p[n] ? 1 : 0 }).inject(:+)
-        n_hits / y.size.to_f
+        super
       end
 
       # Dump marshal data.

data/lib/svmkit/linear_model/logistic_regression.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/base_estimator'
 require 'svmkit/base/classifier'
 
@@ -135,17 +137,6 @@ module SVMKit
         proba
       end
 
-      # Claculate the mean accuracy of the given testing data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) Testing data.
-      # @param y [Numo::Int32] (shape: [n_samples]) True labels for testing data.
-      # @return [Float] Mean accuracy
-      def score(x, y)
-        p = predict(x)
-        n_hits = (y.to_a.map.with_index { |l, n| l == p[n] ? 1 : 0 }).inject(:+)
-        n_hits / y.size.to_f
-      end
-
       # Dump marshal data.
       # @return [Hash] The marshal data about LogisticRegression.
       def marshal_dump

data/lib/svmkit/linear_model/svc.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/base_estimator'
 require 'svmkit/base/classifier'
 
@@ -118,17 +120,6 @@ module SVMKit
         Numo::Int32.cast(decision_function(x).map { |v| v >= 0 ? 1 : -1 })
       end
 
-      # Claculate the mean accuracy of the given testing data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) Testing data.
-      # @param y [Numo::Int32] (shape: [n_samples]) True labels for testing data.
-      # @return [Float] Mean accuracy
-      def score(x, y)
-        p = predict(x)
-        n_hits = (y.to_a.map.with_index { |l, n| l == p[n] ? 1 : 0 }).inject(:+)
-        n_hits / y.size.to_f
-      end
-
       # Dump marshal data.
       # @return [Hash] The marshal data about SVC.
       def marshal_dump

data/lib/svmkit/model_selection/cross_validation.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/splitter'
 
 module SVMKit

data/lib/svmkit/model_selection/k_fold.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/splitter'
 
 module SVMKit

data/lib/svmkit/model_selection/stratified_k_fold.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/splitter'
 
 module SVMKit

data/lib/svmkit/multiclass/one_vs_rest_classifier.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/base_estimator.rb'
 require 'svmkit/base/classifier.rb'
 
@@ -68,17 +70,6 @@ module SVMKit
         Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })
       end
 
-      # Claculate the mean accuracy of the given testing data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) Testing data.
-      # @param y [Numo::Int32] (shape: [n_samples]) True labels for testing data.
-      # @return [Float] Mean accuracy
-      def score(x, y)
-        p = predict(x)
-        n_hits = (y.to_a.map.with_index { |l, n| l == p[n] ? 1 : 0 }).inject(:+)
-        n_hits / y.size.to_f
-      end
-
       # Dump marshal data.
       # @return [Hash] The marshal data about OneVsRestClassifier.
       def marshal_dump

data/lib/svmkit/naive_bayes/naive_bayes.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/base_estimator'
 require 'svmkit/base/classifier'
 
@@ -36,16 +38,6 @@ module SVMKit
       def predict_proba(x)
         Numo::NMath.exp(predict_log_proba(x)).abs
       end
-
-      # Claculate the mean accuracy of the given testing data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) Testing data.
-      # @param y [Numo::Int32] (shape: [n_samples]) True labels for testing data.
-      # @return [Float] Mean accuracy
-      def score(x, y)
-        evaluator = SVMKit::EvaluationMeasure::Accuracy.new
-        evaluator.score(y, predict(x))
-      end
     end
 
     # GaussianNB is a class that implements Gaussian Naive Bayes classifier.
@@ -109,7 +101,7 @@ module SVMKit
       #
       # @return [Hash] The marshal data about GaussianNB.
       def marshal_dump
-        { params: params,
+        { params: @params,
           classes: @classes,
           class_priors: @class_priors,
           means: @means,
@@ -193,7 +185,7 @@ module SVMKit
       #
       # @return [Hash] The marshal data about MultinomialNB.
       def marshal_dump
-        { params: params,
+        { params: @params,
           classes: @classes,
           class_priors: @class_priors,
           feature_probs: @feature_probs }
@@ -283,7 +275,7 @@ module SVMKit
       #
       # @return [Hash] The marshal data about BernoulliNB.
       def marshal_dump
-        { params: params,
+        { params: @params,
           classes: @classes,
           class_priors: @class_priors,
           feature_probs: @feature_probs }

data/lib/svmkit/nearest_neighbors/k_neighbors_classifier.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/base_estimator'
 require 'svmkit/base/classifier'
 
@@ -79,21 +81,10 @@ module SVMKit
         Numo::Int32.asarray(Array.new(n_samples) { |n| @classes[decision_values[n, true].max_index] })
       end
 
-      # Claculate the mean accuracy of the given testing data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) Testing data.
-      # @param y [Numo::Int32] (shape: [n_samples]) True labels for testing data.
-      # @return [Float] Mean accuracy
-      def score(x, y)
-        p = predict(x)
-        n_hits = (y.to_a.map.with_index { |l, n| l == p[n] ? 1 : 0 }).inject(:+)
-        n_hits / y.size.to_f
-      end
-
       # Dump marshal data.
       # @return [Hash] The marshal data about KNeighborsClassifier.
       def marshal_dump
-        { params: params,
+        { params: @params,
           prototypes: @prototypes,
           labels: @labels,
           classes: @classes }

data/lib/svmkit/pairwise_metric.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module SVMKit
   # Module for calculating pairwise distances, similarities, and kernels.
   module PairwiseMetric

data/lib/svmkit/polynomial_model/factorization_machine_classifier.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/base_estimator'
 require 'svmkit/base/classifier'
 
 module SVMKit
-  # This module consists of the classes that implemnt polynomial models.
+  # This module consists of the classes that implement polynomial models.
   module PolynomialModel
     # FactorizationMachineClassifier is a class that
     # implements Fatorization Machine for binary classification
@@ -136,17 +138,6 @@ module SVMKit
         proba
       end
 
-      # Claculate the mean accuracy of the given testing data.
-      #
-      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) Testing data.
-      # @param y [Numo::Int32] (shape: [n_samples]) True labels for testing data.
-      # @return [Float] Mean accuracy
-      def score(x, y)
-        p = predict(x)
-        n_hits = (y.to_a.map.with_index { |l, n| l == p[n] ? 1 : 0 }).inject(:+)
-        n_hits / y.size.to_f
-      end
-
       # Dump marshal data.
       # @return [Hash] The marshal data about FactorizationMachineClassifier
       def marshal_dump

data/lib/svmkit/preprocessing/l2_normalizer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/base_estimator'
 require 'svmkit/base/transformer'
 

data/lib/svmkit/preprocessing/min_max_scaler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/base_estimator'
 require 'svmkit/base/transformer'
 

data/lib/svmkit/preprocessing/standard_scaler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'svmkit/base/base_estimator'
 require 'svmkit/base/transformer'
 

data/lib/svmkit/tree/decision_tree_classifier.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+require 'svmkit/base/base_estimator'
+require 'svmkit/base/classifier'
+require 'ostruct'
+
+module SVMKit
+  # This module consists of the classes that implement tree models.
+  module Tree
+    # DecisionTreeClassifier is a class that implements decision tree for classification.
+    #
+    # @example
+    #   estimator =
+    #     SVMKit::Tree::DecisionTreeClassifier.new(
+    #       criterion: 'gini', max_depth: 3, max_leaf_nodes: 10, min_samples_leaf: 5, random_seed: 1)
+    #   estimator.fit(training_samples, traininig_labels)
+    #   results = estimator.predict(testing_samples)
+    #
+    class DecisionTreeClassifier
+      include Base::BaseEstimator
+      include Base::Classifier
+
+      # Return the class labels.
+      # @return [Numo::Int32] (size: n_classes)
+      attr_reader :classes
+
+      # Return the importance for each feature.
+      # @return [Numo::DFloat] (size: n_features)
+      attr_reader :feature_importances
+
+      # Return the learned tree.
+      # @return [OpenStruct]
+      attr_reader :tree
+
+      # Return the random generator for performing random sampling in the Pegasos algorithm.
+      # @return [Random]
+      attr_reader :rng
+
+      # Return the labels assigned each leaf.
+      # @return [Numo::Int32] (size: n_leafs)
+      attr_reader :leaf_labels
+
+      # Create a new classifier with decision tree algorithm.
+      #
+      # @param criterion [String] The function to evalue spliting point. Supported criteria are 'gini' and 'entropy'.
+      # @param max_depth [Integer] The maximum depth of the tree.
+      #   If nil is given, decision tree grows without concern for depth.
+      # @param max_leaf_nodes [Integer] The maximum number of leaves on decision tree.
+      #   If nil is given, number of leaves is not limited.
+      # @param min_samples_leaf [Integer] The minimum number of samples at a leaf node.
+      # @param max_features [Integer] The number of features to consider when searching optimal split point.
+      #   If nil is given, split process considers all features.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      #   It is used to randomly determine the order of features when deciding spliting point.
+      def initialize(criterion: 'gini', max_depth: nil, max_leaf_nodes: nil, min_samples_leaf: 1, max_features: nil,
+                     random_seed: nil)
+        @params = {}
+        @params[:criterion] = criterion
+        @params[:max_depth] = max_depth
+        @params[:max_leaf_nodes] = max_leaf_nodes
+        @params[:min_samples_leaf] = min_samples_leaf
+        @params[:max_features] = max_features
+        @params[:random_seed] = random_seed
+        @params[:random_seed] ||= srand
+        @rng = Random.new(@params[:random_seed])
+        @tree = nil
+        @classes = nil
+        @feature_importances = nil
+        @n_leaves = nil
+        @leaf_labels = nil
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::Int32] (shape: [n_samples]) The labels to be used for fitting the model.
+      # @return [DecisionTreeClassifier] The learned classifier itself.
+      def fit(x, y)
+        n_samples, n_features = x.shape
+        @params[:max_features] = n_features unless @params[:max_features].is_a?(Integer)
+        @params[:max_features] = [[1, @params[:max_features]].max, n_features].min
+        @classes = Numo::Int32.asarray(y.to_a.uniq.sort)
+        build_tree(x, y)
+        eval_importance(n_samples, n_features)
+        self
+      end
+
+      # Predict class labels for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Predicted class label per sample.
+      def predict(x)
+        @leaf_labels[apply(x)]
+      end
+
+      # Predict probability for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the probailities.
+      # @return [Numo::DFloat] (shape: [n_samples, n_classes]) Predicted probability of each class per sample.
+      def predict_proba(x)
+        probs = Numo::DFloat[*(Array.new(x.shape[0]) { |n| predict_at_node(@tree, x[n, true]) })]
+        probs[true, @classes]
+      end
+
+      # Return the index of the leaf that each sample reached.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Leaf index for sample.
+      def apply(x)
+        Numo::Int32[*(Array.new(x.shape[0]) { |n| apply_at_node(@tree, x[n, true]) })]
+      end
+
+      # Dump marshal data.
+      # @return [Hash] The marshal data about DecisionTreeClassifier
+      def marshal_dump
+        { params: @params, classes: @classes, tree: @tree,
+          feature_importances: @feature_importances, leaf_labels: @leaf_labels,
+          rng: @rng }
+      end
+
+      # Load marshal data.
+      # @return [nil]
+      def marshal_load(obj)
+        @params = obj[:params]
+        @classes = obj[:classes]
+        @tree = obj[:tree]
+        @feature_importances = obj[:feature_importances]
+        @leaf_labels = obj[:leaf_labels]
+        @rng = obj[:rng]
+        nil
+      end
+
+      private
+
+      def predict_at_node(node, sample)
+        return node.probs if node.leaf
+        branch_at_node('predict', node, sample)
+      end
+
+      def apply_at_node(node, sample)
+        return node.leaf_id if node.leaf
+        branch_at_node('apply', node, sample)
+      end
+
+      def branch_at_node(action, node, sample)
+        return send("#{action}_at_node", node.left, sample) if node.right.nil?
+        return send("#{action}_at_node", node.right, sample) if node.left.nil?
+        if sample[node.feature_id] <= node.threshold
+          send("#{action}_at_node", node.left, sample)
+        else
+          send("#{action}_at_node", node.right, sample)
+        end
+      end
+
+      def build_tree(x, y)
+        @n_leaves = 0
+        @leaf_labels = []
+        @tree = grow_node(0, x, y)
+        @leaf_labels = Numo::Int32[*@leaf_labels]
+        nil
+      end
+
+      def grow_node(depth, x, y)
+        if @params[:max_leaf_nodes].is_a?(Integer)
+          return nil if @n_leaves >= @params[:max_leaf_nodes]
+        end
+
+        n_samples, n_features = x.shape
+        if @params[:min_samples_leaf].is_a?(Integer)
+          return nil if n_samples <= @params[:min_samples_leaf]
+        end
+
+        node = OpenStruct.new(depth: depth, impurity: impurity(y), n_samples: n_samples)
+
+        return put_leaf(node, y) if y.to_a.uniq.size == 1
+
+        if @params[:max_depth].is_a?(Integer)
+          return put_leaf(node, y) if depth == @params[:max_depth]
+        end
+
+        feature_id, threshold, left_ids, right_ids, max_gain =
+          rand_ids(n_features).map { |f_id| [f_id, *best_split(x[true, f_id], y)] }.max_by(&:last)
+        return put_leaf(node, y) if max_gain.nil?
+        return put_leaf(node, y) if max_gain.zero?
+
+        node.left = grow_node(depth + 1, x[left_ids, true], y[left_ids])
+        node.right = grow_node(depth + 1, x[right_ids, true], y[right_ids])
+        return put_leaf(node, y) if node.left.nil? && node.right.nil?
+
+        node.feature_id = feature_id
+        node.threshold = threshold
+        node.leaf = false
+        node
+      end
+
+      def put_leaf(node, y)
+        node.probs = y.bincount(minlength: @classes.max + 1) / node.n_samples.to_f
+        node.leaf = true
+        node.leaf_id = @n_leaves
+        @n_leaves += 1
+        @leaf_labels.push(node.probs.max_index)
+        node
+      end
+
+      def rand_ids(n)
+        [*0...n].sample(@params[:max_features], random: @rng)
+      end
+
+      def best_split(features, labels)
+        features.to_a.uniq.sort.each_cons(2).map do |l, r|
+          threshold = 0.5 * (l + r)
+          left_ids, right_ids = splited_ids(features, threshold)
+          [threshold, left_ids, right_ids, gain(labels, labels[left_ids], labels[right_ids])]
+        end.max_by(&:last)
+      end
+
+      def splited_ids(features, threshold)
+        [features.le(threshold).where.to_a, features.gt(threshold).where.to_a]
+      end
+
+      def gain(labels, labels_left, labels_right)
+        prob_left = labels_left.size / labels.size.to_f
+        prob_right = labels_right.size / labels.size.to_f
+        impurity(labels) - prob_left * impurity(labels_left) - prob_right * impurity(labels_right)
+      end
+
+      def impurity(labels)
+        posterior_probs = labels.to_a.uniq.sort.map { |c| labels.eq(c).count / labels.size.to_f }
+        @params[:criterion] == 'entropy' ? entropy(posterior_probs) : gini(posterior_probs)
+      end
+
+      def gini(posterior_probs)
+        1.0 - posterior_probs.map { |p| p**2 }.inject(:+)
+      end
+
+      def entropy(posterior_probs)
+        -posterior_probs.map { |p| p * Math.log(p) }.inject(:+)
+      end
+
+      def eval_importance(n_samples, n_features)
+        @feature_importances = Numo::DFloat.zeros(n_features)
+        eval_importance_at_node(@tree)
+        @feature_importances /= n_samples
+        normalizer = @feature_importances.sum
+        @feature_importances /= normalizer if normalizer > 0.0
+        nil
+      end
+
+      def eval_importance_at_node(node)
+        return nil if node.leaf
+        return nil if node.left.nil? || node.right.nil?
+        gain = node.n_samples * node.impurity -
+               node.left.n_samples * node.left.impurity - node.right.n_samples * node.right.impurity
+        @feature_importances[node.feature_id] += gain
+        eval_importance_at_node(node.left)
+        eval_importance_at_node(node.right)
+      end
+    end
+  end
+end

data/lib/svmkit/version.rb

@@ -1,5 +1,7 @@
-# SVMKit is an experimental library of machine learning in Ruby.
+# frozen_string_literal: true
+
+# SVMKit is a machine learning library in Ruby.
 module SVMKit
   # @!visibility private
-  VERSION = '0.2.5'.freeze
+  VERSION = '0.2.6'
 end

data/svmkit.gemspec

@@ -18,7 +18,7 @@ MSG
 SVMKit is a machine learninig library in Ruby.
 SVMKit provides machine learning algorithms with interfaces similar to Scikit-Learn in Python.
 SVMKit currently supports Linear / Kernel Support Vector Machine,
-Logistic Regression, Factorization Machine, Naive Bayes,
+Logistic Regression, Factorization Machine, Naive Bayes, Decision Tree, Random Forest,
 K-nearest neighbor classifier, and cross-validation.
 MSG
   spec.homepage      = 'https://github.com/yoshoku/svmkit'
@@ -38,7 +38,7 @@ MSG
   spec.add_development_dependency 'bundler', '~> 1.16'
   spec.add_development_dependency 'rake', '~> 12.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'simplecov', '~> 0.15'
+  spec.add_development_dependency 'coveralls', '~> 0.8'
 
   spec.post_install_message = <<MSG
 *************************************************************************

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: svmkit
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.2.6
 platform: ruby
 authors:
 - yoshoku
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-02-24 00:00:00.000000000 Z
+date: 2018-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -67,24 +67,24 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0.8'
 description: |
   SVMKit is a machine learninig library in Ruby.
   SVMKit provides machine learning algorithms with interfaces similar to Scikit-Learn in Python.
   SVMKit currently supports Linear / Kernel Support Vector Machine,
-  Logistic Regression, Factorization Machine, Naive Bayes,
+  Logistic Regression, Factorization Machine, Naive Bayes, Decision Tree, Random Forest,
   K-nearest neighbor classifier, and cross-validation.
 email:
 - yoshoku@outlook.com
@@ -92,6 +92,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coveralls.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
@@ -112,6 +113,7 @@ files:
 - lib/svmkit/base/splitter.rb
 - lib/svmkit/base/transformer.rb
 - lib/svmkit/dataset.rb
+- lib/svmkit/ensemble/random_forest_classifier.rb
 - lib/svmkit/evaluation_measure/accuracy.rb
 - lib/svmkit/evaluation_measure/f_score.rb
 - lib/svmkit/evaluation_measure/precision.rb
@@ -132,6 +134,7 @@ files:
 - lib/svmkit/preprocessing/l2_normalizer.rb
 - lib/svmkit/preprocessing/min_max_scaler.rb
 - lib/svmkit/preprocessing/standard_scaler.rb
+- lib/svmkit/tree/decision_tree_classifier.rb
 - lib/svmkit/version.rb
 - svmkit.gemspec
 homepage: https://github.com/yoshoku/svmkit