rumale-tree 0.24.0

data/lib/rumale/tree/gradient_tree_regressor.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+require 'rumale/base/estimator'
+require 'rumale/base/regressor'
+require 'rumale/validation'
+require 'rumale/tree/ext'
+require 'rumale/tree/node'
+
+module Rumale
+  module Tree
+    # GradientTreeRegressor is a class that implements decision tree for regression with exact gredy algorithm.
+    # This class is used internally for estimators with gradient tree boosting.
+    #
+    # *Reference*
+    # - Friedman, J H., "Greedy Function Approximation: A Gradient Boosting Machine," Annals of Statistics, 29 (5), pp. 1189--1232, 2001.
+    # - Friedman, J H., "Stochastic Gradient Boosting," Computational Statistics and Data Analysis, 38 (4), pp. 367--378, 2002.
+    # - Chen, T., and Guestrin, C., "XGBoost: A Scalable Tree Boosting System,"  Proc. KDD'16, pp. 785--794, 2016.
+    class GradientTreeRegressor < ::Rumale::Base::Estimator
+      include ::Rumale::Base::Regressor
+      include ::Rumale::Tree::ExtGradientTreeRegressor
+
+      # Return the importance for each feature.
+      # The feature importances are calculated based on the numbers of times the feature is used for splitting.
+      # @return [Numo::DFloat] (shape: [n_features])
+      attr_reader :feature_importances
+
+      # Return the learned tree.
+      # @return [Node]
+      attr_reader :tree
+
+      # Return the random generator for random selection of feature index.
+      # @return [Random]
+      attr_reader :rng
+
+      # Return the values assigned each leaf.
+      # @return [Numo::DFloat] (shape: [n_leaves])
+      attr_reader :leaf_weights
+
+      # Initialize a gradient tree regressor
+      #
+      # @param reg_lambda [Float] The L2 regularization term on weight.
+      # @param shrinkage_rate [Float] The shrinkage rate for weight.
+      # @param max_depth [Integer] The maximum depth of the tree.
+      #   If nil is given, decision tree grows without concern for depth.
+      # @param max_leaf_nodes [Integer] The maximum number of leaves on decision tree.
+      #   If nil is given, number of leaves is not limited.
+      # @param min_samples_leaf [Integer] The minimum number of samples at a leaf node.
+      # @param max_features [Integer] The number of features to consider when searching optimal split point.
+      #   If nil is given, split process considers all features.
+      # @param random_seed [Integer] The seed value using to initialize the random generator.
+      #   It is used to randomly determine the order of features when deciding spliting point.
+      def initialize(reg_lambda: 0.0, shrinkage_rate: 1.0,
+                     max_depth: nil, max_leaf_nodes: nil, min_samples_leaf: 1, max_features: nil, random_seed: nil)
+        super()
+        @params = {
+          reg_lambda: reg_lambda,
+          shrinkage_rate: shrinkage_rate,
+          max_depth: max_depth,
+          max_leaf_nodes: max_leaf_nodes,
+          min_samples_leaf: min_samples_leaf,
+          max_features: max_features,
+          random_seed: random_seed || srand
+        }
+        @rng = Random.new(@params[:random_seed])
+      end
+
+      # Fit the model with given training data.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The training data to be used for fitting the model.
+      # @param y [Numo::DFloat] (shape: [n_samples]) The taget values to be used for fitting the model.
+      # @param g [Numo::DFloat] (shape: [n_samples]) The gradient of loss function.
+      # @param h [Numo::DFloat] (shape: [n_samples]) The hessian of loss function.
+      # @return [GradientTreeRegressor] The learned regressor itself.
+      def fit(x, y, g, h)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+        y = ::Rumale::Validation.check_convert_target_value_array(y)
+        ::Rumale::Validation.check_sample_size(x, y)
+
+        # Initialize some variables.
+        n_features = x.shape[1]
+        @params[:max_features] ||= n_features
+        @n_leaves = 0
+        @leaf_weights = []
+        @feature_importances = Numo::DFloat.zeros(n_features)
+        @sub_rng = @rng.dup
+        # Build tree.
+        build_tree(x, y, g, h)
+        @leaf_weights = Numo::DFloat[*@leaf_weights]
+        self
+      end
+
+      # Predict values for samples.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the values.
+      # @return [Numo::DFloat] (size: n_samples) Predicted values per sample.
+      def predict(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        @leaf_weights[apply(x)].dup
+      end
+
+      # Return the index of the leaf that each sample reached.
+      #
+      # @param x [Numo::DFloat] (shape: [n_samples, n_features]) The samples to predict the labels.
+      # @return [Numo::Int32] (shape: [n_samples]) Leaf index for sample.
+      def apply(x)
+        x = ::Rumale::Validation.check_convert_sample_array(x)
+
+        Numo::Int32[*(Array.new(x.shape[0]) { |n| partial_apply(@tree, x[n, true]) })]
+      end
+
+      private
+
+      def partial_apply(tree, sample)
+        node = tree
+        until node.leaf
+          node = if node.right.nil?
+                   node.left
+                 elsif node.left.nil?
+                   node.right
+                 else
+                   sample[node.feature_id] <= node.threshold ? node.left : node.right
+                 end
+        end
+        node.leaf_id
+      end
+
+      def build_tree(x, y, g, h)
+        @feature_ids = Array.new(x.shape[1]) { |v| v }
+        @tree = grow_node(0, x, y, g, h)
+        @feature_ids = nil
+        nil
+      end
+
+      def grow_node(depth, x, y, g, h) # rubocop:disable Metrics/AbcSize
+        # intialize some variables.
+        sum_g = g.sum
+        sum_h = h.sum
+        n_samples = x.shape[0]
+        node = Node.new(depth: depth, n_samples: n_samples)
+
+        # terminate growing.
+        return nil if !@params[:max_leaf_nodes].nil? && @n_leaves >= @params[:max_leaf_nodes]
+        return nil if n_samples < @params[:min_samples_leaf]
+        return put_leaf(node, sum_g, sum_h) if n_samples == @params[:min_samples_leaf]
+        return put_leaf(node, sum_g, sum_h) if !@params[:max_depth].nil? && depth == @params[:max_depth]
+        return put_leaf(node, sum_g, sum_h) if stop_growing?(y)
+
+        # calculate optimal parameters.
+        feature_id, threshold, gain = rand_ids.map { |n| [n, *best_split(x[true, n], g, h, sum_g, sum_h)] }.max_by(&:last)
+
+        return put_leaf(node, sum_g, sum_h) if gain.nil? || gain.zero?
+
+        left_ids = x[true, feature_id].le(threshold).where
+        right_ids = x[true, feature_id].gt(threshold).where
+        node.left = grow_node(depth + 1, x[left_ids, true], y[left_ids], g[left_ids], h[left_ids])
+        node.right = grow_node(depth + 1, x[right_ids, true], y[right_ids], g[right_ids], h[right_ids])
+
+        return put_leaf(node, sum_g, sum_h) if node.left.nil? && node.right.nil?
+
+        @feature_importances[feature_id] += 1.0
+
+        node.feature_id = feature_id
+        node.threshold = threshold
+        node.leaf = false
+        node
+      end
+
+      def stop_growing?(y)
+        y.to_a.uniq.size == 1
+      end
+
+      def put_leaf(node, sum_g, sum_h)
+        node.probs = nil
+        node.leaf = true
+        node.leaf_id = @n_leaves
+        weight = -@params[:shrinkage_rate] * sum_g / (sum_h + @params[:reg_lambda])
+        @leaf_weights.push(weight)
+        @n_leaves += 1
+        node
+      end
+
+      def best_split(f, g, h, sum_g, sum_h)
+        find_split_params(f.sort_index, f, g, h, sum_g, sum_h, @params[:reg_lambda])
+      end
+
+      def rand_ids
+        @feature_ids.sample(@params[:max_features], random: @sub_rng)
+      end
+    end
+  end
+end

data/lib/rumale/tree/node.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Rumale
+  module Tree
+    # Node is a class that implements node used for construction of decision tree.
+    # This class is used for internal data structures.
+    class Node
+      # @!visibility private
+      attr_accessor :depth, :impurity, :n_samples, :probs, :leaf, :leaf_id, :left, :right, :feature_id, :threshold
+
+      # Create a new node for decision tree.
+      #
+      # @param depth [Integer] The depth of the node in tree.
+      # @param impurity [Float] The impurity of the node.
+      # @param n_samples [Integer] The number of the samples in the node.
+      # @param probs [Float] The probability of the node.
+      # @param leaf [Boolean] The flag indicating whether the node is a leaf.
+      # @param leaf_id [Integer] The leaf index of the node.
+      # @param left [Node] The left node.
+      # @param right [Node] The right node.
+      # @param feature_id [Integer] The feature index used for evaluation.
+      # @param threshold [Float] The threshold value of the feature for splitting the node.
+      def initialize(depth: 0, impurity: 0.0, n_samples: 0, probs: 0.0,
+                     leaf: false, leaf_id: nil,
+                     left: nil, right: nil, feature_id: 0, threshold: 0.0)
+        @depth = depth
+        @impurity = impurity
+        @n_samples = n_samples
+        @probs = probs
+        @leaf = leaf
+        @leaf_id = leaf_id
+        @left = left
+        @right = right
+        @feature_id = feature_id
+        @threshold = threshold
+      end
+    end
+  end
+end

data/lib/rumale/tree/version.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Rumale is a machine learning library in Ruby.
+module Rumale
+  # This module consists of the classes that implement tree models.
+  module Tree
+    # @!visibility private
+    VERSION = '0.24.0'
+  end
+end

data/lib/rumale/tree.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'numo/narray'
+
+require_relative 'tree/version'
+
+require_relative 'tree/decision_tree_classifier'
+require_relative 'tree/decision_tree_regressor'
+require_relative 'tree/extra_tree_classifier'
+require_relative 'tree/extra_tree_regressor'
+require_relative 'tree/gradient_tree_regressor'

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: rumale-tree
+version: !ruby/object:Gem::Version
+  version: 0.24.0
+platform: ruby
+authors:
+- yoshoku
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2022-12-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: numo-narray
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+- !ruby/object:Gem::Dependency
+  name: rumale-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.0
+description: Rumale::Tree provides classifier and regression based on decision tree
+  algorithms with Rumale interface.
+email:
+- yoshoku@outlook.com
+executables: []
+extensions:
+- ext/rumale/tree/extconf.rb
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- ext/rumale/tree/ext.c
+- ext/rumale/tree/ext.h
+- ext/rumale/tree/extconf.rb
+- lib/rumale/tree.rb
+- lib/rumale/tree/base_decision_tree.rb
+- lib/rumale/tree/decision_tree_classifier.rb
+- lib/rumale/tree/decision_tree_regressor.rb
+- lib/rumale/tree/extra_tree_classifier.rb
+- lib/rumale/tree/extra_tree_regressor.rb
+- lib/rumale/tree/gradient_tree_regressor.rb
+- lib/rumale/tree/node.rb
+- lib/rumale/tree/version.rb
+homepage: https://github.com/yoshoku/rumale
+licenses:
+- BSD-3-Clause
+metadata:
+  homepage_uri: https://github.com/yoshoku/rumale
+  source_code_uri: https://github.com/yoshoku/rumale/tree/main/rumale-tree
+  changelog_uri: https://github.com/yoshoku/rumale/blob/main/CHANGELOG.md
+  documentation_uri: https://yoshoku.github.io/rumale/doc/
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.26
+signing_key:
+specification_version: 4
+summary: Rumale::Tree provides classifier and regression based on decision tree algorithms
+  with Rumale interface.
+test_files: []