rumale-tree 0.26.0 → 0.28.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eae93f4d3753b16336ffed4b0a467cf09ddb33b0b89005a83ed5088ea2b362ef
-  data.tar.gz: 73d6c88f3653b5d2c70ef884fb7f5ef5a68e91d79abd19570476c13f02c434c6
+  metadata.gz: a437c3b879dd51d2e823f1851ca8b350d0441947fb6cefe727de83db78b4e6d9
+  data.tar.gz: 519665f2baea649ec31c9d5529aa05c67a880daa8ab60149433a53483f630663
 SHA512:
-  metadata.gz: 6a5fa902ab481cc7fe7bdbc38e2e4da2f57c4b712d47fef6daabfd757d263949e19d12c5324a7ff0b0372e656a435908d3049380aef4f241c7ff39fc8914bf44
-  data.tar.gz: 7a5e3b465549928e01143949e4001d7825882a687591adb91f82f7b753f7527c4fa2e5787ad5af263777855f3e8a3e5101386fdc361da3940f7f6dce0933c5bb
+  metadata.gz: aa68d3e2fbe99ef4c8d15a10f64098abfbbcdce9047053967e578edad688fa08a03a1ac4e4d23dbd5e67a4a23370057aa9462107bb961eb4d3f12782ce0c0f18
+  data.tar.gz: 6a6e6f167e31d56051768a881a4db0da7ef02f36977cb3969e8160bad385757153b1f6be32dd364ed3f67ed4672dd0c1bfd14cc80e0b033e00dd282d757955a9

data/ext/rumale/tree/ext.cpp

@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) 2022-2023 Atsushi Tatsuma
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ext.hpp"
+
+extern "C" void Init_ext(void) {
+  VALUE rb_mRumale = rb_define_module("Rumale");
+  VALUE rb_mTree = rb_define_module_under(rb_mRumale, "Tree");
+  ExtDecisionTreeClassifier::define_module(rb_mTree);
+  ExtDecisionTreeRegressor::define_module(rb_mTree);
+  ExtGradientTreeRegressor::define_module(rb_mTree);
+}

data/ext/rumale/tree/ext.hpp

@@ -0,0 +1,550 @@
+/**
+ * Copyright (c) 2022-2023 Atsushi Tatsuma
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef RUMALE_TREE_EXT_HPP
+#define RUMALE_TREE_EXT_HPP 1
+
+#include <cmath>
+#include <limits>
+#include <string>
+#include <vector>
+
+#include <ruby.h>
+
+#include <numo/narray.h>
+#include <numo/template.h>
+
+/**
+ * @!visibility private
+ * Document-module: Rumale::Tree::ExtDecisionTreeClassifier
+ * The mixin module consisting of extension method for DecisionTreeClassifier class.
+ * This module is used internally.
+ */
+class ExtDecisionTreeClassifier {
+public:
+  static void define_module(VALUE& outer) {
+    VALUE rb_mExtDTreeCls = rb_define_module_under(outer, "ExtDecisionTreeClassifier");
+    rb_define_private_method(rb_mExtDTreeCls, "find_split_params", find_split_params_, 6);
+    rb_define_private_method(rb_mExtDTreeCls, "node_impurity", node_impurity_, 3);
+    rb_define_private_method(rb_mExtDTreeCls, "stop_growing?", check_same_label_, 1);
+  }
+
+private:
+  static double calc_impurity_(const std::string& criterion, const std::vector<size_t>& histogram, const size_t& n_elements, const size_t& n_classes) {
+    double impurity = 0.0;
+    if (criterion == "entropy") {
+      double entropy = 0.0;
+      for (size_t i = 0; i < n_classes; i++) {
+        const double el = static_cast<double>(histogram[i]) / static_cast<double>(n_elements);
+        entropy += el * std::log(el + 1.0);
+      }
+      impurity = -entropy;
+    } else {
+      double gini = 0.0;
+      for (size_t i = 0; i < n_classes; i++) {
+        const double el = static_cast<double>(histogram[i]) / static_cast<double>(n_elements);
+        gini += el * el;
+      }
+      impurity = 1.0 - gini;
+    }
+    return impurity;
+  }
+
+  /**
+   * @!visibility private
+   * Find for split point with maximum information gain.
+   *
+   * @overload find_split_params(criterion, impurity, order, features, labels, n_classes) -> Array<Float>
+   *
+   * @param criterion [String] The function to evaluate spliting point. Supported criteria are 'gini' and 'entropy'.
+   * @param impurity [Float] The impurity of whole dataset.
+   * @param order [Numo::Int32] (shape: [n_elements]) The element indices sorted according to feature values.
+   * @param features [Numo::DFloat] (shape: [n_elements]) The feature values.
+   * @param labels [Numo::Int32] (shape: [n_elements]) The labels.
+   * @param n_classes [Integer] The number of classes.
+   * @return [Array<Float>] The array consists of optimal parameters including impurities of child nodes, threshold, and gain.
+   */
+
+  struct FindSplitParamsOpts_ {
+    std::string criterion;
+    size_t n_classes;
+    double impurity;
+  };
+
+  static void iter_find_split_params_(na_loop_t const* lp) {
+    // Obtain iteration variables.
+    const int32_t* o = (int32_t*)NDL_PTR(lp, 0);
+    const size_t n_elements = NDL_SHAPE(lp, 0)[0];
+    const double* f = (double*)NDL_PTR(lp, 1);
+    const int32_t* y = (int32_t*)NDL_PTR(lp, 2);
+    const std::string criterion = ((FindSplitParamsOpts_*)lp->opt_ptr)->criterion;
+    const size_t n_classes = ((FindSplitParamsOpts_*)lp->opt_ptr)->n_classes;
+    const double w_impurity = ((FindSplitParamsOpts_*)lp->opt_ptr)->impurity;
+
+    // Initialize output optimal parameters.
+    double* params = (double*)NDL_PTR(lp, 3);
+    params[0] = 0.0;        // left impurity
+    params[1] = w_impurity; // right impurity
+    params[2] = f[o[0]];    // threshold
+    params[3] = 0.0;        // gain
+
+    // Initialize child node variables.
+    std::vector<size_t> r_histogram(n_classes, 0);
+    for (size_t i = 0; i < n_elements; i++) r_histogram[y[o[i]]] += 1;
+
+    // Find optimal parameters.
+    size_t curr_pos = 0;
+    size_t next_pos = 0;
+    size_t n_l_elements = 0;
+    size_t n_r_elements = n_elements;
+    double curr_el = f[o[0]];
+    const double last_el = f[o[n_elements - 1]];
+    std::vector<size_t> l_histogram(n_classes, 0);
+    while (curr_pos < n_elements && curr_el != last_el) {
+      double next_el = f[o[next_pos]];
+      while (next_pos < n_elements && next_el == curr_el) {
+        l_histogram[y[o[next_pos]]] += 1;
+        n_l_elements++;
+        r_histogram[y[o[next_pos]]] -= 1;
+        n_r_elements--;
+        next_pos++;
+        next_el = f[o[next_pos]];
+      }
+      // Calculate gain of new split.
+      const double l_impurity = calc_impurity_(criterion, l_histogram, n_l_elements, n_classes);
+      const double r_impurity = calc_impurity_(criterion, r_histogram, n_r_elements, n_classes);
+      const double gain = w_impurity - (n_l_elements * l_impurity + n_r_elements * r_impurity) / static_cast<double>(n_elements);
+      // Update optimal parameters.
+      if (gain > params[3]) {
+        params[0] = l_impurity;
+        params[1] = r_impurity;
+        params[2] = 0.5 * (curr_el + next_el);
+        params[3] = gain;
+      }
+      if (next_pos == n_elements) break;
+      curr_pos = next_pos;
+      curr_el = f[o[curr_pos]];
+    }
+  }
+
+  static VALUE find_split_params_(VALUE self, VALUE criterion, VALUE impurity, VALUE order, VALUE features, VALUE labels, VALUE n_classes) {
+    ndfunc_arg_in_t ain[3] = { { numo_cInt32, 1 }, { numo_cDFloat, 1 }, { numo_cInt32, 1 } };
+    size_t out_shape[1] = { 4 };
+    ndfunc_arg_out_t aout[1] = { { numo_cDFloat, 1, out_shape } };
+    ndfunc_t ndf = { (na_iter_func_t)iter_find_split_params_, NO_LOOP, 3, 1, ain, aout };
+    FindSplitParamsOpts_ opts = { std::string(StringValueCStr(criterion)), NUM2SIZET(n_classes), NUM2DBL(impurity) };
+    VALUE params = na_ndloop3(&ndf, &opts, 3, order, features, labels);
+    RB_GC_GUARD(criterion);
+    return params;
+  }
+
+  /**
+   * @!visibility private
+   * Calculate impurity based on criterion.
+   *
+   * @overload node_impurity(criterion, y, n_classes) -> Float
+   *
+   * @param criterion [String] The function to calculate impurity. Supported criteria are 'gini' and 'entropy'.
+   * @param y [Numo::Int32] (shape: [n_samples]) The labels.
+   * @param n_classes [Integer] The number of classes.
+   * @return [Float] impurity
+   */
+
+  struct NodeImpurityOpts_ {
+    std::string criterion;
+    size_t n_classes;
+  };
+
+  static void iter_node_impurity_(na_loop_t const* lp) {
+    const int32_t* y = (int32_t*)NDL_PTR(lp, 0);
+    const size_t n_elements = NDL_SHAPE(lp, 0)[0];
+    const std::string criterion = ((NodeImpurityOpts_*)lp->opt_ptr)->criterion;
+    const size_t n_classes = ((NodeImpurityOpts_*)lp->opt_ptr)->n_classes;
+    double* ret = (double*)NDL_PTR(lp, 1);
+    std::vector<size_t> histogram(n_classes, 0);
+    for (size_t i = 0; i < n_elements; i++) histogram[y[i]] += 1;
+    *ret = calc_impurity_(criterion, histogram, n_elements, n_classes);
+  }
+
+  static VALUE node_impurity_(VALUE self, VALUE criterion, VALUE y, VALUE n_classes) {
+    ndfunc_arg_in_t ain[1] = { { numo_cInt32, 1 } };
+    ndfunc_arg_out_t aout[1] = { { numo_cDFloat, 0 } };
+    ndfunc_t ndf = { (na_iter_func_t)iter_node_impurity_, NO_LOOP | NDF_EXTRACT, 1, 1, ain, aout };
+    NodeImpurityOpts_ opts = { std::string(StringValueCStr(criterion)), NUM2SIZET(n_classes) };
+    VALUE ret = na_ndloop3(&ndf, &opts, 1, y);
+    RB_GC_GUARD(criterion);
+    return ret;
+  }
+
+  /**
+   * @!visibility private
+   * Check all elements have the same value.
+   *
+   * @overload check_same_label(y) -> Boolean
+   *
+   * @param y [Numo::Int32] (shape: [n_samples]) The labels.
+   * @return [Boolean]
+   */
+
+  static void iter_check_same_label_(na_loop_t const* lp) {
+    const int32_t* y = (int32_t*)NDL_PTR(lp, 0);
+    const size_t n_elements = NDL_SHAPE(lp, 0)[0];
+    VALUE* ret = (VALUE*)NDL_PTR(lp, 1);
+    *ret = Qtrue;
+    if (n_elements > 0) {
+      int32_t label = y[0];
+      for (size_t i = 0; i < n_elements; i++) {
+        if (y[i] != label) {
+          *ret = Qfalse;
+          break;
+        }
+      }
+    }
+  }
+
+  static VALUE check_same_label_(VALUE self, VALUE y) {
+    ndfunc_arg_in_t ain[1] = { { numo_cInt32, 1 } };
+    ndfunc_arg_out_t aout[1] = { { numo_cRObject, 0 } };
+    ndfunc_t ndf = { (na_iter_func_t)iter_check_same_label_, NO_LOOP | NDF_EXTRACT, 1, 1, ain, aout };
+    return na_ndloop(&ndf, 1, y);
+  }
+};
+
+/**
+ * @!visibility private
+ * Document-module: Rumale::Tree::ExtDecisionTreeRegressor
+ * The mixin module consisting of extension method for DecisionTreeRegressor class.
+ * This module is used internally.
+ */
+class ExtDecisionTreeRegressor {
+public:
+  static void define_module(VALUE& outer) {
+    VALUE rb_mExtDTreeReg = rb_define_module_under(outer, "ExtDecisionTreeRegressor");
+    rb_define_private_method(rb_mExtDTreeReg, "find_split_params", find_split_params_, 5);
+    rb_define_private_method(rb_mExtDTreeReg, "node_impurity", node_impurity_, 2);
+    rb_define_private_method(rb_mExtDTreeReg, "stop_growing?", check_same_value_, 1);
+  }
+
+private:
+  static double calc_impurity_(const std::string& criterion, const int32_t* order, const double* vecs, const double* mean_vec,
+                               const size_t& n_elements, const size_t& n_outputs, const size_t& order_offset) {
+    const bool is_mae = criterion == "mae";
+    double sum_err = 0.0;
+    for (size_t i = 0; i < n_elements; i++) {
+      double err = 0.0;
+      for (size_t j = 0; j < n_outputs; j++) {
+        const double el = vecs[order[order_offset + i] * n_outputs + j] - mean_vec[j];
+        err += is_mae ? std::fabs(el) : el * el;
+      }
+      err /= static_cast<double>(n_outputs);
+      sum_err += err;
+    }
+    const double impurity = sum_err / static_cast<double>(n_elements);
+    return impurity;
+  }
+
+  /**
+   * @!visibility private
+   * Find for split point with maximum information gain.
+   *
+   * @overload find_split_params(criterion, impurity, order, features, targets) -> Array<Float>
+   *
+   * @param criterion [String] The function to evaluate spliting point. Supported criteria are 'mae' and 'mse'.
+   * @param impurity [Float] The impurity of whole dataset.
+   * @param order [Numo::Int32] (shape: [n_samples]) The element indices sorted according to feature values in ascending order.
+   * @param features [Numo::DFloat] (shape: [n_samples]) The feature values.
+   * @param targets [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values.
+   * @return [Array<Float>] The array consists of optimal parameters including impurities of child nodes, threshold, and gain.
+   */
+
+  struct FindSplitParamsOpts_ {
+    std::string criterion;
+    double impurity;
+  };
+
+  static void iter_find_split_params_(na_loop_t const* lp) {
+    // Obtain iteration variables.
+    const int32_t* order = (int32_t*)NDL_PTR(lp, 0);
+    const size_t n_elements = NDL_SHAPE(lp, 0)[0];
+    const double* f = (double*)NDL_PTR(lp, 1);
+    const double* y = (double*)NDL_PTR(lp, 2);
+    const size_t n_outputs = NDL_SHAPE(lp, 2)[1];
+    const std::string criterion = ((FindSplitParamsOpts_*)lp->opt_ptr)->criterion;
+    const double w_impurity = ((FindSplitParamsOpts_*)lp->opt_ptr)->impurity;
+
+    // Initialize optimal parameters.
+    double* params = (double*)NDL_PTR(lp, 3);
+    params[0] = 0.0;         // left impurity
+    params[1] = w_impurity;  // right impurity
+    params[2] = f[order[0]]; // threshold
+    params[3] = 0.0;         // gain
+
+    // Initialize child node variables.
+    std::vector<double> l_sum_y(n_outputs, 0);
+    std::vector<double> r_sum_y(n_outputs, 0);
+    for (size_t i = 0; i < n_elements; i++) {
+      for (size_t j = 0; j < n_outputs; j++) {
+        r_sum_y[j] += y[order[i] * n_outputs + j];
+      }
+    }
+
+    // Find optimal parameters.
+    size_t curr_pos = 0;
+    size_t next_pos = 0;
+    size_t n_l_elements = 0;
+    size_t n_r_elements = n_elements;
+    std::vector<double> l_mean_y(n_outputs, 0);
+    std::vector<double> r_mean_y(n_outputs, 0);
+    double curr_el = f[order[0]];
+    const double last_el = f[order[n_elements - 1]];
+    while (curr_pos < n_elements && curr_el != last_el) {
+      double next_el = f[order[next_pos]];
+      while (next_pos < n_elements && next_el == curr_el) {
+        for (size_t j = 0; j < n_outputs; j++) {
+          l_sum_y[j] += y[order[next_pos] * n_outputs + j];
+          r_sum_y[j] -= y[order[next_pos] * n_outputs + j];
+        }
+        n_l_elements++;
+        n_r_elements--;
+        next_pos++;
+        next_el = f[order[next_pos]];
+      }
+      // Calculate gain of new split.
+      for (size_t j = 0; j < n_outputs; j++) {
+        l_mean_y[j] = l_sum_y[j] / static_cast<double>(n_l_elements);
+        r_mean_y[j] = r_sum_y[j] / static_cast<double>(n_r_elements);
+      }
+      const double l_impurity = calc_impurity_(criterion, order, y, l_mean_y.data(), n_l_elements, n_outputs, 0);
+      const double r_impurity = calc_impurity_(criterion, order, y, r_mean_y.data(), n_r_elements, n_outputs, next_pos);
+      const double gain = w_impurity - (n_l_elements * l_impurity + n_r_elements * r_impurity) / static_cast<double>(n_elements);
+      // Update optimal parameters.
+      if (gain > params[3]) {
+        params[0] = l_impurity;
+        params[1] = r_impurity;
+        params[2] = 0.5 * (curr_el + next_el);
+        params[3] = gain;
+      }
+      if (next_pos == n_elements) break;
+      curr_pos = next_pos;
+      curr_el = f[order[curr_pos]];
+    }
+  }
+
+  static VALUE find_split_params_(VALUE self, VALUE criterion, VALUE impurity, VALUE order, VALUE features, VALUE targets) {
+    ndfunc_arg_in_t ain[3] = { { numo_cInt32, 1 }, { numo_cDFloat, 1 }, { numo_cDFloat, 2 } };
+    size_t out_shape[1] = { 4 };
+    ndfunc_arg_out_t aout[1] = { { numo_cDFloat, 1, out_shape } };
+    ndfunc_t ndf = { (na_iter_func_t)iter_find_split_params_, NO_LOOP, 3, 1, ain, aout };
+    FindSplitParamsOpts_ opts = { std::string(StringValueCStr(criterion)), NUM2DBL(impurity) };
+    VALUE params = na_ndloop3(&ndf, &opts, 3, order, features, targets);
+    RB_GC_GUARD(criterion);
+    return params;
+  }
+
+  /**
+   * @!visibility private
+   * Calculate impurity based on criterion.
+   *
+   * @overload node_impurity(criterion, y) -> Float
+   *
+   * @param criterion [String] The function to calculate impurity. Supported criteria are 'mae' and 'mse'.
+   * @param y [Array<Float>] (shape: [n_samples, n_outputs]) The taget values.
+   * @return [Float] impurity
+   */
+
+  struct NodeImpurityOpts_ {
+    std::string criterion;
+  };
+
+  static void iter_node_impurity_(na_loop_t const* lp) {
+    const double* y = (double*)NDL_PTR(lp, 0);
+    const size_t n_elements = NDL_SHAPE(lp, 0)[0];
+    const size_t n_outputs = NDL_SHAPE(lp, 0)[1];
+    const std::string criterion = ((NodeImpurityOpts_*)lp->opt_ptr)->criterion;
+
+    std::vector<int32_t> order(n_elements);
+    std::vector<double> mean_y(n_outputs, 0);
+    for (size_t i = 0; i < n_elements; i++) {
+      order[i] = static_cast<int32_t>(i);
+      for (size_t j = 0; j < n_outputs; j++) {
+        mean_y[j] += y[i * n_outputs + j];
+      }
+    }
+    for (size_t j = 0; j < n_outputs; j++) {
+      mean_y[j] /= static_cast<double>(n_elements);
+    }
+
+    double* ret = (double*)NDL_PTR(lp, 1);
+    *ret = calc_impurity_(criterion, order.data(), y, mean_y.data(), n_elements, n_outputs, 0);
+  }
+
+  static VALUE node_impurity_(VALUE self, VALUE criterion, VALUE y) {
+    ndfunc_arg_in_t ain[1] = { { numo_cDFloat, 2 } };
+    ndfunc_arg_out_t aout[1] = { { numo_cDFloat, 0 } };
+    ndfunc_t ndf = { (na_iter_func_t)iter_node_impurity_, NO_LOOP | NDF_EXTRACT, 1, 1, ain, aout };
+    NodeImpurityOpts_ opts = { std::string(StringValueCStr(criterion)) };
+    VALUE ret = na_ndloop3(&ndf, &opts, 1, y);
+    RB_GC_GUARD(criterion);
+    return ret;
+  }
+
+  /**
+   * @!visibility private
+   * Check all elements have the same value/vector.
+   *
+   * @overload check_same_value(y) -> Boolean
+   *
+   * @param y [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values.
+   * @return [Boolean]
+   */
+
+  static void iter_check_same_value_(na_loop_t const* lp) {
+    const double* y = (double*)NDL_PTR(lp, 0);
+    const size_t n_elements = NDL_SHAPE(lp, 0)[0];
+    const size_t n_outputs = NDL_SHAPE(lp, 0)[1];
+    VALUE* ret = (VALUE*)NDL_PTR(lp, 1);
+    const double eps = std::numeric_limits<double>::epsilon();
+    *ret = Qtrue;
+    if (n_elements > 0) {
+      for (size_t i = 1; i < n_elements; i++) {
+        for (size_t j = 0; j < n_outputs; j++) {
+          if (std::abs(y[i * n_outputs + j] - y[j]) > eps) {
+            *ret = Qfalse;
+            break;
+          }
+        }
+        if (*ret == Qfalse) break;
+      }
+    }
+  }
+
+  static VALUE check_same_value_(VALUE self, VALUE y) {
+    ndfunc_arg_in_t ain[1] = { { numo_cDFloat, 2 } };
+    ndfunc_arg_out_t aout[1] = { { numo_cRObject, 0 } };
+    ndfunc_t ndf = { (na_iter_func_t)iter_check_same_value_, NO_LOOP | NDF_EXTRACT, 1, 1, ain, aout };
+    return na_ndloop(&ndf, 1, y);
+  }
+};
+
+/**
+ * @!visibility private
+ * Document-module: Rumale::Tree::ExtGradientTreeRegressor
+ * The mixin module consisting of extension method for GradientTreeRegressor class.
+ * This module is used internally.
+ */
+class ExtGradientTreeRegressor {
+public:
+  static void define_module(VALUE& outer) {
+    VALUE rb_mExtGTreeReg = rb_define_module_under(outer, "ExtGradientTreeRegressor");
+    rb_define_private_method(rb_mExtGTreeReg, "find_split_params", find_split_params_, 7);
+  }
+
+private:
+  /**
+   * @!visibility private
+   * Find for split point with maximum information gain.
+   *
+   * @overload find_split_params(order, features, gradients, hessians, sum_gradient, sum_hessian, reg_lambda) -> Array<Float>
+   *   @param order [Numo::Int32] (shape: [n_elements]) The element indices sorted according to feature values.
+   *   @param features [Numo::DFloat] (shape: [n_elements]) The feature values.
+   *   @param gradients [Numo::DFloat] (shape: [n_elements]) The gradient values.
+   *   @param hessians [Numo::DFloat] (shape: [n_elements]) The hessian values.
+   *   @param sum_gradient [Float] The sum of gradient values.
+   *   @param sum_hessian [Float] The sum of hessian values.
+   *   @param reg_lambda [Float] The L2 regularization term on weight.
+   * @return [Array<Float>] The array consists of optimal parameters including threshold and gain.
+   */
+
+  struct FindSplitParamsOpts_ {
+    double sum_gradient;
+    double sum_hessian;
+    double reg_lambda;
+  };
+
+  static void iter_find_split_params_(na_loop_t const* lp) {
+    // Obtain iteration variables.
+    const int32_t* o = (int32_t*)NDL_PTR(lp, 0);
+    const size_t n_elements = NDL_SHAPE(lp, 0)[0];
+    const double* f = (double*)NDL_PTR(lp, 1);
+    const double* g = (double*)NDL_PTR(lp, 2);
+    const double* h = (double*)NDL_PTR(lp, 3);
+    const double s_grad = ((FindSplitParamsOpts_*)lp->opt_ptr)->sum_gradient;
+    const double s_hess = ((FindSplitParamsOpts_*)lp->opt_ptr)->sum_hessian;
+    const double reg_lambda = ((FindSplitParamsOpts_*)lp->opt_ptr)->reg_lambda;
+
+    // Find optimal parameters.
+    size_t curr_pos = 0;
+    size_t next_pos = 0;
+    double curr_el = f[o[0]];
+    const double last_el = f[o[n_elements - 1]];
+    double l_grad = 0.0;
+    double l_hess = 0.0;
+    double threshold = curr_el;
+    double gain_max = 0.0;
+    while (curr_pos < n_elements && curr_el != last_el) {
+      double next_el = f[o[next_pos]];
+      while (next_pos < n_elements && next_el == curr_el) {
+        l_grad += g[o[next_pos]];
+        l_hess += h[o[next_pos]];
+        next_pos++;
+        next_el = f[o[next_pos]];
+      }
+      // Calculate gain of new split.
+      const double r_grad = s_grad - l_grad;
+      const double r_hess = s_hess - l_hess;
+      const double gain = (l_grad * l_grad) / (l_hess + reg_lambda) + (r_grad * r_grad) / (r_hess + reg_lambda) - (s_grad * s_grad) / (s_hess + reg_lambda);
+      // Update optimal parameters.
+      if (gain > gain_max) {
+        threshold = 0.5 * (curr_el + next_el);
+        gain_max = gain;
+      }
+      if (next_pos == n_elements) break;
+      curr_pos = next_pos;
+      curr_el = f[o[curr_pos]];
+    }
+
+    double* params = (double*)NDL_PTR(lp, 4);
+    params[0] = threshold;
+    params[1] = gain_max;
+  }
+
+  static VALUE find_split_params_(VALUE self, VALUE order, VALUE features, VALUE gradients, VALUE hessians,
+                                  VALUE sum_gradient, VALUE sum_hessian, VALUE reg_lambda) {
+    ndfunc_arg_in_t ain[4] = { { numo_cInt32, 1 }, { numo_cDFloat, 1 }, { numo_cDFloat, 1 }, { numo_cDFloat, 1 } };
+    size_t out_shape[1] = { 2 };
+    ndfunc_arg_out_t aout[1] = { { numo_cDFloat, 1, out_shape } };
+    ndfunc_t ndf = { (na_iter_func_t)iter_find_split_params_, NO_LOOP, 4, 1, ain, aout };
+    FindSplitParamsOpts_ opts = { NUM2DBL(sum_gradient), NUM2DBL(sum_hessian), NUM2DBL(reg_lambda) };
+    VALUE params = na_ndloop3(&ndf, &opts, 4, order, features, gradients, hessians);
+    return params;
+  }
+};
+
+#endif /* RUMALE_TREE_EXT_HPP */

data/ext/rumale/tree/extconf.rb

@@ -3,6 +3,8 @@
 require 'mkmf'
 require 'numo/narray'
 
+abort 'libstdc++ is not found.' unless have_library('stdc++')
+
 $LOAD_PATH.each do |lp|
   if File.exist?(File.join(lp, 'numo/numo/narray.h'))
     $INCFLAGS = "-I#{lp}/numo #{$INCFLAGS}"

data/lib/rumale/tree/base_decision_tree.rb

@@ -63,11 +63,7 @@ module Rumale
       end
 
       def build_tree(x, y)
-        y = y.expand_dims(1).dup if y.shape[1].nil?
-        @feature_ids = Array.new(x.shape[1]) { |v| v }
-        @tree = grow_node(0, x, y, impurity(y))
-        @feature_ids = nil
-        nil
+        raise NotImplementedError, "#{__method__} has to be implemented in #{self.class}."
       end
 
       def grow_node(depth, x, y, impurity) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity

data/lib/rumale/tree/decision_tree_regressor.rb

@@ -68,6 +68,7 @@ module Rumale
         @params[:max_features] = [@params[:max_features], n_features].min
         @n_leaves = 0
         @leaf_values = []
+        @feature_ids = Array.new(x.shape[1]) { |v| v }
         @sub_rng = @rng.dup
         build_tree(x, y)
         eval_importance(n_samples, n_features)
@@ -88,8 +89,10 @@ module Rumale
 
       private
 
-      def stop_growing?(y)
-        y.to_a.uniq.size == 1
+      def build_tree(x, y)
+        y = y.expand_dims(1).dup if y.shape[1].nil?
+        @tree = grow_node(0, x, y, impurity(y))
+        nil
       end
 
       def put_leaf(node, y)
@@ -106,7 +109,7 @@ module Rumale
       end
 
       def impurity(y)
-        node_impurity(@params[:criterion], y.to_a)
+        node_impurity(@params[:criterion], y)
       end
     end
   end

data/lib/rumale/tree/version.rb

@@ -5,6 +5,6 @@ module Rumale
   # This module consists of the classes that implement tree models.
   module Tree
     # @!visibility private
-    VERSION = '0.26.0'
+    VERSION = '0.28.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rumale-tree
 version: !ruby/object:Gem::Version
-  version: 0.26.0
+  version: 0.28.0
 platform: ruby
 authors:
 - yoshoku
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-02-19 00:00:00.000000000 Z
+date: 2023-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.26.0
+        version: 0.28.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.26.0
+        version: 0.28.0
 description: Rumale::Tree provides classifier and regression based on decision tree
   algorithms with Rumale interface.
 email:
@@ -49,8 +49,8 @@ extra_rdoc_files: []
 files:
 - LICENSE.txt
 - README.md
-- ext/rumale/tree/ext.c
-- ext/rumale/tree/ext.h
+- ext/rumale/tree/ext.cpp
+- ext/rumale/tree/ext.hpp
 - ext/rumale/tree/extconf.rb
 - lib/rumale/tree.rb
 - lib/rumale/tree/base_decision_tree.rb
@@ -85,7 +85,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.4.20
 signing_key:
 specification_version: 4
 summary: Rumale::Tree provides classifier and regression based on decision tree algorithms

data/ext/rumale/tree/ext.c

@@ -1,575 +0,0 @@
-#include "ext.h"
-
-double* alloc_dbl_array(const long n_dimensions) {
-  double* arr = ALLOC_N(double, n_dimensions);
-  memset(arr, 0, n_dimensions * sizeof(double));
-  return arr;
-}
-
-double calc_gini_coef(double* histogram, const long n_elements, const long n_classes) {
-  double gini = 0.0;
-
-  for (long i = 0; i < n_classes; i++) {
-    double el = histogram[i] / n_elements;
-    gini += el * el;
-  }
-
-  return 1.0 - gini;
-}
-
-double calc_entropy(double* histogram, const long n_elements, const long n_classes) {
-  double entropy = 0.0;
-
-  for (long i = 0; i < n_classes; i++) {
-    double el = histogram[i] / n_elements;
-    entropy += el * log(el + 1.0);
-  }
-
-  return -entropy;
-}
-
-VALUE
-calc_mean_vec(double* sum_vec, const long n_dimensions, const long n_elements) {
-  VALUE mean_vec = rb_ary_new2(n_dimensions);
-
-  for (long i = 0; i < n_dimensions; i++) {
-    rb_ary_store(mean_vec, i, DBL2NUM(sum_vec[i] / n_elements));
-  }
-
-  return mean_vec;
-}
-
-double calc_vec_mae(VALUE vec_a, VALUE vec_b) {
-  const long n_dimensions = RARRAY_LEN(vec_a);
-  double sum = 0.0;
-
-  for (long i = 0; i < n_dimensions; i++) {
-    double diff = NUM2DBL(rb_ary_entry(vec_a, i)) - NUM2DBL(rb_ary_entry(vec_b, i));
-    sum += fabs(diff);
-  }
-
-  return sum / n_dimensions;
-}
-
-double calc_vec_mse(VALUE vec_a, VALUE vec_b) {
-  const long n_dimensions = RARRAY_LEN(vec_a);
-  double sum = 0.0;
-
-  for (long i = 0; i < n_dimensions; i++) {
-    double diff = NUM2DBL(rb_ary_entry(vec_a, i)) - NUM2DBL(rb_ary_entry(vec_b, i));
-    sum += diff * diff;
-  }
-
-  return sum / n_dimensions;
-}
-
-double calc_mae(VALUE target_vecs, VALUE mean_vec) {
-  const long n_elements = RARRAY_LEN(target_vecs);
-  double sum = 0.0;
-
-  for (long i = 0; i < n_elements; i++) {
-    sum += calc_vec_mae(rb_ary_entry(target_vecs, i), mean_vec);
-  }
-
-  return sum / n_elements;
-}
-
-double calc_mse(VALUE target_vecs, VALUE mean_vec) {
-  const long n_elements = RARRAY_LEN(target_vecs);
-  double sum = 0.0;
-
-  for (long i = 0; i < n_elements; i++) {
-    sum += calc_vec_mse(rb_ary_entry(target_vecs, i), mean_vec);
-  }
-
-  return sum / n_elements;
-}
-
-double calc_impurity_cls(const char* criterion, double* histogram, const long n_elements, const long n_classes) {
-  if (strcmp(criterion, "entropy") == 0) {
-    return calc_entropy(histogram, n_elements, n_classes);
-  }
-  return calc_gini_coef(histogram, n_elements, n_classes);
-}
-
-double calc_impurity_reg(const char* criterion, VALUE target_vecs, double* sum_vec) {
-  const long n_elements = RARRAY_LEN(target_vecs);
-  const long n_dimensions = RARRAY_LEN(rb_ary_entry(target_vecs, 0));
-  VALUE mean_vec = calc_mean_vec(sum_vec, n_dimensions, n_elements);
-
-  if (strcmp(criterion, "mae") == 0) {
-    return calc_mae(target_vecs, mean_vec);
-  }
-  return calc_mse(target_vecs, mean_vec);
-}
-
-void add_sum_vec(double* sum_vec, VALUE target) {
-  const long n_dimensions = RARRAY_LEN(target);
-
-  for (long i = 0; i < n_dimensions; i++) {
-    sum_vec[i] += NUM2DBL(rb_ary_entry(target, i));
-  }
-}
-
-void sub_sum_vec(double* sum_vec, VALUE target) {
-  const long n_dimensions = RARRAY_LEN(target);
-
-  for (long i = 0; i < n_dimensions; i++) {
-    sum_vec[i] -= NUM2DBL(rb_ary_entry(target, i));
-  }
-}
-
-/**
- * @!visibility private
- */
-typedef struct {
-  char* criterion;
-  long n_classes;
-  double impurity;
-} split_opts_cls;
-
-/**
- * @!visibility private
- */
-static void iter_find_split_params_cls(na_loop_t const* lp) {
-  const int32_t* o = (int32_t*)NDL_PTR(lp, 0);
-  const double* f = (double*)NDL_PTR(lp, 1);
-  const int32_t* y = (int32_t*)NDL_PTR(lp, 2);
-  const long n_elements = NDL_SHAPE(lp, 0)[0];
-  const char* criterion = ((split_opts_cls*)lp->opt_ptr)->criterion;
-  const long n_classes = ((split_opts_cls*)lp->opt_ptr)->n_classes;
-  const double w_impurity = ((split_opts_cls*)lp->opt_ptr)->impurity;
-  double* params = (double*)NDL_PTR(lp, 3);
-  long curr_pos = 0;
-  long next_pos = 0;
-  long n_l_elements = 0;
-  long n_r_elements = n_elements;
-  double curr_el = f[o[0]];
-  double last_el = f[o[n_elements - 1]];
-  double next_el;
-  double l_impurity;
-  double r_impurity;
-  double gain;
-  double* l_histogram = alloc_dbl_array(n_classes);
-  double* r_histogram = alloc_dbl_array(n_classes);
-
-  /* Initialize optimal parameters. */
-  params[0] = 0.0;        /* left impurity */
-  params[1] = w_impurity; /* right impurity */
-  params[2] = curr_el;    /* threshold */
-  params[3] = 0.0;        /* gain */
-
-  /* Initialize child node variables. */
-  for (long i = 0; i < n_elements; i++) {
-    r_histogram[y[o[i]]] += 1.0;
-  }
-
-  /* Find optimal parameters. */
-  while (curr_pos < n_elements && curr_el != last_el) {
-    next_el = f[o[next_pos]];
-    while (next_pos < n_elements && next_el == curr_el) {
-      l_histogram[y[o[next_pos]]] += 1;
-      n_l_elements++;
-      r_histogram[y[o[next_pos]]] -= 1;
-      n_r_elements--;
-      next_pos++;
-      next_el = f[o[next_pos]];
-    }
-    /* Calculate gain of new split. */
-    l_impurity = calc_impurity_cls(criterion, l_histogram, n_l_elements, n_classes);
-    r_impurity = calc_impurity_cls(criterion, r_histogram, n_r_elements, n_classes);
-    gain = w_impurity - (n_l_elements * l_impurity + n_r_elements * r_impurity) / n_elements;
-    /* Update optimal parameters. */
-    if (gain > params[3]) {
-      params[0] = l_impurity;
-      params[1] = r_impurity;
-      params[2] = 0.5 * (curr_el + next_el);
-      params[3] = gain;
-    }
-    if (next_pos == n_elements) break;
-    curr_pos = next_pos;
-    curr_el = f[o[curr_pos]];
-  }
-
-  xfree(l_histogram);
-  xfree(r_histogram);
-}
-
-/**
- * @!visibility private
- * Find for split point with maximum information gain.
- *
- * @overload find_split_params(criterion, impurity, order, features, labels, n_classes) -> Array<Float>
- *
- * @param criterion [String] The function to evaluate spliting point. Supported criteria are 'gini' and 'entropy'.
- * @param impurity [Float] The impurity of whole dataset.
- * @param order [Numo::Int32] (shape: [n_elements]) The element indices sorted according to feature values.
- * @param features [Numo::DFloat] (shape: [n_elements]) The feature values.
- * @param labels [Numo::Int32] (shape: [n_elements]) The labels.
- * @param n_classes [Integer] The number of classes.
- * @return [Array<Float>] The array consists of optimal parameters including impurities of child nodes, threshold, and gain.
- */
-static VALUE find_split_params_cls(VALUE self, VALUE criterion, VALUE impurity, VALUE order, VALUE features, VALUE labels,
-                                   VALUE n_classes) {
-  ndfunc_arg_in_t ain[3] = {{numo_cInt32, 1}, {numo_cDFloat, 1}, {numo_cInt32, 1}};
-  size_t out_shape[1] = {4};
-  ndfunc_arg_out_t aout[1] = {{numo_cDFloat, 1, out_shape}};
-  ndfunc_t ndf = {(na_iter_func_t)iter_find_split_params_cls, NO_LOOP, 3, 1, ain, aout};
-  split_opts_cls opts = {StringValuePtr(criterion), NUM2LONG(n_classes), NUM2DBL(impurity)};
-  VALUE params = na_ndloop3(&ndf, &opts, 3, order, features, labels);
-  VALUE results = rb_ary_new2(4);
-  double* params_ptr = (double*)na_get_pointer_for_read(params);
-  rb_ary_store(results, 0, DBL2NUM(params_ptr[0]));
-  rb_ary_store(results, 1, DBL2NUM(params_ptr[1]));
-  rb_ary_store(results, 2, DBL2NUM(params_ptr[2]));
-  rb_ary_store(results, 3, DBL2NUM(params_ptr[3]));
-  RB_GC_GUARD(params);
-  RB_GC_GUARD(criterion);
-  return results;
-}
-
-/**
- * @!visibility private
- */
-typedef struct {
-  char* criterion;
-  double impurity;
-} split_opts_reg;
-
-/**
- * @!visibility private
- */
-static void iter_find_split_params_reg(na_loop_t const* lp) {
-  const int32_t* o = (int32_t*)NDL_PTR(lp, 0);
-  const double* f = (double*)NDL_PTR(lp, 1);
-  const double* y = (double*)NDL_PTR(lp, 2);
-  const long n_elements = NDL_SHAPE(lp, 0)[0];
-  const long n_outputs = NDL_SHAPE(lp, 2)[1];
-  const char* criterion = ((split_opts_reg*)lp->opt_ptr)->criterion;
-  const double w_impurity = ((split_opts_reg*)lp->opt_ptr)->impurity;
-  double* params = (double*)NDL_PTR(lp, 3);
-  long curr_pos = 0;
-  long next_pos = 0;
-  long n_l_elements = 0;
-  long n_r_elements = n_elements;
-  double curr_el = f[o[0]];
-  double last_el = f[o[n_elements - 1]];
-  double next_el;
-  double l_impurity;
-  double r_impurity;
-  double gain;
-  double* l_sum_vec = alloc_dbl_array(n_outputs);
-  double* r_sum_vec = alloc_dbl_array(n_outputs);
-  double target_var;
-  VALUE l_target_vecs = rb_ary_new();
-  VALUE r_target_vecs = rb_ary_new();
-  VALUE target;
-
-  /* Initialize optimal parameters. */
-  params[0] = 0.0;        /* left impurity */
-  params[1] = w_impurity; /* right impurity */
-  params[2] = curr_el;    /* threshold */
-  params[3] = 0.0;        /* gain */
-
-  /* Initialize child node variables. */
-  for (long i = 0; i < n_elements; i++) {
-    target = rb_ary_new2(n_outputs);
-    for (long j = 0; j < n_outputs; j++) {
-      target_var = y[o[i] * n_outputs + j];
-      rb_ary_store(target, j, DBL2NUM(target_var));
-      r_sum_vec[j] += target_var;
-    }
-    rb_ary_push(r_target_vecs, target);
-  }
-
-  /* Find optimal parameters. */
-  while (curr_pos < n_elements && curr_el != last_el) {
-    next_el = f[o[next_pos]];
-    while (next_pos < n_elements && next_el == curr_el) {
-      target = rb_ary_shift(r_target_vecs);
-      n_r_elements--;
-      sub_sum_vec(r_sum_vec, target);
-      rb_ary_push(l_target_vecs, target);
-      n_l_elements++;
-      add_sum_vec(l_sum_vec, target);
-      next_pos++;
-      next_el = f[o[next_pos]];
-    }
-    /* Calculate gain of new split. */
-    l_impurity = calc_impurity_reg(criterion, l_target_vecs, l_sum_vec);
-    r_impurity = calc_impurity_reg(criterion, r_target_vecs, r_sum_vec);
-    gain = w_impurity - (n_l_elements * l_impurity + n_r_elements * r_impurity) / n_elements;
-    /* Update optimal parameters. */
-    if (gain > params[3]) {
-      params[0] = l_impurity;
-      params[1] = r_impurity;
-      params[2] = 0.5 * (curr_el + next_el);
-      params[3] = gain;
-    }
-    if (next_pos == n_elements) break;
-    curr_pos = next_pos;
-    curr_el = f[o[curr_pos]];
-  }
-
-  xfree(l_sum_vec);
-  xfree(r_sum_vec);
-}
-
-/**
- * @!visibility private
- * Find for split point with maximum information gain.
- *
- * @overload find_split_params(criterion, impurity, order, features, targets) -> Array<Float>
- *
- * @param criterion [String] The function to evaluate spliting point. Supported criteria are 'mae' and 'mse'.
- * @param impurity [Float] The impurity of whole dataset.
- * @param order [Numo::Int32] (shape: [n_samples]) The element indices sorted according to feature values in ascending order.
- * @param features [Numo::DFloat] (shape: [n_samples]) The feature values.
- * @param targets [Numo::DFloat] (shape: [n_samples, n_outputs]) The target values.
- * @return [Array<Float>] The array consists of optimal parameters including impurities of child nodes, threshold, and gain.
- */
-static VALUE find_split_params_reg(VALUE self, VALUE criterion, VALUE impurity, VALUE order, VALUE features, VALUE targets) {
-  ndfunc_arg_in_t ain[3] = {{numo_cInt32, 1}, {numo_cDFloat, 1}, {numo_cDFloat, 2}};
-  size_t out_shape[1] = {4};
-  ndfunc_arg_out_t aout[1] = {{numo_cDFloat, 1, out_shape}};
-  ndfunc_t ndf = {(na_iter_func_t)iter_find_split_params_reg, NO_LOOP, 3, 1, ain, aout};
-  split_opts_reg opts = {StringValuePtr(criterion), NUM2DBL(impurity)};
-  VALUE params = na_ndloop3(&ndf, &opts, 3, order, features, targets);
-  VALUE results = rb_ary_new2(4);
-  double* params_ptr = (double*)na_get_pointer_for_read(params);
-  rb_ary_store(results, 0, DBL2NUM(params_ptr[0]));
-  rb_ary_store(results, 1, DBL2NUM(params_ptr[1]));
-  rb_ary_store(results, 2, DBL2NUM(params_ptr[2]));
-  rb_ary_store(results, 3, DBL2NUM(params_ptr[3]));
-  RB_GC_GUARD(params);
-  RB_GC_GUARD(criterion);
-  return results;
-}
-
-/**
- * @!visibility private
- */
-static void iter_find_split_params_grad_reg(na_loop_t const* lp) {
-  const int32_t* o = (int32_t*)NDL_PTR(lp, 0);
-  const double* f = (double*)NDL_PTR(lp, 1);
-  const double* g = (double*)NDL_PTR(lp, 2);
-  const double* h = (double*)NDL_PTR(lp, 3);
-  const double s_grad = ((double*)lp->opt_ptr)[0];
-  const double s_hess = ((double*)lp->opt_ptr)[1];
-  const double reg_lambda = ((double*)lp->opt_ptr)[2];
-  const long n_elements = NDL_SHAPE(lp, 0)[0];
-  double* params = (double*)NDL_PTR(lp, 4);
-  long curr_pos = 0;
-  long next_pos = 0;
-  double curr_el = f[o[0]];
-  double last_el = f[o[n_elements - 1]];
-  double next_el;
-  double l_grad = 0.0;
-  double l_hess = 0.0;
-  double r_grad;
-  double r_hess;
-  double threshold = curr_el;
-  double gain_max = 0.0;
-  double gain;
-
-  /* Find optimal parameters. */
-  while (curr_pos < n_elements && curr_el != last_el) {
-    next_el = f[o[next_pos]];
-    while (next_pos < n_elements && next_el == curr_el) {
-      l_grad += g[o[next_pos]];
-      l_hess += h[o[next_pos]];
-      next_pos++;
-      next_el = f[o[next_pos]];
-    }
-    /* Calculate gain of new split. */
-    r_grad = s_grad - l_grad;
-    r_hess = s_hess - l_hess;
-    gain = (l_grad * l_grad) / (l_hess + reg_lambda) + (r_grad * r_grad) / (r_hess + reg_lambda) -
-           (s_grad * s_grad) / (s_hess + reg_lambda);
-    /* Update optimal parameters. */
-    if (gain > gain_max) {
-      threshold = 0.5 * (curr_el + next_el);
-      gain_max = gain;
-    }
-    if (next_pos == n_elements) {
-      break;
-    }
-    curr_pos = next_pos;
-    curr_el = f[o[curr_pos]];
-  }
-
-  params[0] = threshold;
-  params[1] = gain_max;
-}
-
-/**
- * @!visibility private
- * Find for split point with maximum information gain.
- *
- * @overload find_split_params(order, features, gradients, hessians, sum_gradient, sum_hessian, reg_lambda) -> Array<Float>
- *   @param order [Numo::Int32] (shape: [n_elements]) The element indices sorted according to feature values.
- *   @param features [Numo::DFloat] (shape: [n_elements]) The feature values.
- *   @param gradients [Numo::DFloat] (shape: [n_elements]) The gradient values.
- *   @param hessians [Numo::DFloat] (shape: [n_elements]) The hessian values.
- *   @param sum_gradient [Float] The sum of gradient values.
- *   @param sum_hessian [Float] The sum of hessian values.
- *   @param reg_lambda [Float] The L2 regularization term on weight.
- * @return [Array<Float>] The array consists of optimal parameters including threshold and gain.
- */
-static VALUE find_split_params_grad_reg(VALUE self, VALUE order, VALUE features, VALUE gradients, VALUE hessians,
-                                        VALUE sum_gradient, VALUE sum_hessian, VALUE reg_lambda) {
-  ndfunc_arg_in_t ain[4] = {{numo_cInt32, 1}, {numo_cDFloat, 1}, {numo_cDFloat, 1}, {numo_cDFloat, 1}};
-  size_t out_shape[1] = {2};
-  ndfunc_arg_out_t aout[1] = {{numo_cDFloat, 1, out_shape}};
-  ndfunc_t ndf = {(na_iter_func_t)iter_find_split_params_grad_reg, NO_LOOP, 4, 1, ain, aout};
-  double opts[3] = {NUM2DBL(sum_gradient), NUM2DBL(sum_hessian), NUM2DBL(reg_lambda)};
-  VALUE params = na_ndloop3(&ndf, opts, 4, order, features, gradients, hessians);
-  VALUE results = rb_ary_new2(2);
-  double* params_ptr = (double*)na_get_pointer_for_read(params);
-  rb_ary_store(results, 0, DBL2NUM(params_ptr[0]));
-  rb_ary_store(results, 1, DBL2NUM(params_ptr[1]));
-  RB_GC_GUARD(params);
-  return results;
-}
-
-/**
- * @!visibility private
- */
-typedef struct {
-  char* criterion;
-  long n_classes;
-} node_impurity_cls_opts;
-
-/**
- * @!visibility private
- */
-static void iter_node_impurity_cls(na_loop_t const* lp) {
-  const int32_t* y = (int32_t*)NDL_PTR(lp, 0);
-  const char* criterion = ((node_impurity_cls_opts*)lp->opt_ptr)->criterion;
-  const long n_classes = ((node_impurity_cls_opts*)lp->opt_ptr)->n_classes;
-  const long n_elements = NDL_SHAPE(lp, 0)[0];
-  double* ret = (double*)NDL_PTR(lp, 1);
-  double* histogram = alloc_dbl_array(n_classes);
-  for (long i = 0; i < n_elements; i++) histogram[y[i]] += 1;
-  *ret = calc_impurity_cls(criterion, histogram, n_elements, n_classes);
-  xfree(histogram);
-}
-
-/**
- * @!visibility private
- * Calculate impurity based on criterion.
- *
- * @overload node_impurity(criterion, y, n_classes) -> Float
- *
- * @param criterion [String] The function to calculate impurity. Supported criteria are 'gini' and 'entropy'.
- * @param y [Numo::Int32] (shape: [n_samples]) The labels.
- * @param n_classes [Integer] The number of classes.
- * @return [Float] impurity
- */
-static VALUE node_impurity_cls(VALUE self, VALUE criterion, VALUE y, VALUE n_classes) {
-  ndfunc_arg_in_t ain[1] = {{numo_cInt32, 1}};
-  ndfunc_arg_out_t aout[1] = {{numo_cDFloat, 0}};
-  ndfunc_t ndf = {(na_iter_func_t)iter_node_impurity_cls, NDF_EXTRACT, 1, 1, ain, aout};
-  node_impurity_cls_opts opts = {StringValuePtr(criterion), NUM2LONG(n_classes)};
-  VALUE ret = na_ndloop3(&ndf, &opts, 1, y);
-  RB_GC_GUARD(criterion);
-  return ret;
-}
-
-/**
- * @!visibility private
- */
-static void iter_check_same_label(na_loop_t const* lp) {
-  const int32_t* y = (int32_t*)NDL_PTR(lp, 0);
-  const long n_elements = NDL_SHAPE(lp, 0)[0];
-  int32_t* ret = (int32_t*)NDL_PTR(lp, 1);
-  *ret = 1;
-  if (n_elements > 0) {
-    int32_t label = y[0];
-    for (long i = 0; i < n_elements; i++) {
-      if (y[i] != label) {
-        *ret = 0;
-        break;
-      }
-    }
-  }
-}
-
-/**
- * @!visibility private
- * Check all elements have the save value.
- *
- * @overload check_same_label(y) -> Boolean
- *
- * @param y [Numo::Int32] (shape: [n_samples]) The labels.
- * @return [Boolean]
- */
-static VALUE check_same_label(VALUE self, VALUE y) {
-  ndfunc_arg_in_t ain[1] = {{numo_cInt32, 1}};
-  ndfunc_arg_out_t aout[1] = {{numo_cInt32, 0}};
-  ndfunc_t ndf = {(na_iter_func_t)iter_check_same_label, NDF_EXTRACT, 1, 1, ain, aout};
-  VALUE ret = na_ndloop(&ndf, 1, y);
-  return (NUM2INT(ret) == 1 ? Qtrue : Qfalse);
-}
-
-/**
- * @!visibility private
- * Calculate impurity based on criterion.
- *
- * @overload node_impurity(criterion, y) -> Float
- *
- * @param criterion [String] The function to calculate impurity. Supported criteria are 'mae' and 'mse'.
- * @param y [Array<Float>] (shape: [n_samples, n_outputs]) The taget values.
- * @return [Float] impurity
- */
-static VALUE node_impurity_reg(VALUE self, VALUE criterion, VALUE y) {
-  const long n_elements = RARRAY_LEN(y);
-  const long n_outputs = RARRAY_LEN(rb_ary_entry(y, 0));
-  double* sum_vec = alloc_dbl_array(n_outputs);
-  VALUE target_vecs = rb_ary_new();
-
-  for (long i = 0; i < n_elements; i++) {
-    VALUE target = rb_ary_entry(y, i);
-    add_sum_vec(sum_vec, target);
-    rb_ary_push(target_vecs, target);
-  }
-
-  VALUE ret = DBL2NUM(calc_impurity_reg(StringValuePtr(criterion), target_vecs, sum_vec));
-  xfree(sum_vec);
-  RB_GC_GUARD(criterion);
-  return ret;
-}
-
-void Init_ext(void) {
-  VALUE rb_mRumale = rb_define_module("Rumale");
-  VALUE rb_mTree = rb_define_module_under(rb_mRumale, "Tree");
-
-  /**
-   * Document-module: Rumale::Tree::ExtDecisionTreeClassifier
-   * @!visibility private
-   * The mixin module consisting of extension method for DecisionTreeClassifier class.
-   * This module is used internally.
-   */
-  VALUE rb_mExtDTreeCls = rb_define_module_under(rb_mTree, "ExtDecisionTreeClassifier");
-  /**
-   * Document-module: Rumale::Tree::ExtDecisionTreeRegressor
-   * @!visibility private
-   * The mixin module consisting of extension method for DecisionTreeRegressor class.
-   * This module is used internally.
-   */
-  VALUE rb_mExtDTreeReg = rb_define_module_under(rb_mTree, "ExtDecisionTreeRegressor");
-  /**
-   * Document-module: Rumale::Tree::ExtGradientTreeRegressor
-   * @!visibility private
-   * The mixin module consisting of extension method for GradientTreeRegressor class.
-   * This module is used internally.
-   */
-  VALUE rb_mExtGTreeReg = rb_define_module_under(rb_mTree, "ExtGradientTreeRegressor");
-
-  rb_define_private_method(rb_mExtDTreeCls, "find_split_params", find_split_params_cls, 6);
-  rb_define_private_method(rb_mExtDTreeReg, "find_split_params", find_split_params_reg, 5);
-  rb_define_private_method(rb_mExtGTreeReg, "find_split_params", find_split_params_grad_reg, 7);
-  rb_define_private_method(rb_mExtDTreeCls, "node_impurity", node_impurity_cls, 3);
-  rb_define_private_method(rb_mExtDTreeCls, "stop_growing?", check_same_label, 1);
-  rb_define_private_method(rb_mExtDTreeReg, "node_impurity", node_impurity_reg, 2);
-}

data/ext/rumale/tree/ext.h

@@ -1,12 +0,0 @@
-#ifndef RUMALE_TREE_EXT_H
-#define RUMALE_TREE_EXT_H 1
-
-#include <math.h>
-#include <string.h>
-
-#include <ruby.h>
-
-#include <numo/narray.h>
-#include <numo/template.h>
-
-#endif /* RUMALE_TREE_EXT_H */