xgb 0.8.0 → 0.10.0

data/lib/xgboost/dmatrix.rb

@@ -1,77 +1,70 @@
 module XGBoost
   class DMatrix
-    attr_reader :data, :feature_names, :feature_types
+    include Utils
+
+    attr_reader :handle
 
     def initialize(data, label: nil, weight: nil, missing: Float::NAN)
-      @data = data
-
-      @handle = ::FFI::MemoryPointer.new(:pointer)
-
-      if data
-        if matrix?(data)
-          nrow = data.row_count
-          ncol = data.column_count
-          flat_data = data.to_a.flatten
-        elsif daru?(data)
-          nrow, ncol = data.shape
-          flat_data = data.map_rows(&:to_a).flatten
-          @feature_names = data.each_vector.map(&:name)
-          @feature_types =
-            data.each_vector.map(&:db_type).map do |v|
-              case v
-              when "INTEGER"
-                "int"
-              when "DOUBLE"
-                "float"
-              else
-                raise Error, "Unknown feature type: #{v}"
-              end
+      if data.is_a?(::FFI::AutoPointer)
+        @handle = data
+        return
+      end
+
+      if matrix?(data)
+        nrow = data.row_count
+        ncol = data.column_count
+        flat_data = data.to_a.flatten
+      elsif daru?(data)
+        nrow, ncol = data.shape
+        flat_data = data.map_rows(&:to_a).flatten
+        feature_names = data.each_vector.map(&:name)
+        feature_types =
+          data.each_vector.map(&:db_type).map do |v|
+            case v
+            when "INTEGER"
+              "int"
+            when "DOUBLE"
+              "float"
+            else
+              raise Error, "Unknown feature type: #{v}"
             end
-        elsif numo?(data)
-          nrow, ncol = data.shape
-        elsif rover?(data)
-          nrow, ncol = data.shape
-          @feature_names = data.keys
-          data = data.to_numo
-        else
-          nrow = data.count
-          ncol = data.first.count
-          if !data.all? { |r| r.size == ncol }
-            # TODO raise ArgumentError in 0.8.0
-            raise IndexError, "Rows have different sizes"
           end
-          flat_data = data.flatten
+      elsif numo?(data)
+        nrow, ncol = data.shape
+      elsif rover?(data)
+        nrow, ncol = data.shape
+        feature_names = data.keys
+        data = data.to_numo
+      else
+        nrow = data.count
+        ncol = data.first.count
+        if !data.all? { |r| r.size == ncol }
+          raise ArgumentError, "Rows have different sizes"
         end
+        flat_data = data.flatten
+      end
 
-        c_data = ::FFI::MemoryPointer.new(:float, nrow * ncol)
-        if numo?(data)
-          c_data.write_bytes(data.cast_to(Numo::SFloat).to_string)
-        else
-          handle_missing(flat_data, missing)
-          c_data.write_array_of_float(flat_data)
-        end
-        check_result FFI.XGDMatrixCreateFromMat(c_data, nrow, ncol, missing, @handle)
+      c_data = ::FFI::MemoryPointer.new(:float, nrow * ncol)
+      if numo?(data)
+        c_data.write_bytes(data.cast_to(Numo::SFloat).to_string)
+      else
+        handle_missing(flat_data, missing)
+        c_data.write_array_of_float(flat_data)
+      end
 
-        ObjectSpace.define_finalizer(@handle, self.class.finalize(handle_pointer.to_i))
+      out = ::FFI::MemoryPointer.new(:pointer)
+      check_call FFI.XGDMatrixCreateFromMat(c_data, nrow, ncol, missing, out)
+      @handle = ::FFI::AutoPointer.new(out.read_pointer, FFI.method(:XGDMatrixFree))
 
-        @feature_names ||= ncol.times.map { |i| "f#{i}" }
-      end
+      self.feature_names = feature_names || ncol.times.map { |i| "f#{i}" }
+      self.feature_types = feature_types if feature_types
 
       self.label = label if label
       self.weight = weight if weight
     end
 
-    def self.finalize(addr)
-      # must use proc instead of stabby lambda
-      proc { FFI.XGDMatrixFree(::FFI::Pointer.new(:pointer, addr)) }
-    end
-
-    def label
-      float_info("label")
-    end
-
-    def weight
-      float_info("weight")
+    def save_binary(fname, silent: true)
+      check_call FFI.XGDMatrixSaveBinary(handle, fname, silent ? 1 : 0)
     end
 
     def label=(label)
@@ -85,39 +78,146 @@ module XGBoost
     def group=(group)
       c_data = ::FFI::MemoryPointer.new(:int, group.size)
       c_data.write_array_of_int(group)
-      check_result FFI.XGDMatrixSetUIntInfo(handle_pointer, "group", c_data, group.size)
+      check_call FFI.XGDMatrixSetUIntInfo(handle, "group", c_data, group.size)
+    end
+
+    def label
+      float_info("label")
+    end
+
+    def weight
+      float_info("weight")
     end
 
     def num_row
       out = ::FFI::MemoryPointer.new(:uint64)
-      check_result FFI.XGDMatrixNumRow(handle_pointer, out)
-      read_uint64(out)
+      check_call FFI.XGDMatrixNumRow(handle, out)
+      out.read_uint64
     end
 
     def num_col
       out = ::FFI::MemoryPointer.new(:uint64)
-      check_result FFI.XGDMatrixNumCol(handle_pointer, out)
-      read_uint64(out)
+      check_call FFI.XGDMatrixNumCol(handle, out)
+      out.read_uint64
+    end
+
+    def num_nonmissing
+      out = ::FFI::MemoryPointer.new(:uint64)
+      check_call FFI.XGDMatrixNumNonMissing(handle, out)
+      out.read_uint64
+    end
+
+    def data_split_mode
+      out = ::FFI::MemoryPointer.new(:uint64)
+      check_call FFI.XGDMatrixDataSplitMode(handle, out)
+      out.read_uint64 == 0 ? :row : :col
     end
 
     def slice(rindex)
-      res = DMatrix.new(nil)
       idxset = ::FFI::MemoryPointer.new(:int, rindex.count)
       idxset.write_array_of_int(rindex)
-      check_result FFI.XGDMatrixSliceDMatrix(handle_pointer, idxset, rindex.size, res.handle)
-      res
+      out = ::FFI::MemoryPointer.new(:pointer)
+      check_call FFI.XGDMatrixSliceDMatrix(handle, idxset, rindex.size, out)
+
+      handle = ::FFI::AutoPointer.new(out.read_pointer, FFI.method(:XGDMatrixFree))
+      DMatrix.new(handle)
     end
 
-    def save_binary(fname, silent: true)
-      check_result FFI.XGDMatrixSaveBinary(handle_pointer, fname, silent ? 1 : 0)
+    def feature_names
+      length = ::FFI::MemoryPointer.new(:uint64)
+      sarr = ::FFI::MemoryPointer.new(:pointer)
+      check_call(
+        FFI.XGDMatrixGetStrFeatureInfo(
+          handle,
+          "feature_name",
+          length,
+          sarr
+        )
+      )
+      feature_names = from_cstr_to_rbstr(sarr, length)
+      feature_names.empty? ? nil : feature_names
     end
 
-    def handle
-      @handle
+    def feature_names=(feature_names)
+      if feature_names.nil?
+        check_call(
+          FFI.XGDMatrixSetStrFeatureInfo(
+            handle, "feature_name", nil, 0
+          )
+        )
+        return
+      end
+
+      # validate feature name
+      feature_names =
+        validate_feature_info(
+          feature_names,
+          num_col,
+          data_split_mode == :col,
+          "feature names"
+        )
+      if feature_names.length != feature_names.uniq.length
+        raise ArgumentError, "feature_names must be unique"
+      end
+
+      # prohibit the use symbols that may affect parsing. e.g. []<
+      if !feature_names.all? { |f| f.is_a?(String) && !["[", "]", "<"].any? { |x| f.include?(x) } }
+        raise ArgumentError, "feature_names must be string, and may not contain [, ] or <"
+      end
+
+      c_feature_names = array_of_pointers(feature_names.map { |f| string_pointer(f) })
+      check_call(
+        FFI.XGDMatrixSetStrFeatureInfo(
+          handle,
+          "feature_name",
+          c_feature_names,
+          feature_names.length
+        )
+      )
+    end
+
+    def feature_types
+      length = ::FFI::MemoryPointer.new(:uint64)
+      sarr = ::FFI::MemoryPointer.new(:pointer)
+      check_call(
+        FFI.XGDMatrixGetStrFeatureInfo(
+          handle,
+          "feature_type",
+          length,
+          sarr
+        )
+      )
+      res = from_cstr_to_rbstr(sarr, length)
+      res.empty? ? nil : res
     end
 
-    def handle_pointer
-      @handle.read_pointer
+    def feature_types=(feature_types)
+      if feature_types.nil?
+        check_call(
+          FFI.XGDMatrixSetStrFeatureInfo(
+            handle, "feature_type", nil, 0
+          )
+        )
+        return
+      end
+
+      feature_types =
+        validate_feature_info(
+          feature_types,
+          num_col,
+          data_split_mode == :col,
+          "feature types"
+        )
+
+      c_feature_types = array_of_pointers(feature_types.map { |f| string_pointer(f) })
+      check_call(
+        FFI.XGDMatrixSetStrFeatureInfo(
+          handle,
+          "feature_type",
+          c_feature_types,
+          feature_types.length
+        )
+      )
     end
 
     private
@@ -126,17 +226,31 @@ module XGBoost
       data = data.to_a unless data.is_a?(Array)
       c_data = ::FFI::MemoryPointer.new(:float, data.size)
       c_data.write_array_of_float(data)
-      check_result FFI.XGDMatrixSetFloatInfo(handle_pointer, field.to_s, c_data, data.size)
+      check_call FFI.XGDMatrixSetFloatInfo(handle, field.to_s, c_data, data.size)
     end
 
     def float_info(field)
       num_row ||= num_row()
-      out_len = ::FFI::MemoryPointer.new(:int)
+      out_len = ::FFI::MemoryPointer.new(:uint64)
       out_dptr = ::FFI::MemoryPointer.new(:float, num_row)
-      check_result FFI.XGDMatrixGetFloatInfo(handle_pointer, field, out_len, out_dptr)
+      check_call FFI.XGDMatrixGetFloatInfo(handle, field, out_len, out_dptr)
       out_dptr.read_pointer.read_array_of_float(num_row)
     end
 
+    def validate_feature_info(feature_info, n_features, is_column_split, name)
+      if !feature_info.is_a?(Array)
+        raise TypeError, "Expecting an array of strings for #{name}, got: #{feature_info.class.name}"
+      end
+      if feature_info.length != n_features && n_features != 0 && !is_column_split
+        msg = (
+          "#{name} must have the same length as the number of data columns, " +
+          "expected #{n_features}, got #{feature_info.length}"
+        )
+        raise ArgumentError, msg
+      end
+      feature_info
+    end
+
     def matrix?(data)
       defined?(Matrix) && data.is_a?(Matrix)
     end
@@ -156,7 +270,5 @@ module XGBoost
     def handle_missing(data, missing)
       data.map! { |v| v.nil? ? missing : v }
     end
-
-    include Utils
   end
 end

data/lib/xgboost/early_stopping.rb

@@ -0,0 +1,132 @@
+module XGBoost
+  class EarlyStopping < TrainingCallback
+    def initialize(
+      rounds:,
+      metric_name: nil,
+      data_name: nil,
+      maximize: nil,
+      save_best: false,
+      min_delta: 0.0
+    )
+      @data = data_name
+      @metric_name = metric_name
+      @rounds = rounds
+      @save_best = save_best
+      @maximize = maximize
+      @stopping_history = {}
+      @min_delta = min_delta
+      if @min_delta < 0
+        raise ArgumentError, "min_delta must be greater or equal to 0."
+      end
+
+      @current_rounds = 0
+      @best_scores = {}
+      @starting_round = 0
+      super()
+    end
+
+    def before_training(model)
+      @starting_round = model.num_boosted_rounds
+      model
+    end
+
+    def after_iteration(model, epoch, evals_log)
+      epoch += @starting_round
+      msg = "Must have at least 1 validation dataset for early stopping."
+      if evals_log.keys.length < 1
+        raise ArgumentError, msg
+      end
+
+      # Get data name
+      if @data
+        data_name = @data
+      else
+        # Use the last one as default.
+        data_name = evals_log.keys[-1]
+      end
+      if !evals_log.include?(data_name)
+        raise ArgumentError, "No dataset named: #{data_name}"
+      end
+
+      if !data_name.is_a?(String)
+        raise TypeError, "The name of the dataset should be a string. Got: #{data_name.class.name}"
+      end
+      data_log = evals_log[data_name]
+
+      # Get metric name
+      if @metric_name
+        metric_name = @metric_name
+      else
+        # Use last metric by default.
+        metric_name = data_log.keys[-1]
+      end
+      if !data_log.include?(metric_name)
+        raise ArgumentError, "No metric named: #{metric_name}"
+      end
+
+      # The latest score
+      score = data_log[metric_name][-1]
+      update_rounds(
+        score, data_name, metric_name, model, epoch
+      )
+    end
+
+    def after_training(model)
+      if !@save_best
+        return model
+      end
+
+      best_iteration = model.best_iteration
+      best_score = model.best_score
+      # model = model[..(best_iteration + 1)]
+      model.best_iteration = best_iteration
+      model.best_score = best_score
+      model
+    end
+
+    private
+
+    def update_rounds(score, name, metric, model, epoch)
+      get_s = lambda do |value|
+        value.is_a?(Array) ? value[0] : value
+      end
+
+      maximize = lambda do |new_, best|
+        get_s.(new_) - @min_delta > get_s.(best)
+      end
+
+      minimize = lambda do |new_, best|
+        get_s.(best) - @min_delta > get_s.(new_)
+      end
+
+      improve_op = @maximize ? maximize : minimize
+
+      if @stopping_history.empty?
+        # First round
+        @current_rounds = 0
+        @stopping_history[name] = {}
+        @stopping_history[name][metric] = [score]
+        @best_scores[name] = {}
+        @best_scores[name][metric] = [score]
+        model.set_attr(best_score: get_s.(score), best_iteration: epoch)
+      elsif !improve_op.(score, @best_scores[name][metric][-1])
+        # Not improved
+        @stopping_history[name][metric] << score
+        @current_rounds += 1
+      else
+        # Improved
+        @stopping_history[name][metric] << score
+        @best_scores[name][metric] << score
+        record = @stopping_history[name][metric][-1]
+        model.set_attr(best_score: get_s.(record), best_iteration: epoch)
+        @current_rounds = 0
+      end
+
+      if @current_rounds >= @rounds
+        # Should stop
+        return true
+      end
+      false
+    end
+  end
+end

data/lib/xgboost/evaluation_monitor.rb

@@ -0,0 +1,44 @@
+module XGBoost
+  class EvaluationMonitor < TrainingCallback
+    def initialize(period:, show_stdv: false)
+      @show_stdv = show_stdv
+      @period = period
+    end
+
+    def after_iteration(model, epoch, evals_log)
+      if evals_log.empty?
+        return false
+      end
+
+      msg = "[#{epoch}]"
+      evals_log.each do |data, metric|
+        metric.each do |metric_name, log|
+          stdv = nil
+          if log[-1].is_a?(Array)
+            score = log[-1][0]
+            stdv = log[-1][1]
+          else
+            score = log[-1]
+          end
+          msg += fmt_metric(data, metric_name, score, stdv)
+        end
+      end
+      msg += "\n"
+
+      if epoch % @period == 0
+        puts msg
+      end
+      false
+    end
+
+    private
+
+    def fmt_metric(data, metric, score, std)
+      if !std.nil? && @show_stdv
+        "\t%s:%.5f+%.5f" % [data + "-" + metric, score, std]
+      else
+        "\t%s:%.5f" % [data + "-" + metric, score]
+      end
+    end
+  end
+end

data/lib/xgboost/ffi.rb

@@ -22,8 +22,12 @@ module XGBoost
     # dmatrix
     attach_function :XGDMatrixCreateFromMat, %i[pointer uint64 uint64 float pointer], :int
     attach_function :XGDMatrixSetUIntInfo, %i[pointer string pointer uint64], :int
+    attach_function :XGDMatrixSetStrFeatureInfo, %i[pointer string pointer uint64], :int
+    attach_function :XGDMatrixGetStrFeatureInfo, %i[pointer string pointer pointer], :int
     attach_function :XGDMatrixNumRow, %i[pointer pointer], :int
     attach_function :XGDMatrixNumCol, %i[pointer pointer], :int
+    attach_function :XGDMatrixNumNonMissing, %i[pointer pointer], :int
+    attach_function :XGDMatrixDataSplitMode, %i[pointer pointer], :int
     attach_function :XGDMatrixSliceDMatrix, %i[pointer pointer uint64 pointer], :int
     attach_function :XGDMatrixFree, %i[pointer], :int
     attach_function :XGDMatrixSaveBinary, %i[pointer string int], :int
@@ -35,13 +39,19 @@ module XGBoost
     attach_function :XGBoosterUpdateOneIter, %i[pointer int pointer], :int
     attach_function :XGBoosterEvalOneIter, %i[pointer int pointer pointer uint64 pointer], :int
     attach_function :XGBoosterFree, %i[pointer], :int
+    attach_function :XGBoosterReset, %i[pointer], :int
+    attach_function :XGBoosterBoostedRounds, %i[pointer pointer], :int
     attach_function :XGBoosterSetParam, %i[pointer string string], :int
+    attach_function :XGBoosterGetNumFeature, %i[pointer pointer], :int
     attach_function :XGBoosterPredict, %i[pointer pointer int int int pointer pointer], :int
     attach_function :XGBoosterLoadModel, %i[pointer string], :int
     attach_function :XGBoosterSaveModel, %i[pointer string], :int
+    attach_function :XGBoosterSaveJsonConfig, %i[pointer pointer pointer], :int
     attach_function :XGBoosterDumpModelExWithFeatures, %i[pointer int pointer pointer int string pointer pointer], :int
-    attach_function :XGBoosterGetAttr, %i[pointer pointer pointer pointer], :int
-    attach_function :XGBoosterSetAttr, %i[pointer pointer pointer], :int
+    attach_function :XGBoosterGetAttr, %i[pointer string pointer pointer], :int
+    attach_function :XGBoosterSetAttr, %i[pointer string string], :int
     attach_function :XGBoosterGetAttrNames, %i[pointer pointer pointer], :int
+    attach_function :XGBoosterSetStrFeatureInfo, %i[pointer string pointer uint64], :int
+    attach_function :XGBoosterGetStrFeatureInfo, %i[pointer string pointer pointer], :int
   end
 end

data/lib/xgboost/model.rb

@@ -2,10 +2,11 @@ module XGBoost
   class Model
     attr_reader :booster
 
-    def initialize(n_estimators: 100, importance_type: "gain", **options)
+    def initialize(n_estimators: 100, importance_type: "gain", early_stopping_rounds: nil, **options)
       @params = options
       @n_estimators = n_estimators
       @importance_type = importance_type
+      @early_stopping_rounds = early_stopping_rounds
     end
 
     def predict(data)

data/lib/xgboost/packed_booster.rb

@@ -0,0 +1,51 @@
+module XGBoost
+  class PackedBooster
+    def initialize(cvfolds)
+      @cvfolds = cvfolds
+    end
+
+    def update(iteration)
+      @cvfolds.each do |fold|
+        fold.update(iteration)
+      end
+    end
+
+    def set_attr(**kwargs)
+      @cvfolds.each do |f|
+        f.bst.set_attr(**kwargs)
+      end
+    end
+
+    def attr(key)
+      @cvfolds[0].bst.attr(key)
+    end
+
+    def eval_set(iteration)
+      @cvfolds.map { |f| f.eval_set(iteration) }
+    end
+
+    def best_iteration
+      @cvfolds[0].bst.best_iteration
+    end
+
+    def best_iteration=(iteration)
+      @cvfolds.each do |fold|
+        fold.best_iteration = iteration
+      end
+    end
+
+    def best_score
+      @cvfolds[0].bst.best_score
+    end
+
+    def best_score=(score)
+      @cvfolds.each do |fold|
+        fold.best_score = score
+      end
+    end
+
+    def num_boosted_rounds
+      @cvfolds[0].num_boosted_rounds
+    end
+  end
+end

data/lib/xgboost/regressor.rb

@@ -10,7 +10,7 @@ module XGBoost
 
       @booster = XGBoost.train(@params, dtrain,
         num_boost_round: @n_estimators,
-        early_stopping_rounds: early_stopping_rounds,
+        early_stopping_rounds: early_stopping_rounds || @early_stopping_rounds,
         verbose_eval: verbose,
         evals: evals
       )

data/lib/xgboost/training_callback.rb

@@ -0,0 +1,23 @@
+module XGBoost
+  class TrainingCallback
+    def before_training(model)
+      # Run before training starts
+      model
+    end
+
+    def after_training(model)
+      # Run after training is finished
+      model
+    end
+
+    def before_iteration(model, epoch, evals_log)
+      # Run before each iteration. Returns true when training should stop.
+      false
+    end
+
+    def after_iteration(model, epoch, evals_log)
+      # Run after each iteration. Returns true when training should stop.
+      false
+    end
+  end
+end

data/lib/xgboost/utils.rb

@@ -2,7 +2,7 @@ module XGBoost
   module Utils
     private
 
-    def check_result(err)
+    def check_call(err)
       if err != 0
         # make friendly
         message = FFI.XGBGetLastError.split("\n").first.split(/:\d+: /, 2).last
@@ -10,9 +10,24 @@ module XGBoost
       end
     end
 
-    # read_uint64 not available on JRuby
-    def read_uint64(ptr)
-      ptr.read_array_of_uint64(1).first
+    def array_of_pointers(values)
+      arr = ::FFI::MemoryPointer.new(:pointer, values.size)
+      arr.write_array_of_pointer(values)
+      # keep reference for string pointers
+      arr.instance_variable_set(:@xgboost_ref, values)
+      arr
+    end
+
+    def string_pointer(value)
+      ::FFI::MemoryPointer.from_string(value.to_s)
+    end
+
+    def from_cstr_to_rbstr(data, length)
+      res = []
+      length.read_uint64.times do |i|
+        res << data.read_pointer[i * ::FFI::Pointer.size].read_pointer.read_string.force_encoding(Encoding::UTF_8)
+      end
+      res
     end
   end
 end

data/lib/xgboost/version.rb

@@ -1,3 +1,3 @@
 module XGBoost
-  VERSION = "0.8.0"
+  VERSION = "0.10.0"
 end